From a058fd8e45f6ef4c804a5f75b2f22a023cd4126b Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Tue, 27 Oct 2026 13:54:04 -0500 Subject: [PATCH 01/13] feat: record metric inputs and run metadata during generation Generation, annotation and retargeting now record what dataset metrics need, without computing any metric: - Every producer records step_success from the task's success term. Generation also records the per-step EEF and commanded target poses, but not object poses, which annotation keeps for the source pool. - Each exported dataset gets an autodata entry in env_args: run id, exported count, pose-action layout, robot joint names with simulator limits, and optional datasheet hardware_limits from the embodiment. - --result_file adds a run block (run id, seed, revision, input digests, hardware, setup and generation time) and an attempt log, one line per attempt with its failure cause and algorithm extras. - GenerationAlgorithm.attempt_extras() lets an algorithm report per-attempt facts; SkillGen reports its motion-planning time. --- autodata_core/algorithms.py | 17 ++- autodata_core/data_generator.py | 25 ++++- .../embodiments/franka_ik_rel.yaml | 13 +++ .../embodiments/franka_ik_rel_skillgen.yaml | 13 +++ .../absolute_pose_embodiment_adapter.py | 9 ++ .../bimanual_embodiment_adapter.py | 8 ++ .../embodiments/embodiment_adapter.py | 9 ++ autodata_interfaces/embodiments/factory.py | 3 +- .../single_arm_embodiment_adapter.py | 6 ++ autodata_interfaces/env/__init__.py | 2 + .../env/datagen_info_recorder.py | 79 ++++++++++++++ .../env/isaaclab_env_interface.py | 37 ++++++- autodata_retargeting/runner.py | 9 +- autodata_tests/core/test_data_generator.py | 17 ++- autodata_tests/e2e/test_annotation.py | 4 +- .../e2e/test_mimicgen_data_generation.py | 6 +- .../e2e/test_skillgen_bin_data_generation.py | 5 +- autodata_tests/utils/utils.py | 41 +++++++ autodata_utils/dataset_metadata.py | 38 +++++++ autodata_utils/generation_result.py | 51 +++++++++ scripts/annotate_demos.py | 61 +++++------ scripts/generate_dataset.py | 101 +++++++++++++++++- .../references/cli-reference.md | 2 +- 23 files changed, 507 insertions(+), 49 deletions(-) create mode 100644 autodata_interfaces/env/datagen_info_recorder.py create mode 100644 autodata_utils/dataset_metadata.py diff --git a/autodata_core/algorithms.py b/autodata_core/algorithms.py index 65d6daf..8c969be 100644 --- a/autodata_core/algorithms.py +++ b/autodata_core/algorithms.py @@ -15,7 +15,8 @@ from __future__ import annotations -from collections.abc import Iterator +import time +from collections.abc import Iterable, Iterator from typing import TYPE_CHECKING, Any if TYPE_CHECKING: @@ -71,6 +72,14 @@ def validate_setup(self, datastream: Datastream) -> None: or ``datastream.get_env()`` (escape hatch) to enforce algorithm-specific invariants. """ + def attempt_extras(self, eef_states: Iterable[_EEFGenerationState]) -> dict[str, Any]: + """Algorithm-specific facts about one finished attempt, written to its attempt-log line. Default: none. + + Args: + eef_states: Per-EEF generation state of the attempt. + """ + return {} + def plan_subtask_trajectory( self, *, @@ -190,6 +199,10 @@ def __init__(self, motion_planners: dict[int, Any]) -> None: assert motion_planners, "SkillGen requires at least one motion planner" self.motion_planners = motion_planners + def attempt_extras(self, eef_states: Iterable[_EEFGenerationState]) -> dict[str, Any]: + """Report the attempt's wall-clock motion-planning time [s] as ``planning_time_s``.""" + return {"planning_time_s": sum(state.planning_time_s for state in eef_states)} + def plan_subtask_trajectory( self, *, @@ -238,6 +251,7 @@ def plan_subtask_trajectory( ) planner = self.motion_planners[env_id] + start_time = time.perf_counter() planning_success = planner.update_world_and_plan_motion( target_pose=target_pose, expected_attached_object=expected_attached_object, @@ -245,6 +259,7 @@ def plan_subtask_trajectory( step_size=getattr(planner, "step_size", None), enable_retiming=(hasattr(planner, "step_size") and planner.step_size is not None), ) + eef_state.planning_time_s += time.perf_counter() - start_time if not planning_success: return None diff --git a/autodata_core/data_generator.py b/autodata_core/data_generator.py index 59481cb..59d1e84 100644 --- a/autodata_core/data_generator.py +++ b/autodata_core/data_generator.py @@ -15,7 +15,7 @@ import torch from copy import deepcopy from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Literal from isaaclab.managers import TerminationTermCfg @@ -47,10 +47,21 @@ async def _optional_lock(lock: asyncio.Lock | None): @dataclass(frozen=True) class GenerationResult: - """Output of one :meth:`DataGenerator.generate` invocation.""" + """Output of one :meth:`DataGenerator.generate` invocation. + + Attributes: + initial_state: Scene state of every env right after this env's reset. + success: Whether the task succeeded at any step of the attempt. + failure_cause: ``"planner"`` when motion planning aborted the attempt, otherwise ``"execution"`` + for a failed attempt and None on success. + extras: Algorithm-specific facts about the attempt, from + :meth:`~autodata_core.algorithms.GenerationAlgorithm.attempt_extras`. + """ initial_state: dict success: bool + failure_cause: Literal["planner", "execution"] | None = None + extras: dict[str, Any] = field(default_factory=dict) @dataclass @@ -76,6 +87,7 @@ class _EEFGenerationState: # subtask trajectory here so the follow-up call splices it in. pending_subtask_trajectory: WaypointTrajectory | None = None is_in_motion_plan: bool = False + planning_time_s: float = 0.0 class DataGenerator: @@ -569,7 +581,12 @@ async def generate( ) if result is None: # Planning failure (e.g. SkillGen motion planner). Abort with no success. - return GenerationResult(initial_state=initial_state, success=False) + return GenerationResult( + initial_state=initial_state, + success=False, + failure_cause="planner", + extras=self.algorithm.attempt_extras(eef_states.values()), + ) next_trajectory, _is_motion_plan = result eef_state.current_trajectory = next_trajectory eef_state.subtask_step_index = 0 @@ -620,6 +637,8 @@ async def generate( return GenerationResult( initial_state=initial_state, success=buffers.success, + failure_cause=None if buffers.success else "execution", + extras=self.algorithm.attempt_extras(eef_states.values()), ) # ------------------------------------------------------------------ diff --git a/autodata_examples/embodiments/franka_ik_rel.yaml b/autodata_examples/embodiments/franka_ik_rel.yaml index 2ad1e97..3194ae2 100644 --- a/autodata_examples/embodiments/franka_ik_rel.yaml +++ b/autodata_examples/embodiments/franka_ik_rel.yaml @@ -31,3 +31,16 @@ action_layout: # that same frame, so no shift is needed. SkillGen's cube-stack dataset is annotated in the # panda_hand frame, so it needs [0, 0, 0.1034] — use franka_cube_stack_embodiment_skillgen.yaml. eef_offset: [0.0, 0.0, 0.0] + +# Franka FCI datasheet joint limits, keyed by joint-name pattern. Read only by the dataset metrics. +hardware_limits: + velocity: # [rad/s] + panda_joint[1-4]: 2.175 + panda_joint[5-7]: 2.61 + acceleration: # [rad/s^2] + panda_joint1: 15.0 + panda_joint2: 7.5 + panda_joint3: 10.0 + panda_joint4: 12.5 + panda_joint5: 15.0 + panda_joint[6-7]: 20.0 diff --git a/autodata_examples/embodiments/franka_ik_rel_skillgen.yaml b/autodata_examples/embodiments/franka_ik_rel_skillgen.yaml index 5e27381..618a461 100644 --- a/autodata_examples/embodiments/franka_ik_rel_skillgen.yaml +++ b/autodata_examples/embodiments/franka_ik_rel_skillgen.yaml @@ -35,3 +35,16 @@ action_layout: # approach axis) while the SkillGen dataset is annotated in the panda_hand frame, so the adapter # shifts reports back to panda_hand to keep planned transits and recorded skill segments aligned. eef_offset: [0.0, 0.0, 0.1034] + +# Franka FCI datasheet joint limits, keyed by joint-name pattern. Read only by the dataset metrics. +hardware_limits: + velocity: # [rad/s] + panda_joint[1-4]: 2.175 + panda_joint[5-7]: 2.61 + acceleration: # [rad/s^2] + panda_joint1: 15.0 + panda_joint2: 7.5 + panda_joint3: 10.0 + panda_joint4: 12.5 + panda_joint5: 15.0 + panda_joint[6-7]: 20.0 diff --git a/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py b/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py index c1d6c09..33cc04c 100644 --- a/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py +++ b/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py @@ -77,6 +77,15 @@ def from_dict(cls, data: dict[str, Any]) -> AbsolutePoseEmbodimentAdapter: """Build the adapter from a validated descriptor.""" return cls(**data) + def metric_context(self) -> dict[str, Any]: + """Adds the pose-action columns of every EEF that commands a pose; they are not clipped.""" + pose_action = { + name: {"columns": list(spec["pose_action_slice"]), "bound": None} + for name, spec in self.eefs.items() + if spec.get("pose_action_slice") is not None + } + return {**super().metric_context(), "pose_action": pose_action} + def get_eef_names(self) -> tuple[str, ...]: return tuple(self.eefs) diff --git a/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py b/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py index 206cc86..ad698e4 100644 --- a/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py +++ b/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py @@ -191,6 +191,14 @@ def action_dim(self) -> int: channel_ends = [channel_slice[1] for channel_slice in self.passthrough_channels.values()] return max([self.hand_joints_slice[1], *channel_ends]) + def metric_context(self) -> dict[str, Any]: + """Adds the pose-action columns of both hands; they are not clipped.""" + pose_action = { + self.left.name: {"columns": list(self.left_pose_slice), "bound": None}, + self.right.name: {"columns": list(self.right_pose_slice), "bound": None}, + } + return {**super().metric_context(), "pose_action": pose_action} + def action_to_target_eef_pose(self, action: torch.Tensor) -> dict[str, torch.Tensor]: """Convert env action to target EEF poses for both arms. diff --git a/autodata_interfaces/embodiments/embodiment_adapter.py b/autodata_interfaces/embodiments/embodiment_adapter.py index f230bfe..a219c6b 100644 --- a/autodata_interfaces/embodiments/embodiment_adapter.py +++ b/autodata_interfaces/embodiments/embodiment_adapter.py @@ -50,6 +50,15 @@ class EmbodimentAdapter(ABC): robot_asset_name: str = "robot" raw_config: dict[str, Any] | None = None + def metric_context(self) -> dict[str, Any]: + """Facts about this embodiment that the dataset metrics read, stored with every dataset it records. + + Subclasses add their pose-action layout. The base adds the optional ``hardware_limits`` section of the + embodiment config (datasheet joint limits, keyed by joint-name pattern). + """ + hardware_limits = (self.raw_config or {}).get("hardware_limits") + return {"hardware_limits": hardware_limits} if hardware_limits else {} + def get_navigation_eef_name(self) -> str | None: """Return the configured navigation channel without requiring a live environment.""" diff --git a/autodata_interfaces/embodiments/factory.py b/autodata_interfaces/embodiments/factory.py index fb1cc98..6267e59 100644 --- a/autodata_interfaces/embodiments/factory.py +++ b/autodata_interfaces/embodiments/factory.py @@ -70,7 +70,8 @@ def embodiment_adapter_from_dict(data: dict[str, Any], env: Any = None) -> Embod type_key in EMBODIMENT_TYPE_REGISTRY ), f"Unknown embodiment type {type_key!r}. Registered: {sorted(EMBODIMENT_TYPE_REGISTRY)}" cls = EMBODIMENT_TYPE_REGISTRY[type_key] - payload = {k: v for k, v in data.items() if k != "type"} + # ``hardware_limits`` is read from ``raw_config`` by :meth:`EmbodimentAdapter.metric_context`. + payload = {k: v for k, v in data.items() if k not in ("type", "hardware_limits")} adapter = cls.from_dict(payload) # type: ignore[attr-defined] adapter.raw_config = data # the full parsed config, for sections no typed adapter field models if env is not None: diff --git a/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py b/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py index 2c332a0..3b25dec 100644 --- a/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py +++ b/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py @@ -133,6 +133,12 @@ class DeltaPoseIKSingleArmAdapter(SingleArmEmbodimentAdapter): clip_pose_action_to_unit: bool = True + def metric_context(self) -> dict[str, Any]: + """Adds the pose-action columns and their clip bound.""" + bound = 1.0 if self.clip_pose_action_to_unit else None + pose_action = {self.eef_name: {"columns": [0, _DELTA_POSE_ACTION_DIM], "bound": bound}} + return {**super().metric_context(), "pose_action": pose_action} + @property def action_dim(self) -> int: """Total width of the env action vector.""" diff --git a/autodata_interfaces/env/__init__.py b/autodata_interfaces/env/__init__.py index a673b88..b75c594 100644 --- a/autodata_interfaces/env/__init__.py +++ b/autodata_interfaces/env/__init__.py @@ -6,6 +6,7 @@ from autodata_interfaces.env.env_profile import EnvironmentProfile from autodata_interfaces.env.isaaclab_env_interface import ( apply_env_profile, + collect_autodata_entry, env_loop, get_env_name_from_dataset, setup_env_config, @@ -18,5 +19,6 @@ "get_env_name_from_dataset", "setup_output_paths", "env_loop", + "collect_autodata_entry", "setup_env_config", ] diff --git a/autodata_interfaces/env/datagen_info_recorder.py b/autodata_interfaces/env/datagen_info_recorder.py new file mode 100644 index 0000000..15bb80b --- /dev/null +++ b/autodata_interfaces/env/datagen_info_recorder.py @@ -0,0 +1,79 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Recorder terms that store the per-step data annotation, generation and the dataset metrics read.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from isaaclab.envs.mdp.recorders.recorders_cfg import ActionStateRecorderManagerCfg +from isaaclab.managers import RecorderTerm, RecorderTermCfg, TerminationTermCfg +from isaaclab.utils import configclass + +if TYPE_CHECKING: + from autodata_interfaces.datastream import Datastream + + +class PreStepDatagenInfoRecorder(RecorderTerm): + """Records end-effector and commanded target poses, and optionally object poses, before each step. + + Attributes: + datastream: Datastream to read. Set it on the class once the Datastream exists, since Isaac Lab + copies recorder configurations when building the env. + """ + + datastream: Datastream | None = None + + def record_pre_step(self): + assert self.datastream is not None, "Set PreStepDatagenInfoRecorder.datastream before stepping the env." + datagen_info = { + "eef_pose": self.datastream.embodiment_adapter.get_eef_poses(env_ids=None), + "target_eef_pose": self.datastream.action_to_target_eef_pose(self._env.action_manager.action), + } + if self.cfg.record_object_poses: + datagen_info["object_pose"] = self.datastream.get_object_poses() + return "obs/datagen_info", datagen_info + + +@configclass +class PreStepDatagenInfoRecorderCfg(RecorderTermCfg): + """Configuration for :class:`PreStepDatagenInfoRecorder`.""" + + class_type: type[RecorderTerm] = PreStepDatagenInfoRecorder + record_object_poses: bool = True + """Annotation needs object poses for the source pool; generation leaves them out, as no metric reads them.""" + + +class PostStepSuccessRecorder(RecorderTerm): + """Records whether the task's success term holds after each step, as ``step_success``.""" + + def record_post_step(self): + success_term = self.cfg.success_term + return "step_success", success_term.func(self._env, **success_term.params) + + +@configclass +class PostStepSuccessRecorderCfg(RecorderTermCfg): + """Configuration for :class:`PostStepSuccessRecorder`.""" + + class_type: type[RecorderTerm] = PostStepSuccessRecorder + success_term: TerminationTermCfg | None = None + """The task's success term; the env itself runs without termination terms.""" + + +@configclass +class DatagenInfoRecorderManagerCfg(ActionStateRecorderManagerCfg): + """Action/state recorder plus the per-step ``obs/datagen_info`` term.""" + + record_pre_step_datagen_info = PreStepDatagenInfoRecorderCfg() + + +def add_step_success_recorder(recorder_cfg: object, success_term: TerminationTermCfg) -> None: + """Add the ``step_success`` term to any recorder manager configuration. + + Args: + recorder_cfg: Recorder manager configuration the env will be built with. + success_term: The task's success term. + """ + recorder_cfg.record_post_step_success = PostStepSuccessRecorderCfg(success_term=success_term) diff --git a/autodata_interfaces/env/isaaclab_env_interface.py b/autodata_interfaces/env/isaaclab_env_interface.py index 3a8f916..80754f0 100644 --- a/autodata_interfaces/env/isaaclab_env_interface.py +++ b/autodata_interfaces/env/isaaclab_env_interface.py @@ -18,7 +18,7 @@ import contextlib import os import torch -from typing import Any +from typing import TYPE_CHECKING, Any import isaaclab.sim as sim_utils from isaaclab.assets import RigidObjectCfg @@ -32,9 +32,43 @@ from isaaclab_physx.sim.schemas import PhysxRigidBodyPropertiesCfg from isaaclab_tasks.utils.parse_cfg import parse_env_cfg +from autodata_interfaces.env.datagen_info_recorder import add_step_success_recorder from autodata_interfaces.env.env_profile import EnvironmentProfile, convert_event_params from autodata_interfaces.env.pink_ik_action import PinkInverseKinematicsActionSharedUrdf from autodata_interfaces.tasks.generation_policy_spec import GenerationPolicy +from autodata_utils.dataset_metadata import AUTODATA_ENTRY_VERSION +from autodata_utils.tensor_utils import as_torch + +if TYPE_CHECKING: + from autodata_interfaces.embodiments.embodiment_adapter import EmbodimentAdapter + + +def collect_autodata_entry(env: Any, embodiment_adapter: EmbodimentAdapter, run_id: str) -> dict[str, Any]: + """The ``autodata`` entry of the datasets this env exports, read while the env is alive. + + It holds the run id, the embodiment's metric context, and the robot's joint names with env 0's simulator + position limits [rad or m] and velocity limits [rad/s or m/s]. Unbounded limits are stored as None. + :func:`~autodata_utils.dataset_metadata.merge_autodata_entry` writes it once the env has closed the file. + + Args: + env: Live environment. + embodiment_adapter: Adapter of the embodiment the env drives. + run_id: Identifier shared by everything this run writes. + """ + entry: dict[str, Any] = {"version": AUTODATA_ENTRY_VERSION, "run_id": run_id, **embodiment_adapter.metric_context()} + robot = env.scene.articulations.get(embodiment_adapter.robot_asset_name) + if robot is not None: + entry["joints"] = { + "names": list(robot.joint_names), + "position_limits": _finite_or_none(as_torch(robot.data.joint_pos_limits)[0].cpu()), + "velocity_limits": _finite_or_none(as_torch(robot.data.joint_vel_limits)[0].cpu()), + } + return entry + + +def _finite_or_none(values: torch.Tensor) -> list: + """Nested lists of ``values`` with non-finite entries as None, so the JSON stays standard.""" + return [_finite_or_none(row) if row.dim() else (row.item() if torch.isfinite(row) else None) for row in values] def setup_output_paths(output_file_path: str) -> tuple[str, str]: @@ -231,6 +265,7 @@ def setup_env_config( env_cfg.recorders = ActionStateRecorderManagerCfg() else: env_cfg.recorders = recorder_cfg + add_step_success_recorder(env_cfg.recorders, success_term) env_cfg.recorders.dataset_export_dir_path = output_dir env_cfg.recorders.dataset_filename = output_file_name diff --git a/autodata_retargeting/runner.py b/autodata_retargeting/runner.py index f1e9ddb..e4a05d2 100644 --- a/autodata_retargeting/runner.py +++ b/autodata_retargeting/runner.py @@ -7,13 +7,15 @@ import numpy as np import random import torch +import uuid from copy import deepcopy from isaaclab.utils.datasets import HDF5DatasetFileHandler from autodata_interfaces.embodiments import embodiment_adapter_from_yaml -from autodata_interfaces.env import setup_env_config, setup_output_paths +from autodata_interfaces.env import collect_autodata_entry, setup_env_config, setup_output_paths from autodata_interfaces.tasks.generation_policy_spec import GenerationPolicy +from autodata_utils.dataset_metadata import exported_dataset_files, merge_autodata_entry from .config import RetargetConfig from .eef_offset import build_eef_offsets, compose_retarget_eef_offsets_from_adapters @@ -117,8 +119,10 @@ def run(args, simulation_app) -> int: torch.manual_seed(generation_policy.seed) dataset_file_handler = None + autodata_entry = None try: target_adapter.bind_env(env) + autodata_entry = collect_autodata_entry(env, target_adapter, uuid.uuid4().hex) env.reset() # Snapshot the target's home scene state once; each replay restores its robot from here # (deepcopy so later sim steps cannot mutate the cached tensors). @@ -180,3 +184,6 @@ def run(args, simulation_app) -> int: if dataset_file_handler is not None: dataset_file_handler.close() env.close() + if autodata_entry is not None: + for dataset_file in exported_dataset_files(output_dir, output_file_name): + merge_autodata_entry(dataset_file, autodata_entry) diff --git a/autodata_tests/core/test_data_generator.py b/autodata_tests/core/test_data_generator.py index 806c0e9..6667ca1 100644 --- a/autodata_tests/core/test_data_generator.py +++ b/autodata_tests/core/test_data_generator.py @@ -196,6 +196,21 @@ def test_navigation_does_not_skip_playback_after_cancellation_or_while_paused(pa assert all(state.subtask_step_index == 4 for state in states.values()) +def test_planning_failure_is_attributed_to_the_planner(monkeypatch): + generator = _navigation_generator(active=False, reached=True) + generator.datastream.env = SimpleNamespace(recorder_manager=Mock()) + monkeypatch.setattr(generator.algorithm, "plan_subtask_trajectory", lambda **kwargs: None) + monkeypatch.setattr(generator, "_reset_environment_for_generation", AsyncMock(return_value=(torch.tensor([0]), {}))) + + result = asyncio.run( + generator.generate( + env_id=0, success_term=None, env_reset_queue=asyncio.Queue(), env_action_queue=asyncio.Queue() + ) + ) + + assert result.success is False and result.failure_cause == "planner" + + def test_unresponsive_navigation_fails_and_finalizes_the_attempt(monkeypatch): generator = _navigation_generator(active=True, reached=False) generator.datastream.generation_policy.max_navigation_hold_steps = 3 @@ -219,7 +234,7 @@ def test_unresponsive_navigation_fails_and_finalizes_the_attempt(monkeypatch): ) ) - assert result.success is False + assert result.success is False and result.failure_cause == "execution" assert execute.await_count == 3 assert recorder.set_success_to_episodes.call_args.args[1].item() is False recorder.export_episodes.assert_called_once() diff --git a/autodata_tests/e2e/test_annotation.py b/autodata_tests/e2e/test_annotation.py index f5383d3..bedc99c 100644 --- a/autodata_tests/e2e/test_annotation.py +++ b/autodata_tests/e2e/test_annotation.py @@ -12,7 +12,7 @@ from autodata_tests.utils.constants import TestPaths from autodata_tests.utils.subprocess import run_subprocess -from autodata_tests.utils.utils import assert_valid_dataset +from autodata_tests.utils.utils import assert_recorded_episodes, assert_valid_dataset @pytest.mark.with_subprocess @@ -38,6 +38,8 @@ def test_automatic_annotation_from_dataset(tmp_path): str(output_file), ]) assert_valid_dataset(str(output_file), min_num_demos=1) + entry = assert_recorded_episodes(str(output_file)) + assert entry["annotation"]["annotated_episodes"] == entry["exported_episodes"] with h5py.File(input_file, "r") as source, h5py.File(output_file, "r") as annotated: assert len(annotated["data"]) == len(source["data"]) assert json.loads(annotated["data"].attrs["env_args"])["env_name"] == "IsaacContrib-Stack-Cube-Franka-IK-Rel" diff --git a/autodata_tests/e2e/test_mimicgen_data_generation.py b/autodata_tests/e2e/test_mimicgen_data_generation.py index cfb02b9..a97271e 100644 --- a/autodata_tests/e2e/test_mimicgen_data_generation.py +++ b/autodata_tests/e2e/test_mimicgen_data_generation.py @@ -8,7 +8,7 @@ from autodata_tests.utils.constants import TestPaths from autodata_tests.utils.subprocess import run_subprocess -from autodata_tests.utils.utils import assert_valid_dataset +from autodata_tests.utils.utils import assert_recorded_run, assert_valid_dataset HEADLESS = True GENERATION_NUM_TRIALS = 1 @@ -19,6 +19,7 @@ def _run_franka_cube_stack_mimicgen(num_envs: int, device: str) -> None: with tempfile.TemporaryDirectory() as temp_dir: output_file = os.path.join(temp_dir, "generated.hdf5") + result_file = os.path.join(temp_dir, "generation_result.json") args = [ TestPaths.python_path, @@ -35,6 +36,8 @@ def _run_franka_cube_stack_mimicgen(num_envs: int, device: str) -> None: os.path.join(TestPaths.test_data_dir, "annotated_dataset_franka_stack_mimicgen.hdf5"), "--output_file", output_file, + "--result_file", + result_file, "--generation_num_trials", str(GENERATION_NUM_TRIALS), "--num_envs", @@ -47,6 +50,7 @@ def _run_franka_cube_stack_mimicgen(num_envs: int, device: str) -> None: run_subprocess(args) assert_valid_dataset(output_file, min_num_demos=GENERATION_NUM_TRIALS) + assert_recorded_run(output_file, result_file) @pytest.mark.with_subprocess diff --git a/autodata_tests/e2e/test_skillgen_bin_data_generation.py b/autodata_tests/e2e/test_skillgen_bin_data_generation.py index 0627f18..90b5a17 100644 --- a/autodata_tests/e2e/test_skillgen_bin_data_generation.py +++ b/autodata_tests/e2e/test_skillgen_bin_data_generation.py @@ -17,7 +17,7 @@ from autodata_tests.utils.constants import TestPaths from autodata_tests.utils.subprocess import run_subprocess -from autodata_tests.utils.utils import assert_valid_dataset +from autodata_tests.utils.utils import assert_recorded_run, assert_valid_dataset HEADLESS = True GENERATION_NUM_TRIALS = 1 @@ -62,6 +62,9 @@ def _run_franka_bin_stack_skillgen(num_envs: int, device: str) -> None: run_subprocess(args) assert_valid_dataset(output_file, min_num_demos=GENERATION_NUM_TRIALS) + attempts = assert_recorded_run(output_file, result_file) + # Every SkillGen attempt plans at least one transit, and the algorithm reports how long it took. + assert all(attempt["extras"]["planning_time_s"] > 0 for attempt in attempts) with open(result_file, encoding="utf-8") as result_handle: result = json.load(result_handle) diff --git a/autodata_tests/utils/utils.py b/autodata_tests/utils/utils.py index f9cbde4..dd528a3 100644 --- a/autodata_tests/utils/utils.py +++ b/autodata_tests/utils/utils.py @@ -41,3 +41,44 @@ def read_generation_result(result_file: str) -> tuple[int, int]: assert num_attempts > 0, f"Generation result has invalid num_attempts: {num_attempts}." assert num_success + num_failures == num_attempts, "Generation result counts do not add up." return num_success, num_attempts + + +def assert_recorded_episodes(dataset_file: str) -> dict: + """Assert the per-step recording and the ``autodata`` entry agree with the dataset's episodes. + + Every episode has one ``step_success`` value per action, its success attribute equals whether + ``step_success`` ever held, and the entry counts every exported episode. + + Returns: + The dataset's ``autodata`` entry. + """ + + with h5py.File(dataset_file, "r") as h5_file: + data = h5_file["data"] + entry = json.loads(data.attrs["env_args"])["autodata"] + assert entry["exported_episodes"] == len(data), f"The entry counts {entry['exported_episodes']} episodes." + for name, episode in data.items(): + step_success = episode["step_success"][:] + assert len(step_success) == len(episode["actions"]), f"{name}: step_success and actions differ in length." + assert bool(step_success.any()) == bool(episode.attrs["success"]), f"{name}: step_success disagrees." + return entry + + +def assert_recorded_run(dataset_file: str, result_file: str) -> list[dict]: + """Assert a generation run's dataset, result JSON and attempt log describe the same run. + + Returns: + The attempt-log lines. + """ + + entry = assert_recorded_episodes(dataset_file) + with open(result_file, encoding="utf-8") as result_handle: + result = json.load(result_handle) + attempts_file = os.path.join(os.path.dirname(result_file), result["run"]["attempts_file"]) + with open(attempts_file, encoding="utf-8") as attempts_handle: + attempts = [json.loads(line) for line in attempts_handle if line.strip()] + assert entry["exported_episodes"] == result["num_success"], "The dataset and result count different successes." + assert len(attempts) == result["num_attempts"], "The attempt log and result count different attempts." + run_ids = {entry["run_id"], result["run"]["run_id"], *(attempt["run_id"] for attempt in attempts)} + assert len(run_ids) == 1, f"The dataset, result and attempt log carry different run ids: {run_ids}" + return attempts diff --git a/autodata_utils/dataset_metadata.py b/autodata_utils/dataset_metadata.py new file mode 100644 index 0000000..745077b --- /dev/null +++ b/autodata_utils/dataset_metadata.py @@ -0,0 +1,38 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The ``autodata`` entry of a dataset's ``env_args``: what the dataset metrics need to know about its making.""" + +from __future__ import annotations + +import h5py +import json +import os +from collections.abc import Mapping +from typing import Any + +AUTODATA_ENTRY_VERSION = 1 + + +def exported_dataset_files(output_dir: str, output_file_name: str) -> list[str]: + """The HDF5 files a recorder exported: the dataset and, when failed episodes were kept, its ``_failed`` file.""" + candidates = [os.path.join(output_dir, f"{output_file_name}{suffix}.hdf5") for suffix in ("", "_failed")] + return [path for path in candidates if os.path.exists(path)] + + +def merge_autodata_entry(dataset_path: str, entry: Mapping[str, Any]) -> None: + """Store ``entry`` under ``autodata`` in the dataset's ``env_args``, with the number of exported episodes. + + Call it after the env has closed the file. The episode count lets a reader detect a dataset merged from + several runs, since merging keeps the first file's ``env_args``. + + Args: + dataset_path: Exported HDF5 dataset. + entry: The entry, as built by :func:`~autodata_interfaces.env.collect_autodata_entry`. + """ + with h5py.File(dataset_path, "r+") as h5_file: + data = h5_file["data"] + raw_env_args = data.attrs.get("env_args", "{}") + env_args = json.loads(raw_env_args.decode() if isinstance(raw_env_args, bytes) else raw_env_args) + env_args["autodata"] = {**entry, "exported_episodes": len(data)} + data.attrs["env_args"] = json.dumps(env_args) diff --git a/autodata_utils/generation_result.py b/autodata_utils/generation_result.py index c38213d..090d4db 100644 --- a/autodata_utils/generation_result.py +++ b/autodata_utils/generation_result.py @@ -6,11 +6,15 @@ from __future__ import annotations import contextlib +import hashlib import json import os +import shutil +import subprocess import tempfile from collections.abc import Mapping from pathlib import Path +from typing import Any def write_generation_result( @@ -19,6 +23,7 @@ def write_generation_result( requested_trials: int, stats: Mapping[str, int], env_profile: Mapping[str, str | None] | None = None, + run_metadata: Mapping[str, Any] | None = None, ) -> None: """Atomically write the final outcome of a completed data-generation run. @@ -31,6 +36,7 @@ def write_generation_result( during generation, or None when the run used the unmodified base task. Recorded here because the generated dataset stores only the base env id and would otherwise not reveal that the scene was modified. + run_metadata: How the run was configured and how long it took, stored under ``run``. """ required_stats = ("num_success", "num_failures", "num_attempts") for stat_name in required_stats: @@ -51,6 +57,8 @@ def write_generation_result( } if env_profile is not None: result["env_profile"] = dict(env_profile) + if run_metadata is not None: + result["run"] = dict(run_metadata) file_descriptor, temporary_file_name = tempfile.mkstemp( prefix=f".{result_path.name}.", suffix=".tmp", dir=result_path.parent, text=True @@ -65,3 +73,46 @@ def write_generation_result( finally: with contextlib.suppress(FileNotFoundError): os.unlink(temporary_file_name) + + +def attempts_file_for(result_file: str) -> str: + """Path of the attempt log written next to ``result_file``.""" + return str(Path(result_file).with_suffix(".attempts.jsonl")) + + +class AttemptLog: + """Appends one JSON line per completed generation attempt, stamped with the run id.""" + + def __init__(self, path: str, run_id: str) -> None: + self._path = Path(path) + self._run_id = run_id + self._path.parent.mkdir(parents=True, exist_ok=True) + self._path.write_text("", encoding="utf-8") + + def write(self, record: Mapping[str, Any]) -> None: + with self._path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps({"run_id": self._run_id, **record}, sort_keys=True) + "\n") + + +def input_record(path: str | None) -> dict[str, str] | None: + """Path and SHA-256 of a run input, or None when the input was not given.""" + if path is None: + return None + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return {"path": path, "sha256": digest.hexdigest()} + + +def code_revision() -> str | None: + """Git commit of the AutoData checkout, or None without git or outside a checkout.""" + if shutil.which("git") is None: + return None + result = subprocess.run( + ["git", "-C", str(Path(__file__).resolve().parent), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=False, + ) + return result.stdout.strip() if result.returncode == 0 else None diff --git a/scripts/annotate_demos.py b/scripts/annotate_demos.py index b0d6b2b..d27984c 100644 --- a/scripts/annotate_demos.py +++ b/scripts/annotate_demos.py @@ -104,23 +104,33 @@ import math # noqa: E402 import os # noqa: E402 import sys # noqa: E402 +import time # noqa: E402 import torch # noqa: E402 import traceback # noqa: E402 +import uuid # noqa: E402 from collections.abc import Callable # noqa: E402 import isaaclab_tasks # noqa: F401, E402 (registers gym envs) from isaaclab.devices import Se3Keyboard, Se3KeyboardCfg # noqa: E402 -from isaaclab.envs.mdp.recorders.recorders_cfg import ActionStateRecorderManagerCfg # noqa: E402 -from isaaclab.managers import DatasetExportMode, RecorderTerm, RecorderTermCfg, TerminationTermCfg # noqa: E402 -from isaaclab.utils import configclass # noqa: E402 +from isaaclab.managers import DatasetExportMode, TerminationTermCfg # noqa: E402 from isaaclab.utils.datasets import EpisodeData, HDF5DatasetFileHandler # noqa: E402 from autodata_core.pool import DataGenInfoPool # noqa: E402 from autodata_interfaces.datastream import Datastream # noqa: E402 from autodata_interfaces.embodiments import embodiment_adapter_from_yaml # noqa: E402 -from autodata_interfaces.env import get_env_name_from_dataset, setup_env_config, setup_output_paths # noqa: E402 +from autodata_interfaces.env import ( # noqa: E402 + collect_autodata_entry, + get_env_name_from_dataset, + setup_env_config, + setup_output_paths, +) +from autodata_interfaces.env.datagen_info_recorder import ( # noqa: E402 + DatagenInfoRecorderManagerCfg, + PreStepDatagenInfoRecorder, +) from autodata_interfaces.env.registration import register_external_environment # noqa: E402 from autodata_interfaces.tasks.task_descriptor import TaskDescriptor # noqa: E402 +from autodata_utils.dataset_metadata import exported_dataset_files, merge_autodata_entry # noqa: E402 is_paused = False current_action_index = 0 @@ -151,34 +161,6 @@ def mark_subtask_cb() -> None: print(f"Marked a subtask signal at action index: {current_action_index}") -# Recorder term that writes per-step datagen info read through the Datastream. -class PreStepDatagenInfoRecorder(RecorderTerm): - """Recorder term that records ``obs/datagen_info`` from the Datastream on each step.""" - - def record_pre_step(self): - assert _datastream is not None, "Datastream must be initialized before recording." - datagen_info = { - "object_pose": _datastream.get_object_poses(), - "eef_pose": _datastream.embodiment_adapter.get_eef_poses(env_ids=None), - "target_eef_pose": _datastream.action_to_target_eef_pose(self._env.action_manager.action), - } - return "obs/datagen_info", datagen_info - - -@configclass -class PreStepDatagenInfoRecorderCfg(RecorderTermCfg): - """Configuration for :class:`PreStepDatagenInfoRecorder`.""" - - class_type: type[RecorderTerm] = PreStepDatagenInfoRecorder - - -@configclass -class AnnotationRecorderManagerCfg(ActionStateRecorderManagerCfg): - """Action/state recorder plus the datagen-info term used during annotation.""" - - record_pre_step_datagen_info = PreStepDatagenInfoRecorderCfg() - - def _build_signal_names( task_descriptor: TaskDescriptor, annotate_start_signals: bool, @@ -264,13 +246,14 @@ def main() -> int: num_envs=1, device=args_cli.device, generation_policy_params=generation_policy, - recorder_cfg=AnnotationRecorderManagerCfg(), + recorder_cfg=DatagenInfoRecorderManagerCfg(), enable_cameras=args_cli.record_images, ) # Only export episodes we explicitly mark successful (i.e. fully annotated). env_cfg.recorders.dataset_export_mode = DatasetExportMode.EXPORT_SUCCEEDED_ONLY env = gym.make(env_name, cfg=env_cfg).unwrapped + autodata_entry = None try: # Create the Datastream embodiment_adapter = embodiment_adapter_from_yaml(args_cli.embodiment) @@ -287,6 +270,8 @@ def main() -> int: source_pool=empty_pool, uses_start_signals=annotate_start_signals, ) + PreStepDatagenInfoRecorder.datastream = _datastream + autodata_entry = collect_autodata_entry(env, embodiment_adapter, uuid.uuid4().hex) env.reset() @@ -311,6 +296,7 @@ def main() -> int: exported_episode_count = 0 processed_episode_count = 0 + annotation_start = time.monotonic() with contextlib.suppress(KeyboardInterrupt), torch.inference_mode(): for episode_index, episode_name in enumerate(episode_names): if not simulation_app.is_running() or simulation_app.is_exiting(): @@ -342,10 +328,19 @@ def main() -> int: f"\nExported {exported_episode_count} (out of {processed_episode_count}) annotated" f" episode{'s' if exported_episode_count != 1 else ''} to {args_cli.output_file}." ) + autodata_entry["annotation"] = { + "input_episodes": len(episode_names), + "processed_episodes": processed_episode_count, + "annotated_episodes": exported_episode_count, + "time_s": time.monotonic() - annotation_start, + } print("Exiting the app.") return exported_episode_count finally: env.close() + if autodata_entry is not None: + for dataset_file in exported_dataset_files(output_dir, output_file_name): + merge_autodata_entry(dataset_file, autodata_entry) def replay_episode( diff --git a/scripts/generate_dataset.py b/scripts/generate_dataset.py index d6f5167..5a3bd48 100755 --- a/scripts/generate_dataset.py +++ b/scripts/generate_dataset.py @@ -33,6 +33,7 @@ """Launch Isaac Sim Simulator first.""" import argparse +import time from isaaclab.app import AppLauncher @@ -42,6 +43,8 @@ from autodata_interfaces.tasks.task_descriptor_utils import resolve_task_algorithm from autodata_utils.cli import callable_path +PROCESS_START = time.monotonic() + # Hardcoded to keep argparse importable without pulling in the heavy core package. # Add new algorithms here when registering them in autodata_core.algorithms. _ALG_CHOICES = ["mimicgen", "dexmimicgen", "skillgen"] @@ -97,7 +100,10 @@ "--result_file", type=str, default=None, - help="Optional JSON path for the final structured generation result.", + help=( + "Optional JSON path for the final structured generation result. An attempt log with one JSON " + "line per attempt is written next to it (.attempts.jsonl)." + ), ) parser.add_argument( "--pause_subtask", @@ -137,24 +143,41 @@ import asyncio # noqa: E402 import gymnasium as gym # noqa: E402 import numpy as np # noqa: E402 +import platform # noqa: E402 import random # noqa: E402 import sys # noqa: E402 import torch # noqa: E402 import traceback # noqa: E402 +import uuid # noqa: E402 +from pathlib import Path # noqa: E402 from typing import Any # noqa: E402 from autodata_core import DataGenerator, get_algorithm # noqa: E402 from autodata_core.algorithms import REGISTERED_ALGORITHMS # noqa: E402 +from autodata_core.data_generator import GenerationResult # noqa: E402 from autodata_interfaces.datastream import Datastream # noqa: E402 from autodata_interfaces.env import ( # noqa: E402 EnvironmentProfile, + collect_autodata_entry, env_loop, get_env_name_from_dataset, setup_env_config, setup_output_paths, ) +from autodata_interfaces.env.datagen_info_recorder import ( # noqa: E402 + DatagenInfoRecorderManagerCfg, + PreStepDatagenInfoRecorder, +) from autodata_interfaces.env.registration import load_callable, register_external_environment # noqa: E402 -from autodata_utils.generation_result import write_generation_result # noqa: E402 +from autodata_utils.dataset_metadata import exported_dataset_files, merge_autodata_entry # noqa: E402 +from autodata_utils.generation_result import ( # noqa: E402 + AttemptLog, + attempts_file_for, + code_revision, + input_record, + write_generation_result, +) +from autodata_utils.tensor_utils import as_torch # noqa: E402 async def run_data_generator( @@ -166,9 +189,11 @@ async def run_data_generator( success_term, pause_subtask: bool, stats: dict, + attempt_log: AttemptLog | None, ) -> None: - """Repeatedly call ``data_generator.generate`` and tally outcomes into ``stats``.""" + """Repeatedly call ``data_generator.generate``, tally outcomes into ``stats`` and log each attempt.""" while True: + attempt_start = time.monotonic() try: result = await data_generator.generate( env_id=env_id, @@ -187,6 +212,45 @@ async def run_data_generator( else: stats["num_failures"] += 1 stats["num_attempts"] += 1 + if attempt_log is not None: + attempt_log.write(_attempt_record(env_id, result, attempt_start, time.monotonic())) + + +def _attempt_record(env_id: int, result: GenerationResult, start_time: float, end_time: float) -> dict: + """One attempt-log line; times [s] are monotonic-clock readings.""" + return { + "env_id": env_id, + "success": result.success, + "failure_cause": result.failure_cause, + "start_time_s": start_time, + "end_time_s": end_time, + "initial_object_positions": { + name: as_torch(state["root_pose"])[env_id, :2].tolist() + for name, state in result.initial_state.get("rigid_object", {}).items() + }, + "extras": result.extras, + } + + +def _hardware(device: str) -> dict[str, str | None]: + """Model names of the GPU and CPU the run used; physics runs on the CPU when ``device`` is ``cpu``.""" + gpu = None + if torch.cuda.is_available(): + index = torch.device(device).index if device.startswith("cuda") else None + gpu = torch.cuda.get_device_name(torch.cuda.current_device() if index is None else index) + cpu = None + cpuinfo = Path("/proc/cpuinfo") + if cpuinfo.exists(): + model_lines = (line for line in cpuinfo.read_text().splitlines() if line.startswith("model name")) + cpu = next((line.split(":", 1)[1].strip() for line in model_lines), None) + return {"gpu": gpu, "num_gpus": 1 if gpu else 0, "cpu": cpu or platform.processor() or None} + + +def _generation_recorder_cfg() -> DatagenInfoRecorderManagerCfg: + """Actions and states, plus the per-step poses the dataset metrics read; object poses are left out.""" + recorder_cfg = DatagenInfoRecorderManagerCfg() + recorder_cfg.record_pre_step_datagen_info.record_object_poses = False + return recorder_cfg def build_datastream( @@ -216,6 +280,7 @@ def setup_async_generation( success_term, algorithm, pause_subtask: bool, + attempt_log: AttemptLog | None = None, ) -> dict: """Build a single :class:`DataGenerator` over ``datastream`` and ``num_envs`` async tasks.""" asyncio_event_loop = asyncio.get_event_loop() @@ -238,6 +303,7 @@ def setup_async_generation( success_term=success_term, pause_subtask=pause_subtask, stats=stats, + attempt_log=attempt_log, ) ) tasks.append(task) @@ -327,7 +393,7 @@ def main() -> None: num_envs=args_cli.num_envs, device=args_cli.device, generation_policy_params=generation_policy_params, - recorder_cfg=load_callable(args_cli.recorder_cfg)() if args_cli.recorder_cfg else None, + recorder_cfg=load_callable(args_cli.recorder_cfg)() if args_cli.recorder_cfg else _generation_recorder_cfg(), env_profile=env_profile, enable_cameras=args_cli.record_images, ) @@ -347,6 +413,9 @@ def main() -> None: task_descriptor=task_descriptor, embodiment_adapter=embodiment_adapter, ) + PreStepDatagenInfoRecorder.datastream = datastream + run_id = uuid.uuid4().hex + autodata_entry = collect_autodata_entry(env, embodiment_adapter, run_id) # SkillGen needs one curobo planner per env. Mimic/DexMimic take no kwargs. motion_planners: dict | None = None @@ -362,6 +431,8 @@ def main() -> None: alg_kwargs["motion_planners"] = motion_planners algorithm = get_algorithm(args_cli.alg, **alg_kwargs) + attempts_file = attempts_file_for(args_cli.result_file) if args_cli.result_file else None + attempt_log = AttemptLog(attempts_file, run_id) if attempts_file else None try: async_components = setup_async_generation( datastream=datastream, @@ -369,10 +440,12 @@ def main() -> None: success_term=success_term, algorithm=algorithm, pause_subtask=args_cli.pause_subtask, + attempt_log=attempt_log, ) data_gen_tasks = asyncio.ensure_future(asyncio.gather(*async_components["tasks"])) generation_completed = False + generation_start = time.monotonic() try: generation_completed = env_loop( env, @@ -383,6 +456,7 @@ def main() -> None: stats=async_components["stats"], data_gen_tasks=data_gen_tasks, ) + generation_end = time.monotonic() except asyncio.CancelledError: print("Async tasks cancelled.") finally: @@ -406,9 +480,28 @@ def main() -> None: if env_profile else None ), + run_metadata={ + "run_id": run_id, + "seed": generation_policy_params.seed, + "revision": code_revision(), + "inputs": { + "source": input_record(args_cli.input_file), + "embodiment": input_record(args_cli.embodiment), + "task_descriptor": input_record(args_cli.task_descriptor), + "env_profile": input_record(args_cli.env_profile), + }, + "device": args_cli.device, + **_hardware(args_cli.device), + "num_envs": args_cli.num_envs, + "setup_time_s": generation_start - PROCESS_START, + "generation_time_s": generation_end - generation_start, + "attempts_file": Path(attempts_file).name, + }, ) finally: env.close() + for dataset_file in exported_dataset_files(output_dir, output_file_name): + merge_autodata_entry(dataset_file, autodata_entry) if __name__ == "__main__": diff --git a/skills/autodata-generate-datasets/references/cli-reference.md b/skills/autodata-generate-datasets/references/cli-reference.md index 19a5e3b..cfb4183 100644 --- a/skills/autodata-generate-datasets/references/cli-reference.md +++ b/skills/autodata-generate-datasets/references/cli-reference.md @@ -25,7 +25,7 @@ YAML, live env, and HDF5 source dataset, then runs the selected generation algor | `--output_file ` | no | `./datasets/output_dataset.hdf5` | Destination for generated demos. | | `--generation_num_trials ` | no | `None` | Number of demos to generate. | | `--num_envs ` | no | `1` | Number of parallel simulation environments. | -| `--result_file ` | no | — | Where to write the generation result summary. | +| `--result_file ` | no | — | Where to write the generation result summary. An attempt log (`.attempts.jsonl`) is written next to it. | | `--pause_subtask` | no | off | Pause at subtask boundaries (debugging). | | `--visualize_plan` | no | off | Visualize the motion plan (SkillGen debugging). | From 96a15bde9133fcb641cde03bf28b1f57bb518f5f Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Tue, 27 Oct 2026 15:36:40 -0500 Subject: [PATCH 02/13] docs: describe what generation records per step, attempt and run --- docs/pages/concepts/data_generator.rst | 35 +++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/pages/concepts/data_generator.rst b/docs/pages/concepts/data_generator.rst index b182383..514111b 100644 --- a/docs/pages/concepts/data_generator.rst +++ b/docs/pages/concepts/data_generator.rst @@ -121,9 +121,15 @@ report progress and decide when to stop: Results and Statistics ---------------------- -Three layers report what happened: - -* **Per attempt** — ``GenerationResult``: ``success`` and ``initial_state``. +Four layers report what happened: + +* **Per step** — every exported episode records ``step_success``, whether the task's success term + held after each step. Generation also records the end-effector pose and its commanded target under + ``obs/datagen_info``; object poses are recorded there only during annotation, for the source pool. +* **Per attempt** — ``GenerationResult``: ``success``, ``failure_cause`` (``planner`` or + ``execution``), ``extras`` from the algorithm's ``attempt_extras()``, and ``initial_state``. With + ``--result_file``, each completed attempt is also written as one line of + ``.attempts.jsonl``. * **Per run, in memory** — the ``stats`` counters: ``num_success``, ``num_failures``, ``num_attempts`` (``num_success + num_failures = num_attempts``). * **Per run, on disk** — with ``--result_file``, a JSON sidecar is written atomically at the @@ -141,6 +147,20 @@ Three layers report what happened: "name": "franka_bin_stack", "path": "autodata_examples/env_profiles/franka_bin_stack.yaml", "planner": "franka_stack_cube_bin" + }, + "run": { + "run_id": "3f2a9c...", + "seed": 1, + "revision": "a058fd8...", + "inputs": { + "source": {"path": "datasets/source.hdf5", "sha256": "9b1c..."}, + "embodiment": {"path": "autodata_examples/embodiments/franka_ik_rel.yaml", "sha256": "4e07..."}, + "task_descriptor": {"path": "autodata_examples/tasks/franka_cube_stack.yaml", "sha256": "c2d8..."}, + "env_profile": null + }, + "device": "cuda:0", "gpu": "...", "num_gpus": 1, "cpu": "...", "num_envs": 1, + "setup_time_s": 22.3, "generation_time_s": 228.5, + "attempts_file": "generation_result.attempts.jsonl" } } @@ -148,6 +168,12 @@ Three layers report what happened: applied — the generated dataset itself records only the base env id, so the sidecar is what documents that the scene was modified. +Every exported dataset, whether from generation, annotation or retargeting, also carries an ``autodata`` +entry in its ``env_args`` attribute. The entry holds the ``run_id`` and the number of exported +episodes, the embodiment's pose-action layout, the robot's joint names with their simulator limits, +and the embodiment's optional datasheet ``hardware_limits``. The ``run_id`` ties the dataset to its +result file and attempt log. + The GenerationAlgorithm Plug-In ------------------------------- @@ -176,6 +202,9 @@ follow. Five class attributes declare how the algorithm differs from a vanilla M The shipped algorithms — ``mimicgen``, ``dexmimicgen``, ``skillgen`` — are described in :doc:`algorithms`. +An algorithm can also override ``attempt_extras()`` to report facts about each attempt in its +attempt-log line; SkillGen reports its motion-planning time as ``planning_time_s``. + The behavioral hook: ``plan_subtask_trajectory`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 5f3264bf2c967be453a9015be1d0d40511c27847 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Tue, 27 Oct 2026 14:31:50 -0500 Subject: [PATCH 03/13] feat: add the autodata_metrics registry and the run report Metrics are plain functions registered with @metric. Each declares the evidence it requires (an HDF5 path, an env_args, run or attempt-log field, or the source dataset); without it the report says which key is missing instead of computing anything. A metric that raises is reported as an error without hiding the others. The report refuses a dataset and run record whose run ids or exported counts differ. validate_dataset.py --report computes every registered metric, --metrics loads more from a module, and --gate fails the command when a gated metric is out of bounds, not applicable or at another version. Built-in run metrics: generation success rate (stop-corrected, Wilson interval), demos per GPU-hour, time to dataset, annotation pass rate, and the failure-share and planning-time diagnostics. --- autodata_metrics/__init__.py | 13 ++ autodata_metrics/evidence.py | 94 ++++++++++ autodata_metrics/registry.py | 104 +++++++++++ autodata_metrics/report.py | 167 ++++++++++++++++++ autodata_metrics/viability.py | 160 +++++++++++++++++ .../e2e/test_mimicgen_data_generation.py | 16 +- .../e2e/test_skillgen_bin_data_generation.py | 16 +- autodata_tests/metrics/__init__.py | 2 + autodata_tests/metrics/test_report.py | 89 ++++++++++ autodata_tests/metrics/test_viability.py | 28 +++ autodata_tests/utils/utils.py | 17 ++ scripts/validate_dataset.py | 39 +++- setup.py | 1 + 13 files changed, 742 insertions(+), 4 deletions(-) create mode 100644 autodata_metrics/__init__.py create mode 100644 autodata_metrics/evidence.py create mode 100644 autodata_metrics/registry.py create mode 100644 autodata_metrics/report.py create mode 100644 autodata_metrics/viability.py create mode 100644 autodata_tests/metrics/__init__.py create mode 100644 autodata_tests/metrics/test_report.py create mode 100644 autodata_tests/metrics/test_viability.py diff --git a/autodata_metrics/__init__.py b/autodata_metrics/__init__.py new file mode 100644 index 0000000..c9d8546 --- /dev/null +++ b/autodata_metrics/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Offline metrics of AutoData datasets and generation runs. + +Producers record evidence; :func:`build_report` computes every registered metric from it without a simulator. +Importing this package registers the built-in metrics. +""" + +from autodata_metrics import viability +from autodata_metrics.evidence import Episode, Evidence +from autodata_metrics.registry import AREAS, REGISTERED_METRICS, MetricSpec, NotApplicable, Value, metric +from autodata_metrics.report import build_report, format_report, gate_violations diff --git a/autodata_metrics/evidence.py b/autodata_metrics/evidence.py new file mode 100644 index 0000000..0cb1ed9 --- /dev/null +++ b/autodata_metrics/evidence.py @@ -0,0 +1,94 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""What a metric reads: a dataset's episodes and ``env_args``, and its generation run's record and attempt log.""" + +from __future__ import annotations + +import h5py +import json +import numpy as np +from collections.abc import Mapping +from typing import Any + +_MISSING = object() + + +def lookup(record: Any, dotted_key: str) -> Any: + """``record[a][b]`` for ``dotted_key`` ``"a.b"``, or a sentinel that :func:`is_present` rejects.""" + for part in dotted_key.split("."): + if not isinstance(record, Mapping) or part not in record: + return _MISSING + record = record[part] + return record + + +def is_present(value: Any) -> bool: + """Whether a :func:`lookup` found its key.""" + return value is not _MISSING + + +class Episode: + """One episode of a dataset; arrays are read when accessed.""" + + def __init__(self, group: h5py.Group) -> None: + self.name = group.name.rsplit("/", 1)[-1] + self._group = group + + def __contains__(self, key: str) -> bool: + return key in self._group + + def __getitem__(self, key: str) -> np.ndarray: + return self._group[key][()] + + +class Evidence: + """A dataset, and optionally the generation run that produced it, as metrics read them. + + Attributes: + episodes: The dataset's episodes. + env_args: The dataset's ``env_args``; its ``autodata`` entry was written by the producer. + result: Generation result, or None for a dataset without a run record or an interrupted run. + run: The result's ``run`` block, or empty. + attempts: Attempt-log lines, or None. + source: Evidence of the source dataset the run started from, or None. + """ + + def __init__( + self, + h5_file: h5py.File, + result: Mapping[str, Any] | None = None, + attempts: list[dict[str, Any]] | None = None, + source: Evidence | None = None, + ) -> None: + data = h5_file["data"] + raw_env_args = data.attrs.get("env_args", "{}") + self.env_args: dict[str, Any] = json.loads( + raw_env_args.decode() if isinstance(raw_env_args, bytes) else raw_env_args + ) + self.episodes = [Episode(group) for group in data.values()] + self.result = result + self.run: Mapping[str, Any] = (result or {}).get("run", {}) + self.attempts = attempts + self.source = source + + def has(self, key: str) -> bool: + """Whether the evidence key ``key`` is present. + + A key is one of: + + - an HDF5 path present in every episode, such as ``step_success``; + - ``env_args.`` or ``run.``, a dotted path into ``env_args`` or the run block; + - ``attempts``, the attempt log, or ``attempts.``, a dotted path present in every attempt line; + - ``source``, the source dataset, or ``source.``, a key of the source dataset. + """ + head, _, rest = key.partition(".") + if head == "source": + return self.source is not None and (not rest or self.source.has(rest)) + if head == "attempts": + return bool(self.attempts) and (not rest or all(is_present(lookup(line, rest)) for line in self.attempts)) + if head == "env_args": + return is_present(lookup(self.env_args, rest)) + if head == "run": + return is_present(lookup(self.run, rest)) + return bool(self.episodes) and all(key in episode for episode in self.episodes) diff --git a/autodata_metrics/registry.py b/autodata_metrics/registry.py new file mode 100644 index 0000000..c043c6f --- /dev/null +++ b/autodata_metrics/registry.py @@ -0,0 +1,104 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Registry of dataset and run metrics. + +A metric is a function of :class:`~autodata_metrics.evidence.Evidence` registered with :func:`metric`. It declares +the evidence it ``requires``; the runner reports it as not applicable, naming the missing key, instead of calling it +without that evidence. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Literal + +if TYPE_CHECKING: + from autodata_metrics.evidence import Evidence + +AREAS = ("validity", "diversity", "viability", "diagnostics") + + +class NotApplicable(Exception): + """Raised by a metric whose evidence is present but does not let it apply.""" + + +@dataclass(frozen=True) +class Value: + """What a metric computed. + + Attributes: + value: A number, or named numbers such as percentiles. + n: Sample count the value rests on. + details: Context a reader needs to interpret the value. + """ + + value: float | dict[str, float] + n: int + details: Mapping[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class MetricSpec: + """A registered metric and how to report it.""" + + name: str + compute: Callable[[Evidence], Value] + area: str + signal: str + scope: Literal["dataset", "run"] + unit: str + better: Literal["higher", "lower"] | None + requires: tuple[str, ...] + primary: str | None + version: int + + +REGISTERED_METRICS: dict[str, MetricSpec] = {} + + +def metric( + name: str, + *, + area: str, + signal: str, + scope: Literal["dataset", "run"], + unit: str, + better: Literal["higher", "lower"] | None = None, + requires: tuple[str, ...] | list[str] = (), + primary: str | None = None, + version: int = 1, +) -> Callable[[Callable[[Evidence], Value]], Callable[[Evidence], Value]]: + """Register the decorated function as the metric ``name``. + + Args: + name: Unique metric name, used in reports and gates. + area: One of :data:`AREAS`. + signal: What the metric measures within its area, e.g. ``"task correctness"``. + scope: ``"dataset"`` metrics read episodes; ``"run"`` metrics read attempts and the run record. + unit: Unit of the value. + better: Which direction is better, if either. + requires: Evidence keys; see :meth:`~autodata_metrics.evidence.Evidence.has`. + primary: For a dict value, the entry a gate checks by default. + version: Raise it whenever the definition changes, so gates pinned to the old one fail. + """ + assert area in AREAS, f"Metric {name!r} has area {area!r}; expected one of {AREAS}." + + def register(compute: Callable[[Evidence], Value]) -> Callable[[Evidence], Value]: + assert name not in REGISTERED_METRICS, f"Metric {name!r} is already registered." + REGISTERED_METRICS[name] = MetricSpec( + name=name, + compute=compute, + area=area, + signal=signal, + scope=scope, + unit=unit, + better=better, + requires=tuple(requires), + primary=primary, + version=version, + ) + return compute + + return register diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py new file mode 100644 index 0000000..efaf837 --- /dev/null +++ b/autodata_metrics/report.py @@ -0,0 +1,167 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Compute the registered metrics of a dataset and its run, render them, and check them against a gate.""" + +from __future__ import annotations + +import contextlib +import h5py +import json +import yaml +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from autodata_metrics.evidence import Evidence +from autodata_metrics.registry import AREAS, REGISTERED_METRICS, MetricSpec, NotApplicable +from autodata_utils.generation_result import attempts_file_for, code_revision, input_record + +SCHEMA_VERSION = 1 + + +def read_run(result_file: str) -> tuple[dict[str, Any] | None, list[dict[str, Any]] | None]: + """The generation result and its attempt log, each None when its file is missing. + + An interrupted run leaves an attempt log without a result. + """ + result_path = Path(result_file) + attempts_path = Path(attempts_file_for(result_file)) + result = json.loads(result_path.read_text(encoding="utf-8")) if result_path.exists() else None + attempts = None + if attempts_path.exists(): + lines = attempts_path.read_text(encoding="utf-8").splitlines() + attempts = [json.loads(line) for line in lines if line.strip()] + return result, attempts + + +def check_same_run(evidence: Evidence) -> list[str]: + """Refuse a dataset and run record that come from different runs; return warnings for unverifiable data. + + Run ids alone are not enough: merging datasets keeps the first file's ``env_args``, so the exported episode + count must match too. + """ + num_episodes = len(evidence.episodes) + if evidence.result is not None: + assert ( + num_episodes == evidence.result["num_success"] + ), f"The dataset holds {num_episodes} episodes but the run counts {evidence.result['num_success']} successes." + entry = evidence.env_args.get("autodata") + if entry is None: + return ["the dataset has no autodata entry, so it is matched to its run by episode count only"] + assert entry["exported_episodes"] == num_episodes, ( + f"The dataset holds {num_episodes} episodes but its run exported {entry['exported_episodes']};" + " it was merged or edited after export." + ) + run_ids = {entry["run_id"]} | {line["run_id"] for line in evidence.attempts or []} + if "run_id" in evidence.run: + run_ids.add(evidence.run["run_id"]) + assert len(run_ids) == 1, f"The dataset, result and attempt log come from different runs: {sorted(run_ids)}" + return [] + + +def evaluate(spec: MetricSpec, evidence: Evidence) -> dict[str, Any]: + """One report row: the metric's value, or why it is not applicable or failed.""" + row = { + "area": spec.area, + "signal": spec.signal, + "scope": spec.scope, + "unit": spec.unit, + "better": spec.better, + "primary": spec.primary, + "version": spec.version, + } + missing = next((key for key in spec.requires if not evidence.has(key)), None) + if missing is not None: + return {**row, "status": "not_applicable", "reason": f"missing {missing}"} + try: + value = spec.compute(evidence) + except NotApplicable as exc: + return {**row, "status": "not_applicable", "reason": str(exc)} + except Exception as exc: # A failing metric must not sink the report of every other metric. + return {**row, "status": "error", "reason": f"{type(exc).__name__}: {exc}"} + return {**row, "status": "ok", "value": value.value, "n": value.n, "details": dict(value.details)} + + +def build_report( + dataset: str, + result_file: str | None = None, + source: str | None = None, + metrics: Mapping[str, MetricSpec] | None = None, +) -> dict[str, Any]: + """Compute every metric for a dataset and, when ``result_file`` is given, its generation run. + + Args: + dataset: HDF5 dataset. + result_file: Generation result (``generate_dataset.py --result_file``); its attempt log is read beside it. + source: Source dataset the run started from. + metrics: Metrics to compute; defaults to every registered metric. + """ + result, attempts = read_run(result_file) if result_file else (None, None) + with contextlib.ExitStack() as stack: + source_evidence = Evidence(stack.enter_context(h5py.File(source, "r"))) if source else None + evidence = Evidence(stack.enter_context(h5py.File(dataset, "r")), result, attempts, source_evidence) + warnings = check_same_run(evidence) + rows = {name: evaluate(spec, evidence) for name, spec in (metrics or REGISTERED_METRICS).items()} + run_id = evidence.env_args.get("autodata", {}).get("run_id") + num_episodes = len(evidence.episodes) + return { + "schema_version": SCHEMA_VERSION, + "inputs": { + "dataset": input_record(dataset), + "result_file": result_file, + "source": input_record(source), + "run_id": run_id, + "revision": code_revision(), + }, + "num_episodes": num_episodes, + "warnings": warnings, + "metrics": rows, + } + + +def gate_violations(report: Mapping[str, Any], gate_file: str) -> list[str]: + """Bounds in ``gate_file`` that the report violates. + + The gate maps metric names to ``min``/``max`` bounds, an optional ``key`` into a dict value (default: the + metric's primary entry) and an optional ``version``. A gated metric that is missing, not applicable, errored or + at another version is a violation. + """ + violations = [] + for name, bound in yaml.safe_load(Path(gate_file).read_text(encoding="utf-8")).items(): + row = report["metrics"].get(name) + if row is None or row["status"] != "ok": + violations.append(f"{name}: {'not reported' if row is None else row['status']}") + continue + if "version" in bound and bound["version"] != row["version"]: + violations.append(f"{name}: version {row['version']}, gate written for version {bound['version']}") + continue + key = bound.get("key", row["primary"]) + value = row["value"][key] if key else row["value"] + assert not isinstance(value, Mapping), f"Gate entry {name!r} needs a key into {sorted(value)}." + if "min" in bound and value < bound["min"]: + violations.append(f"{name}: {value:.4g} is below {bound['min']}") + if "max" in bound and value > bound["max"]: + violations.append(f"{name}: {value:.4g} is above {bound['max']}") + return violations + + +def _format_value(value: Any) -> str: + if isinstance(value, Mapping): + return ", ".join(f"{key}={_format_value(item)}" for key, item in value.items()) or "none" + return f"{value:.3g}" if isinstance(value, float) else str(value) + + +def format_report(report: Mapping[str, Any]) -> str: + """The report as an aligned text table, grouped by area.""" + rows = [("Metric", "Area", "Value", "Unit", "n")] + for name, row in sorted(report["metrics"].items(), key=lambda item: (AREAS.index(item[1]["area"]), item[0])): + if row["status"] == "ok": + rows.append((name, row["area"], _format_value(row["value"]), row["unit"], str(row["n"]))) + else: + rows.append((name, row["area"], f"{row['status']}: {row['reason']}", "", "")) + widths = [max(len(row[column]) for row in rows) for column in range(len(rows[0]))] + lines = [" ".join(cell.ljust(width) for cell, width in zip(row, widths)).rstrip() for row in rows] + lines.insert(1, " ".join("-" * width for width in widths)) + lines += [f"warning: {warning}" for warning in report["warnings"]] + return "\n".join(lines) diff --git a/autodata_metrics/viability.py b/autodata_metrics/viability.py new file mode 100644 index 0000000..06446f5 --- /dev/null +++ b/autodata_metrics/viability.py @@ -0,0 +1,160 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Viability metrics: what producing the dataset cost, and diagnostics of why attempts failed.""" + +from __future__ import annotations + +import math +import numpy as np +from collections import Counter +from typing import Any + +from autodata_metrics.evidence import Evidence +from autodata_metrics.registry import NotApplicable, Value, metric + +Z_95 = 1.959963984540054 +MIN_ATTEMPTS_PER_ENV_FOR_STOP_CORRECTION = 3 +"""Below this many logged attempts in some env, the stop correction is itself biased.""" + + +def wilson_interval(successes: int, trials: int, z: float = Z_95) -> tuple[float, float]: + """Wilson score interval for a binomial proportion.""" + assert trials > 0, "trials must be positive" + p = successes / trials + denominator = 1 + z**2 / trials + center = (p + z**2 / (2 * trials)) / denominator + half_width = z * math.sqrt(p * (1 - p) / trials + z**2 / (4 * trials**2)) / denominator + return max(0.0, center - half_width), min(1.0, center + half_width) + + +def attempts_before_stop(attempts: list[dict[str, Any]], num_envs: int) -> list[dict[str, Any]] | None: + """Attempts that started before the first env completed its last logged attempt. + + Attempts running when generation stops are cancelled unlogged, and long attempts are the likelier to be running. + Each env's cancelled attempt started after that env's last completion, so this start-time subset was fully + logged. None when some env logged fewer than :data:`MIN_ATTEMPTS_PER_ENV_FOR_STOP_CORRECTION` attempts. + """ + last_end: dict[int, float] = {} + logged: Counter[int] = Counter() + for attempt in attempts: + last_end[attempt["env_id"]] = max(last_end.get(attempt["env_id"], 0.0), attempt["end_time_s"]) + logged[attempt["env_id"]] += 1 + if len(logged) < num_envs or min(logged.values()) < MIN_ATTEMPTS_PER_ENV_FOR_STOP_CORRECTION: + return None + cutoff = min(last_end.values()) + return [attempt for attempt in attempts if attempt["start_time_s"] < cutoff] + + +def _counted_attempts(ev: Evidence) -> tuple[list[dict[str, Any]], bool]: + """The stop-corrected attempts when the log allows the correction, else all attempts; and which one it is.""" + corrected = attempts_before_stop(ev.attempts, ev.run["num_envs"]) if "num_envs" in ev.run else None + return (corrected, True) if corrected is not None else (ev.attempts, False) + + +@metric( + "generation_success_rate", + area="viability", + signal="generation performance", + scope="run", + unit="fraction", + better="higher", + requires=["attempts"], +) +def generation_success_rate(ev: Evidence) -> Value: + attempts, corrected = _counted_attempts(ev) + successes = sum(attempt["success"] for attempt in attempts) + low, high = wilson_interval(successes, len(attempts)) + raw_rate = sum(attempt["success"] for attempt in ev.attempts) / len(ev.attempts) + details = {"interval_95": [low, high], "raw_rate": raw_rate, "stop_corrected": corrected} + return Value(successes / len(attempts), n=len(attempts), details=details) + + +@metric( + "demos_per_gpu_hour", + area="viability", + signal="generation performance", + scope="run", + unit="demos/GPU-h", + better="higher", + requires=["run.generation_time_s", "run.num_gpus", "env_args.autodata.exported_episodes"], +) +def demos_per_gpu_hour(ev: Evidence) -> Value: + if ev.run["num_gpus"] == 0: + raise NotApplicable("the run used no GPU") + gpu_hours = ev.run["generation_time_s"] / 3600 * ev.run["num_gpus"] + exported = ev.env_args["autodata"]["exported_episodes"] + details = {key: ev.run.get(key) for key in ("gpu", "cpu", "device", "num_envs")} + return Value(exported / gpu_hours, n=exported, details=details) + + +@metric( + "time_to_dataset", + area="viability", + signal="time to dataset", + scope="run", + unit="s", + better="lower", + requires=["run.setup_time_s", "run.generation_time_s"], +) +def time_to_dataset(ev: Evidence) -> Value: + annotation = ev.source.env_args.get("autodata", {}).get("annotation", {}) if ev.source else {} + parts = { + "annotation_s": annotation.get("time_s"), + "setup_s": ev.run["setup_time_s"], + "generation_s": ev.run["generation_time_s"], + } + details = {**parts, "note": "automated steps only; manual recording and annotation time is not measured"} + return Value(sum(seconds for seconds in parts.values() if seconds is not None), n=1, details=details) + + +@metric( + "annotation_pass_rate", + area="viability", + signal="time to dataset", + scope="run", + unit="fraction", + better="higher", + requires=["source.env_args.autodata.annotation"], +) +def annotation_pass_rate(ev: Evidence) -> Value: + annotation = ev.source.env_args["autodata"]["annotation"] + if annotation["processed_episodes"] == 0: + raise NotApplicable("annotation processed no episodes") + rate = annotation["annotated_episodes"] / annotation["processed_episodes"] + return Value(rate, n=annotation["processed_episodes"], details={"input_episodes": annotation["input_episodes"]}) + + +@metric( + "failure_share", + area="diagnostics", + signal="failure causes", + scope="run", + unit="fraction of failures", + requires=["attempts"], +) +def failure_share(ev: Evidence) -> Value: + attempts, corrected = _counted_attempts(ev) + causes = Counter(attempt["failure_cause"] for attempt in attempts if not attempt["success"]) + total = sum(causes.values()) + return Value( + {cause: count / total for cause, count in causes.items()}, n=total, details={"stop_corrected": corrected} + ) + + +@metric( + "planning_time", + area="diagnostics", + signal="generation performance", + scope="run", + unit="s per attempt", + better="lower", + primary="p95", + requires=["attempts.extras.planning_time_s"], +) +def planning_time(ev: Evidence) -> Value: + times = np.array([attempt["extras"]["planning_time_s"] for attempt in ev.attempts]) + p50, p95 = np.percentile(times, [50, 95]) + generation_time = ev.run.get("generation_time_s") + share = float(times.sum() / generation_time) if generation_time else None + return Value({"p50": float(p50), "p95": float(p95)}, n=len(times), details={"share_of_generation_time": share}) diff --git a/autodata_tests/e2e/test_mimicgen_data_generation.py b/autodata_tests/e2e/test_mimicgen_data_generation.py index a97271e..b43439d 100644 --- a/autodata_tests/e2e/test_mimicgen_data_generation.py +++ b/autodata_tests/e2e/test_mimicgen_data_generation.py @@ -8,10 +8,12 @@ from autodata_tests.utils.constants import TestPaths from autodata_tests.utils.subprocess import run_subprocess -from autodata_tests.utils.utils import assert_recorded_run, assert_valid_dataset +from autodata_tests.utils.utils import assert_ok_metrics, assert_recorded_run, assert_valid_dataset HEADLESS = True GENERATION_NUM_TRIALS = 1 +# Metrics a MimicGen run reported with its result file must compute. +RUN_METRICS = {"generation_success_rate", "demos_per_gpu_hour", "time_to_dataset", "failure_share"} def _run_franka_cube_stack_mimicgen(num_envs: int, device: str) -> None: @@ -51,6 +53,18 @@ def _run_franka_cube_stack_mimicgen(num_envs: int, device: str) -> None: assert_valid_dataset(output_file, min_num_demos=GENERATION_NUM_TRIALS) assert_recorded_run(output_file, result_file) + report_file = os.path.join(temp_dir, "report.json") + run_subprocess([ + TestPaths.python_path, + os.path.join(TestPaths.scripts_dir, "validate_dataset.py"), + output_file, + "--report", + "--result_file", + result_file, + "--report_file", + report_file, + ]) + assert_ok_metrics(report_file, RUN_METRICS) @pytest.mark.with_subprocess diff --git a/autodata_tests/e2e/test_skillgen_bin_data_generation.py b/autodata_tests/e2e/test_skillgen_bin_data_generation.py index 90b5a17..24814a7 100644 --- a/autodata_tests/e2e/test_skillgen_bin_data_generation.py +++ b/autodata_tests/e2e/test_skillgen_bin_data_generation.py @@ -17,10 +17,12 @@ from autodata_tests.utils.constants import TestPaths from autodata_tests.utils.subprocess import run_subprocess -from autodata_tests.utils.utils import assert_recorded_run, assert_valid_dataset +from autodata_tests.utils.utils import assert_ok_metrics, assert_recorded_run, assert_valid_dataset HEADLESS = True GENERATION_NUM_TRIALS = 1 +# Metrics a SkillGen run reported with its result file must compute; planning time is SkillGen's own. +RUN_METRICS = {"generation_success_rate", "demos_per_gpu_hour", "time_to_dataset", "failure_share", "planning_time"} def _run_franka_bin_stack_skillgen(num_envs: int, device: str) -> None: @@ -65,6 +67,18 @@ def _run_franka_bin_stack_skillgen(num_envs: int, device: str) -> None: attempts = assert_recorded_run(output_file, result_file) # Every SkillGen attempt plans at least one transit, and the algorithm reports how long it took. assert all(attempt["extras"]["planning_time_s"] > 0 for attempt in attempts) + report_file = os.path.join(temp_dir, "report.json") + run_subprocess([ + TestPaths.python_path, + os.path.join(TestPaths.scripts_dir, "validate_dataset.py"), + output_file, + "--report", + "--result_file", + result_file, + "--report_file", + report_file, + ]) + assert_ok_metrics(report_file, RUN_METRICS) with open(result_file, encoding="utf-8") as result_handle: result = json.load(result_handle) diff --git a/autodata_tests/metrics/__init__.py b/autodata_tests/metrics/__init__.py new file mode 100644 index 0000000..57c822a --- /dev/null +++ b/autodata_tests/metrics/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py new file mode 100644 index 0000000..f87bf11 --- /dev/null +++ b/autodata_tests/metrics/test_report.py @@ -0,0 +1,89 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests of how the report runs metrics, refuses mismatched runs and gates values.""" + +import h5py +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from autodata_metrics import MetricSpec, Value, build_report, gate_violations + + +def _spec(name: str, compute, requires: tuple[str, ...] = ()) -> MetricSpec: + return MetricSpec( + name=name, + compute=compute, + area="viability", + signal="test", + scope="run", + unit="fraction", + better="higher", + requires=requires, + primary=None, + version=1, + ) + + +def _dataset(path: Path, num_episodes: int = 1, entry: dict | None = None) -> str: + with h5py.File(path, "w") as h5_file: + data = h5_file.create_group("data") + data.attrs["env_args"] = json.dumps({"env_name": "Test-v0", **({"autodata": entry} if entry else {})}) + for index in range(num_episodes): + data.create_group(f"demo_{index}").create_dataset("actions", data=[[0.0, 1.0]]) + return str(path) + + +def _run(tmp_path: Path, run_id: str, num_success: int = 1) -> str: + result_file = tmp_path / "result.json" + result_file.write_text(json.dumps({"num_success": num_success, "run": {"run_id": run_id}})) + (tmp_path / "result.attempts.jsonl").write_text(json.dumps({"run_id": run_id, "success": True}) + "\n") + return str(result_file) + + +def test_missing_evidence_is_not_applicable_and_names_the_key(tmp_path): + report = build_report( + _dataset(tmp_path / "d.hdf5"), metrics={"m": _spec("m", lambda ev: Value(1.0, n=1), ("step_success",))} + ) + + assert report["metrics"]["m"]["status"] == "not_applicable" + assert report["metrics"]["m"]["reason"] == "missing step_success" + + +def test_failing_metric_is_an_error_and_the_rest_are_still_computed(tmp_path): + metrics = {"broken": _spec("broken", lambda ev: 1 / 0), "fine": _spec("fine", lambda ev: Value(0.5, n=2))} + + rows = build_report(_dataset(tmp_path / "d.hdf5"), metrics=metrics)["metrics"] + + assert rows["broken"]["status"] == "error" and "ZeroDivisionError" in rows["broken"]["reason"] + assert rows["fine"]["status"] == "ok" and rows["fine"]["value"] == 0.5 + + +def test_report_refuses_data_from_another_run_or_merged_data(tmp_path): + entry = {"run_id": "run-a", "exported_episodes": 1} + with pytest.raises(AssertionError, match="different runs"): + build_report(_dataset(tmp_path / "d.hdf5", entry=entry), result_file=_run(tmp_path, "run-b"), metrics={}) + with pytest.raises(AssertionError, match="merged or edited"): + build_report(_dataset(tmp_path / "merged.hdf5", num_episodes=2, entry=entry), metrics={}) + + +def test_gate_fails_on_bound_inapplicable_metric_and_outdated_version(tmp_path): + row = {"status": "ok", "value": 0.4, "primary": None, "version": 2} + report = {"metrics": {"low": row, "old": row, "fine": row, "absent": {"status": "not_applicable"}}} + gate = tmp_path / "gate.yaml" + gate.write_text("low: {min: 0.5}\nold: {version: 1}\nfine: {min: 0.3, max: 0.5, version: 2}\nabsent: {min: 0}\n") + + violations = gate_violations(report, str(gate)) + + assert [violation.split(":")[0] for violation in violations] == ["low", "old", "absent"] + + +def test_metrics_import_without_torch_or_isaaclab(): + code = "import sys, autodata_metrics; print(sorted({'torch', 'isaaclab', 'autodata_core'} & set(sys.modules)))" + output = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True).stdout + + assert output.strip() == "[]" diff --git a/autodata_tests/metrics/test_viability.py b/autodata_tests/metrics/test_viability.py new file mode 100644 index 0000000..ecaa78e --- /dev/null +++ b/autodata_tests/metrics/test_viability.py @@ -0,0 +1,28 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Known-answer tests of the viability statistics.""" + +import pytest + +from autodata_metrics.viability import attempts_before_stop, wilson_interval + + +def test_wilson_interval_matches_known_values(): + assert wilson_interval(0, 10) == pytest.approx((0.0, 0.2775), abs=1e-4) + assert wilson_interval(10, 10) == pytest.approx((0.7225, 1.0), abs=1e-4) + + +def _attempt(env_id: int, success: bool, start: float, duration: float) -> dict: + return {"env_id": env_id, "success": success, "start_time_s": start, "end_time_s": start + duration} + + +def test_attempts_before_stop_keeps_attempts_started_before_the_first_env_finished(): + long_successes = [_attempt(0, True, 4.0 * k, 4.0) for k in range(3)] # env 0 last finishes at 12 s + short_failures = [_attempt(1, False, 1.6 * k, 1.6) for k in range(10)] # one runs from 11.2 to 12.8 s + + kept = attempts_before_stop(long_successes + short_failures, num_envs=2) + + assert len(kept) == 11 and sum(attempt["success"] for attempt in kept) == 3 + assert attempts_before_stop(long_successes[:2] + short_failures, num_envs=2) is None + assert attempts_before_stop(short_failures, num_envs=2) is None diff --git a/autodata_tests/utils/utils.py b/autodata_tests/utils/utils.py index dd528a3..287137c 100644 --- a/autodata_tests/utils/utils.py +++ b/autodata_tests/utils/utils.py @@ -82,3 +82,20 @@ def assert_recorded_run(dataset_file: str, result_file: str) -> list[dict]: run_ids = {entry["run_id"], result["run"]["run_id"], *(attempt["run_id"] for attempt in attempts)} assert len(run_ids) == 1, f"The dataset, result and attempt log carry different run ids: {run_ids}" return attempts + + +def assert_ok_metrics(report_file: str, expected_ok: set[str]) -> dict: + """Assert exactly ``expected_ok`` metrics computed a value; any other metric must say why it did not. + + A renamed recorded key or an adapter that stops recording a stream turns a metric not applicable, which + this catches. + + Returns: + The report's metric rows. + """ + + with open(report_file, encoding="utf-8") as report_handle: + rows = json.load(report_handle)["metrics"] + ok = {name for name, row in rows.items() if row["status"] == "ok"} + assert ok == expected_ok, f"Computed {sorted(ok)}, expected {sorted(expected_ok)}: {rows}" + return rows diff --git a/scripts/validate_dataset.py b/scripts/validate_dataset.py index a53db2d..a341278 100644 --- a/scripts/validate_dataset.py +++ b/scripts/validate_dataset.py @@ -9,18 +9,22 @@ python scripts/validate_dataset.py [ ...] python scripts/validate_dataset.py directory/*.hdf5 + python scripts/validate_dataset.py --report --result_file [--gate ] -The command exits nonzero if any supplied file is invalid. +The command exits nonzero if any supplied file is invalid or a gated metric violates its bound. """ from __future__ import annotations import argparse import h5py +import importlib import json import os import sys +from autodata_metrics import build_report, format_report, gate_violations + REQUIRED_DEMO_FIELDS = { "actions": h5py.Dataset, "initial_state": h5py.Group, @@ -163,7 +167,17 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Validate AutoData demonstration HDF5 datasets.") parser.add_argument("files", nargs="+", help="HDF5 dataset file(s) to validate.") + parser.add_argument("--report", action="store_true", help="Also report the metrics of a single dataset.") + parser.add_argument("--result_file", help="Generation result (generate_dataset.py --result_file) of the dataset.") + parser.add_argument("--source", help="Source dataset the generation started from.") + parser.add_argument( + "--metrics", action="append", default=[], help="Module registering more metrics; may be repeated." + ) + parser.add_argument("--gate", help="YAML of metric bounds; a violated bound makes the command fail.") + parser.add_argument("--report_file", help="Also write the report as JSON to this file.") args = parser.parse_args(argv) + if args.report and len(args.files) != 1: + parser.error("--report takes exactly one dataset") results = [validate_file(path) for path in args.files] print_results(results) @@ -173,7 +187,28 @@ def main(argv: list[str] | None = None) -> int: print() print(f"All {total} file(s) valid." if num_invalid == 0 else f"{num_invalid} of {total} file(s) invalid.") - return 1 if num_invalid else 0 + # A run whose every attempt failed leaves an empty dataset; its run metrics are still worth reporting. + violations = [] + if args.report and results[0].num_episodes is not None: + for module in args.metrics: + importlib.import_module(module) + report = build_report(args.files[0], result_file=args.result_file, source=args.source) + print() + print(format_report(report)) + if args.report_file: + with open(args.report_file, "w", encoding="utf-8") as report_handle: + json.dump(report, report_handle, indent=2, default=_json_value) + report_handle.write("\n") + if args.gate: + violations = gate_violations(report, args.gate) + print("\nGate passed." if not violations else "\nGate failed:\n " + "\n ".join(violations)) + + return 1 if num_invalid or violations else 0 + + +def _json_value(value: object) -> object: + """JSON form of the numpy scalars and arrays that metric values may hold.""" + return value.tolist() if hasattr(value, "tolist") else str(value) if __name__ == "__main__": diff --git a/setup.py b/setup.py index 4c412ae..35b3a9f 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ include=[ "autodata_core*", "autodata_interfaces*", + "autodata_metrics*", "autodata_utils*", "autodata_examples*", ] From 2a5d4957fc5a07726d95b10327c003b679e6a0e0 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Tue, 27 Oct 2026 15:38:38 -0500 Subject: [PATCH 04/13] docs: add the metrics reference page with a table generated from the metric declarations --- autodata_metrics/viability.py | 6 +++ docs/_ext/autodata_doc_tools.py | 35 ++++++++++++ docs/index.rst | 1 + docs/pages/references/metrics.rst | 88 +++++++++++++++++++++++++++++++ 4 files changed, 130 insertions(+) create mode 100644 docs/pages/references/metrics.rst diff --git a/autodata_metrics/viability.py b/autodata_metrics/viability.py index 06446f5..62e91e9 100644 --- a/autodata_metrics/viability.py +++ b/autodata_metrics/viability.py @@ -62,6 +62,7 @@ def _counted_attempts(ev: Evidence) -> tuple[list[dict[str, Any]], bool]: requires=["attempts"], ) def generation_success_rate(ev: Evidence) -> Value: + """Successful attempts over attempts, stop-corrected when the attempt log allows, with a Wilson 95% interval.""" attempts, corrected = _counted_attempts(ev) successes = sum(attempt["success"] for attempt in attempts) low, high = wilson_interval(successes, len(attempts)) @@ -80,6 +81,7 @@ def generation_success_rate(ev: Evidence) -> Value: requires=["run.generation_time_s", "run.num_gpus", "env_args.autodata.exported_episodes"], ) def demos_per_gpu_hour(ev: Evidence) -> Value: + """Exported demonstrations per GPU-hour of generation time.""" if ev.run["num_gpus"] == 0: raise NotApplicable("the run used no GPU") gpu_hours = ev.run["generation_time_s"] / 3600 * ev.run["num_gpus"] @@ -98,6 +100,7 @@ def demos_per_gpu_hour(ev: Evidence) -> Value: requires=["run.setup_time_s", "run.generation_time_s"], ) def time_to_dataset(ev: Evidence) -> Value: + """Wall time of the automated steps: annotation of the source, setup and generation.""" annotation = ev.source.env_args.get("autodata", {}).get("annotation", {}) if ev.source else {} parts = { "annotation_s": annotation.get("time_s"), @@ -118,6 +121,7 @@ def time_to_dataset(ev: Evidence) -> Value: requires=["source.env_args.autodata.annotation"], ) def annotation_pass_rate(ev: Evidence) -> Value: + """Share of processed recordings that annotation exported into the source dataset.""" annotation = ev.source.env_args["autodata"]["annotation"] if annotation["processed_episodes"] == 0: raise NotApplicable("annotation processed no episodes") @@ -134,6 +138,7 @@ def annotation_pass_rate(ev: Evidence) -> Value: requires=["attempts"], ) def failure_share(ev: Evidence) -> Value: + """Share of failed attempts by cause, ``planner`` or ``execution``.""" attempts, corrected = _counted_attempts(ev) causes = Counter(attempt["failure_cause"] for attempt in attempts if not attempt["success"]) total = sum(causes.values()) @@ -153,6 +158,7 @@ def failure_share(ev: Evidence) -> Value: requires=["attempts.extras.planning_time_s"], ) def planning_time(ev: Evidence) -> Value: + """p50 and p95 motion-planning time per attempt, and its share of generation time.""" times = np.array([attempt["extras"]["planning_time_s"] for attempt in ev.attempts]) p50, p95 = np.percentile(times, [50, 95]) generation_time = ev.run.get("generation_time_s") diff --git a/docs/_ext/autodata_doc_tools.py b/docs/_ext/autodata_doc_tools.py index feb9f27..72db1fd 100644 --- a/docs/_ext/autodata_doc_tools.py +++ b/docs/_ext/autodata_doc_tools.py @@ -8,7 +8,9 @@ in one place and can switch between internal/external targets on release. """ +import ast import re +from pathlib import Path from typing import Any from sphinx.application import Sphinx @@ -69,8 +71,41 @@ def curobo_replacer(_: Any) -> str: source[0] = re.sub(r":docker_run_curobo:", curobo_replacer, source[0]) +def autodata_metrics_table(app: Sphinx, _: Any, source: list[str]) -> None: + """Replaces ``:autodata_metrics_table:`` with a table of the built-in metrics. + + The rows are read from the ``@metric`` declarations in ``autodata_metrics/`` instead of importing them, so the + docs build needs none of the metrics' dependencies and the table cannot drift from the code. + """ + if ":autodata_metrics_table:" not in source[0]: + return + rows = [] + for path in sorted((Path(app.confdir).parent / "autodata_metrics").glob("*.py")): + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): + if not isinstance(node, ast.FunctionDef): + continue + for decorator in node.decorator_list: + if isinstance(decorator, ast.Call) and getattr(decorator.func, "id", None) == "metric": + options = {keyword.arg: ast.literal_eval(keyword.value) for keyword in decorator.keywords} + requires = ", ".join(f"``{key}``" for key in options.get("requires", ())) or "-" + definition = (ast.get_docstring(node) or "").split("\n")[0] + name = ast.literal_eval(decorator.args[0]) + rows.append((options["area"], name, options["unit"], requires, definition)) + table = [ + ".. list-table::", + " :header-rows: 1", + " :widths: 20 12 14 26 28", + "", + " * - Metric\n - Area\n - Unit\n - Requires\n - Definition", + ] + for area, name, unit, requires, definition in sorted(rows): + table.append(f" * - ``{name}``\n - {area}\n - {unit}\n - {requires}\n - {definition}") + source[0] = source[0].replace(":autodata_metrics_table:", "\n".join(table) + "\n") + + def setup(app: Sphinx) -> None: app.connect("source-read", autodata_git_clone_code_block) app.connect("source-read", autodata_code_link) app.connect("source-read", docker_run_command_replacer) + app.connect("source-read", autodata_metrics_table) app.add_config_value("autodata_docs_config", {}, "env") diff --git a/docs/index.rst b/docs/index.rst index 6d4c9bf..853caf5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -146,5 +146,6 @@ Table of Contents :maxdepth: 1 :caption: References + pages/references/metrics pages/references/troubleshooting pages/references/release_notes diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst new file mode 100644 index 0000000..a2b8073 --- /dev/null +++ b/docs/pages/references/metrics.rst @@ -0,0 +1,88 @@ +Metrics +======= + +``scripts/validate_dataset.py --report`` computes the metrics of a dataset and, given its result +file, of the generation run that produced it. Metrics are computed from recorded files and need no +simulator. + +.. code-block:: bash + + python scripts/validate_dataset.py datasets/generated.hdf5 --report \ + --result_file datasets/generated.json \ + --source datasets/annotated_datasets/dataset_franka_annotated.hdf5 \ + --report_file datasets/generated_report.json + +Each metric reports one of three things: + +- a value, with its unit and sample count; +- ``not_applicable``, with the reason, such as a missing recorded input; +- ``error``, when it fails. The other metrics are still computed. + +The report refuses a dataset and a result file from different runs, because their run ids and exported +episode counts must match. + +Gating +------ + +``--gate`` reads metric bounds from YAML and makes the command fail when one is violated: + +.. code-block:: yaml + + generation_success_rate: {min: 0.3, version: 1} + planning_time: {max: 5.0, key: p95} + +A gated metric also violates the gate if it is not applicable, fails, or reports a different +``version``. + +Built-in metrics +---------------- + +:autodata_metrics_table: + +Adding a metric +--------------- + +A metric is a function registered with ``autodata_metrics.metric``. It names the evidence it +requires. Without that evidence, the report says which key is missing and does not call the function. + +.. code-block:: python + + import numpy as np + + from autodata_metrics import Value, metric + + + @metric( + "planning_time", + area="diagnostics", + signal="generation performance", + scope="run", + unit="s per attempt", + better="lower", + primary="p95", + requires=["attempts.extras.planning_time_s"], + ) + def planning_time(ev): + """p50 and p95 motion-planning time per attempt.""" + times = np.array([attempt["extras"]["planning_time_s"] for attempt in ev.attempts]) + p50, p95 = np.percentile(times, [50, 95]) + return Value({"p50": float(p50), "p95": float(p95)}, n=len(times)) + +An evidence key is one of: + +- an HDF5 path present in every episode, such as ``step_success``; +- ``env_args.`` or ``run.``; +- ``attempts`` or ``attempts.``; +- ``source`` or ``source.``. + +Built-in metrics live in ``autodata_metrics/``. Load your own with +``--metrics my_package.my_metrics``. Raise a metric's ``version`` whenever its definition changes. + +When you add an algorithm or an embodiment: + +- An algorithm reports per-attempt facts by overriding ``attempt_extras()``. Metrics that require + those facts then apply to its runs. +- An embodiment's adapter describes its pose actions through ``metric_context()``. Add datasheet + ``hardware_limits`` to its YAML to enable the joint-limit metrics. +- List the workflow's expected metrics in its end-to-end test, so a metric that stops applying fails + the test. From 5303c8fd8d6c2cfb48232214c74aaaf2836eeebd Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Tue, 27 Oct 2026 21:12:36 -0500 Subject: [PATCH 05/13] fix: keep earlier runs' records and label only this run's datasets - Each run writes its own attempt log, ..attempts.jsonl, so rerunning with the same --result_file keeps earlier logs, and an interrupted rerun leaves the earlier result and its log consistent. - A _failed dataset gets this run's autodata entry only when this run kept failed episodes, so one left by an earlier run is not relabelled. - The e2e helper checks the logged successes against the result. - Docs: only generated datasets have a result file and attempt log. --- autodata_retargeting/runner.py | 2 +- autodata_tests/utils/test_dataset_metadata.py | 14 ++++++++++++++ autodata_tests/utils/test_generation_result.py | 13 ++++++++++++- autodata_tests/utils/utils.py | 3 +++ autodata_utils/dataset_metadata.py | 16 +++++++++++++--- autodata_utils/generation_result.py | 6 +++--- docs/pages/concepts/data_generator.rst | 8 ++++---- scripts/annotate_demos.py | 2 +- scripts/generate_dataset.py | 6 +++--- .../references/cli-reference.md | 2 +- 10 files changed, 55 insertions(+), 17 deletions(-) create mode 100644 autodata_tests/utils/test_dataset_metadata.py diff --git a/autodata_retargeting/runner.py b/autodata_retargeting/runner.py index e4a05d2..cb82c9b 100644 --- a/autodata_retargeting/runner.py +++ b/autodata_retargeting/runner.py @@ -185,5 +185,5 @@ def run(args, simulation_app) -> int: dataset_file_handler.close() env.close() if autodata_entry is not None: - for dataset_file in exported_dataset_files(output_dir, output_file_name): + for dataset_file in exported_dataset_files(output_dir, output_file_name, generation_policy.keep_failed): merge_autodata_entry(dataset_file, autodata_entry) diff --git a/autodata_tests/utils/test_dataset_metadata.py b/autodata_tests/utils/test_dataset_metadata.py new file mode 100644 index 0000000..99fe7cf --- /dev/null +++ b/autodata_tests/utils/test_dataset_metadata.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +from autodata_utils.dataset_metadata import exported_dataset_files + + +def test_failed_file_left_by_an_earlier_run_is_not_exported_again(tmp_path: Path) -> None: + for name in ("generated.hdf5", "generated_failed.hdf5"): + (tmp_path / name).touch() + + assert exported_dataset_files(str(tmp_path), "generated", kept_failed=False) == [str(tmp_path / "generated.hdf5")] + assert len(exported_dataset_files(str(tmp_path), "generated", kept_failed=True)) == 2 diff --git a/autodata_tests/utils/test_generation_result.py b/autodata_tests/utils/test_generation_result.py index 31bb6b9..37fe93b 100644 --- a/autodata_tests/utils/test_generation_result.py +++ b/autodata_tests/utils/test_generation_result.py @@ -4,8 +4,9 @@ """Tests for structured data-generation result files.""" import json +from pathlib import Path -from autodata_utils.generation_result import write_generation_result +from autodata_utils.generation_result import AttemptLog, attempts_file_for, write_generation_result def test_write_generation_result_writes_completed_stats_atomically(tmp_path): @@ -52,3 +53,13 @@ def test_write_generation_result_records_env_profile_when_given(tmp_path): "path": "autodata_examples/env_profiles/franka_bin_stack.yaml", "planner": "franka_stack_cube_bin", } + + +def test_rerun_with_the_same_result_file_keeps_the_earlier_attempt_log(tmp_path: Path) -> None: + result_file = str(tmp_path / "result.json") + AttemptLog(attempts_file_for(result_file, "run-a"), "run-a").write({"success": True}) + + AttemptLog(attempts_file_for(result_file, "run-b"), "run-b") + + earlier = Path(attempts_file_for(result_file, "run-a")).read_text().splitlines() + assert [json.loads(line)["run_id"] for line in earlier] == ["run-a"] diff --git a/autodata_tests/utils/utils.py b/autodata_tests/utils/utils.py index dd528a3..c300e97 100644 --- a/autodata_tests/utils/utils.py +++ b/autodata_tests/utils/utils.py @@ -79,6 +79,9 @@ def assert_recorded_run(dataset_file: str, result_file: str) -> list[dict]: attempts = [json.loads(line) for line in attempts_handle if line.strip()] assert entry["exported_episodes"] == result["num_success"], "The dataset and result count different successes." assert len(attempts) == result["num_attempts"], "The attempt log and result count different attempts." + assert ( + sum(attempt["success"] for attempt in attempts) == result["num_success"] + ), "The attempt log and result count different successes." run_ids = {entry["run_id"], result["run"]["run_id"], *(attempt["run_id"] for attempt in attempts)} assert len(run_ids) == 1, f"The dataset, result and attempt log carry different run ids: {run_ids}" return attempts diff --git a/autodata_utils/dataset_metadata.py b/autodata_utils/dataset_metadata.py index 745077b..35ac6e9 100644 --- a/autodata_utils/dataset_metadata.py +++ b/autodata_utils/dataset_metadata.py @@ -14,9 +14,19 @@ AUTODATA_ENTRY_VERSION = 1 -def exported_dataset_files(output_dir: str, output_file_name: str) -> list[str]: - """The HDF5 files a recorder exported: the dataset and, when failed episodes were kept, its ``_failed`` file.""" - candidates = [os.path.join(output_dir, f"{output_file_name}{suffix}.hdf5") for suffix in ("", "_failed")] +def exported_dataset_files(output_dir: str, output_file_name: str, kept_failed: bool) -> list[str]: + """The HDF5 files this run's recorder exported. + + A ``_failed`` file counts only when this run kept failed episodes; otherwise one left by an earlier run would be + labelled with this run's id. + + Args: + output_dir: Export directory. + output_file_name: Dataset file name without the extension. + kept_failed: Whether the run exported failed episodes to a separate ``_failed`` file. + """ + suffixes = ("", "_failed") if kept_failed else ("",) + candidates = [os.path.join(output_dir, f"{output_file_name}{suffix}.hdf5") for suffix in suffixes] return [path for path in candidates if os.path.exists(path)] diff --git a/autodata_utils/generation_result.py b/autodata_utils/generation_result.py index 090d4db..fc5bc4e 100644 --- a/autodata_utils/generation_result.py +++ b/autodata_utils/generation_result.py @@ -75,9 +75,9 @@ def write_generation_result( os.unlink(temporary_file_name) -def attempts_file_for(result_file: str) -> str: - """Path of the attempt log written next to ``result_file``.""" - return str(Path(result_file).with_suffix(".attempts.jsonl")) +def attempts_file_for(result_file: str, run_id: str) -> str: + """Path of the run's attempt log, next to ``result_file``; each run writes its own, so a rerun keeps earlier logs.""" + return str(Path(result_file).with_suffix(f".{run_id}.attempts.jsonl")) class AttemptLog: diff --git a/docs/pages/concepts/data_generator.rst b/docs/pages/concepts/data_generator.rst index 514111b..787d88d 100644 --- a/docs/pages/concepts/data_generator.rst +++ b/docs/pages/concepts/data_generator.rst @@ -129,7 +129,7 @@ Four layers report what happened: * **Per attempt** — ``GenerationResult``: ``success``, ``failure_cause`` (``planner`` or ``execution``), ``extras`` from the algorithm's ``attempt_extras()``, and ``initial_state``. With ``--result_file``, each completed attempt is also written as one line of - ``.attempts.jsonl``. + ``..attempts.jsonl``. * **Per run, in memory** — the ``stats`` counters: ``num_success``, ``num_failures``, ``num_attempts`` (``num_success + num_failures = num_attempts``). * **Per run, on disk** — with ``--result_file``, a JSON sidecar is written atomically at the @@ -160,7 +160,7 @@ Four layers report what happened: }, "device": "cuda:0", "gpu": "...", "num_gpus": 1, "cpu": "...", "num_envs": 1, "setup_time_s": 22.3, "generation_time_s": 228.5, - "attempts_file": "generation_result.attempts.jsonl" + "attempts_file": "generation_result.3f2a9c.attempts.jsonl" } } @@ -171,8 +171,8 @@ Four layers report what happened: Every exported dataset, whether from generation, annotation or retargeting, also carries an ``autodata`` entry in its ``env_args`` attribute. The entry holds the ``run_id`` and the number of exported episodes, the embodiment's pose-action layout, the robot's joint names with their simulator limits, -and the embodiment's optional datasheet ``hardware_limits``. The ``run_id`` ties the dataset to its -result file and attempt log. +and the embodiment's optional datasheet ``hardware_limits``. For a generated dataset, the ``run_id`` +ties it to its result file and attempt log. The GenerationAlgorithm Plug-In ------------------------------- diff --git a/scripts/annotate_demos.py b/scripts/annotate_demos.py index d27984c..95d370f 100644 --- a/scripts/annotate_demos.py +++ b/scripts/annotate_demos.py @@ -339,7 +339,7 @@ def main() -> int: finally: env.close() if autodata_entry is not None: - for dataset_file in exported_dataset_files(output_dir, output_file_name): + for dataset_file in exported_dataset_files(output_dir, output_file_name, kept_failed=False): merge_autodata_entry(dataset_file, autodata_entry) diff --git a/scripts/generate_dataset.py b/scripts/generate_dataset.py index 5a3bd48..19791a5 100755 --- a/scripts/generate_dataset.py +++ b/scripts/generate_dataset.py @@ -102,7 +102,7 @@ default=None, help=( "Optional JSON path for the final structured generation result. An attempt log with one JSON " - "line per attempt is written next to it (.attempts.jsonl)." + "line per attempt is written next to it (..attempts.jsonl)." ), ) parser.add_argument( @@ -431,7 +431,7 @@ def main() -> None: alg_kwargs["motion_planners"] = motion_planners algorithm = get_algorithm(args_cli.alg, **alg_kwargs) - attempts_file = attempts_file_for(args_cli.result_file) if args_cli.result_file else None + attempts_file = attempts_file_for(args_cli.result_file, run_id) if args_cli.result_file else None attempt_log = AttemptLog(attempts_file, run_id) if attempts_file else None try: async_components = setup_async_generation( @@ -500,7 +500,7 @@ def main() -> None: ) finally: env.close() - for dataset_file in exported_dataset_files(output_dir, output_file_name): + for dataset_file in exported_dataset_files(output_dir, output_file_name, generation_policy_params.keep_failed): merge_autodata_entry(dataset_file, autodata_entry) diff --git a/skills/autodata-generate-datasets/references/cli-reference.md b/skills/autodata-generate-datasets/references/cli-reference.md index cfb4183..9062e4a 100644 --- a/skills/autodata-generate-datasets/references/cli-reference.md +++ b/skills/autodata-generate-datasets/references/cli-reference.md @@ -25,7 +25,7 @@ YAML, live env, and HDF5 source dataset, then runs the selected generation algor | `--output_file ` | no | `./datasets/output_dataset.hdf5` | Destination for generated demos. | | `--generation_num_trials ` | no | `None` | Number of demos to generate. | | `--num_envs ` | no | `1` | Number of parallel simulation environments. | -| `--result_file ` | no | — | Where to write the generation result summary. An attempt log (`.attempts.jsonl`) is written next to it. | +| `--result_file ` | no | — | Where to write the generation result summary. An attempt log (`..attempts.jsonl`) is written next to it. | | `--pause_subtask` | no | off | Pause at subtask boundaries (debugging). | | `--visualize_plan` | no | off | Visualize the motion plan (SkillGen debugging). | From 9ca575be99aea366823bcf7f15ddbd94e4d63627 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Tue, 27 Oct 2026 21:16:45 -0500 Subject: [PATCH 06/13] fix: report a run from the attempt log of the dataset's own run Each run now writes its own attempt log. The report reads the log named by the result when the result belongs to the dataset's run; otherwise it sets the result aside with a warning and reads the log named by the dataset's run id, so an interrupted rerun is still reported and runs are never mixed. --- autodata_metrics/evidence.py | 14 +++++++----- autodata_metrics/report.py | 33 +++++++++++++++++++-------- autodata_tests/metrics/test_report.py | 29 ++++++++++++++++++----- 3 files changed, 54 insertions(+), 22 deletions(-) diff --git a/autodata_metrics/evidence.py b/autodata_metrics/evidence.py index 0cb1ed9..972b628 100644 --- a/autodata_metrics/evidence.py +++ b/autodata_metrics/evidence.py @@ -23,6 +23,12 @@ def lookup(record: Any, dotted_key: str) -> Any: return record +def read_env_args(h5_file: h5py.File) -> dict[str, Any]: + """The dataset's ``env_args``; its ``autodata`` entry was written by the producer.""" + raw_env_args = h5_file["data"].attrs.get("env_args", "{}") + return json.loads(raw_env_args.decode() if isinstance(raw_env_args, bytes) else raw_env_args) + + def is_present(value: Any) -> bool: """Whether a :func:`lookup` found its key.""" return value is not _MISSING @@ -61,12 +67,8 @@ def __init__( attempts: list[dict[str, Any]] | None = None, source: Evidence | None = None, ) -> None: - data = h5_file["data"] - raw_env_args = data.attrs.get("env_args", "{}") - self.env_args: dict[str, Any] = json.loads( - raw_env_args.decode() if isinstance(raw_env_args, bytes) else raw_env_args - ) - self.episodes = [Episode(group) for group in data.values()] + self.env_args = read_env_args(h5_file) + self.episodes = [Episode(group) for group in h5_file["data"].values()] self.result = result self.run: Mapping[str, Any] = (result or {}).get("run", {}) self.attempts = attempts diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index efaf837..c11ac92 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -13,26 +13,38 @@ from pathlib import Path from typing import Any -from autodata_metrics.evidence import Evidence +from autodata_metrics.evidence import Evidence, read_env_args from autodata_metrics.registry import AREAS, REGISTERED_METRICS, MetricSpec, NotApplicable from autodata_utils.generation_result import attempts_file_for, code_revision, input_record SCHEMA_VERSION = 1 -def read_run(result_file: str) -> tuple[dict[str, Any] | None, list[dict[str, Any]] | None]: - """The generation result and its attempt log, each None when its file is missing. +def read_run( + result_file: str, run_id: str | None +) -> tuple[dict[str, Any] | None, list[dict[str, Any]] | None, list[str]]: + """The generation result and attempt log of run ``run_id``, each None when missing, and any warnings. - An interrupted run leaves an attempt log without a result. + A rerun with the same result file leaves the earlier result in place until it completes, so a result from + another run is set aside and the run is reported from its own attempt log. """ result_path = Path(result_file) - attempts_path = Path(attempts_file_for(result_file)) result = json.loads(result_path.read_text(encoding="utf-8")) if result_path.exists() else None + warnings = [] + if result is not None and run_id is not None and result.get("run", {}).get("run_id") != run_id: + warnings.append(f"{result_file} belongs to another run, so the run is reported from its attempt log") + result = None + if result is not None and "attempts_file" in result.get("run", {}): + attempts_path = result_path.parent / result["run"]["attempts_file"] + elif run_id is not None: + attempts_path = Path(attempts_file_for(result_file, run_id)) + else: + return result, None, warnings attempts = None if attempts_path.exists(): lines = attempts_path.read_text(encoding="utf-8").splitlines() attempts = [json.loads(line) for line in lines if line.strip()] - return result, attempts + return result, attempts, warnings def check_same_run(evidence: Evidence) -> list[str]: @@ -97,13 +109,14 @@ def build_report( source: Source dataset the run started from. metrics: Metrics to compute; defaults to every registered metric. """ - result, attempts = read_run(result_file) if result_file else (None, None) with contextlib.ExitStack() as stack: + h5_file = stack.enter_context(h5py.File(dataset, "r")) + run_id = read_env_args(h5_file).get("autodata", {}).get("run_id") + result, attempts, warnings = read_run(result_file, run_id) if result_file else (None, None, []) source_evidence = Evidence(stack.enter_context(h5py.File(source, "r"))) if source else None - evidence = Evidence(stack.enter_context(h5py.File(dataset, "r")), result, attempts, source_evidence) - warnings = check_same_run(evidence) + evidence = Evidence(h5_file, result, attempts, source_evidence) + warnings += check_same_run(evidence) rows = {name: evaluate(spec, evidence) for name, spec in (metrics or REGISTERED_METRICS).items()} - run_id = evidence.env_args.get("autodata", {}).get("run_id") num_episodes = len(evidence.episodes) return { "schema_version": SCHEMA_VERSION, diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py index f87bf11..7986217 100644 --- a/autodata_tests/metrics/test_report.py +++ b/autodata_tests/metrics/test_report.py @@ -38,10 +38,17 @@ def _dataset(path: Path, num_episodes: int = 1, entry: dict | None = None) -> st return str(path) -def _run(tmp_path: Path, run_id: str, num_success: int = 1) -> str: +def _log(tmp_path: Path, run_id: str, successes: list[bool]) -> None: + lines = [json.dumps({"run_id": run_id, "success": success}) for success in successes] + (tmp_path / f"result.{run_id}.attempts.jsonl").write_text("\n".join(lines) + "\n") + + +def _run(tmp_path: Path, run_id: str) -> str: + """A completed run: its result file and attempt log.""" result_file = tmp_path / "result.json" - result_file.write_text(json.dumps({"num_success": num_success, "run": {"run_id": run_id}})) - (tmp_path / "result.attempts.jsonl").write_text(json.dumps({"run_id": run_id, "success": True}) + "\n") + run = {"run_id": run_id, "attempts_file": f"result.{run_id}.attempts.jsonl"} + result_file.write_text(json.dumps({"num_success": 1, "num_attempts": 1, "run": run})) + _log(tmp_path, run_id, [True]) return str(result_file) @@ -63,10 +70,20 @@ def test_failing_metric_is_an_error_and_the_rest_are_still_computed(tmp_path): assert rows["fine"]["status"] == "ok" and rows["fine"]["value"] == 0.5 -def test_report_refuses_data_from_another_run_or_merged_data(tmp_path): +def test_interrupted_rerun_is_reported_from_its_own_attempt_log(tmp_path): + result_file = _run(tmp_path, "run-a") + _log(tmp_path, "run-b", [True, False]) + dataset = _dataset(tmp_path / "d.hdf5", entry={"run_id": "run-b", "exported_episodes": 1}) + count = {"attempts": _spec("attempts", lambda ev: Value(len(ev.attempts), n=1), ("attempts",))} + + report = build_report(dataset, result_file=result_file, metrics=count) + + assert report["metrics"]["attempts"]["value"] == 2 + assert "belongs to another run" in report["warnings"][0] + + +def test_report_refuses_merged_data(tmp_path): entry = {"run_id": "run-a", "exported_episodes": 1} - with pytest.raises(AssertionError, match="different runs"): - build_report(_dataset(tmp_path / "d.hdf5", entry=entry), result_file=_run(tmp_path, "run-b"), metrics={}) with pytest.raises(AssertionError, match="merged or edited"): build_report(_dataset(tmp_path / "merged.hdf5", num_episodes=2, entry=entry), metrics={}) From 5948fc060560f3674a14203602098a52fd4328f9 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Tue, 27 Oct 2026 21:33:20 -0500 Subject: [PATCH 07/13] docs: say the result file and attempt log come with --result_file --- docs/pages/concepts/data_generator.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/pages/concepts/data_generator.rst b/docs/pages/concepts/data_generator.rst index 787d88d..6fa9454 100644 --- a/docs/pages/concepts/data_generator.rst +++ b/docs/pages/concepts/data_generator.rst @@ -171,8 +171,8 @@ Four layers report what happened: Every exported dataset, whether from generation, annotation or retargeting, also carries an ``autodata`` entry in its ``env_args`` attribute. The entry holds the ``run_id`` and the number of exported episodes, the embodiment's pose-action layout, the robot's joint names with their simulator limits, -and the embodiment's optional datasheet ``hardware_limits``. For a generated dataset, the ``run_id`` -ties it to its result file and attempt log. +and the embodiment's optional datasheet ``hardware_limits``. For a dataset generated with +``--result_file``, the ``run_id`` ties it to that run's result file and attempt log. The GenerationAlgorithm Plug-In ------------------------------- From 77b511de37d24a0993432a7707738ea6fd4947c5 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Wed, 28 Oct 2026 13:51:26 -0500 Subject: [PATCH 08/13] feat: make metric recording opt-in and keep algorithm diagnostics in extras - Metric recording is off by default, so datasets and result files are unchanged unless --record_metric_inputs is given to generation, annotation or retargeting. It records the per-step poses and task success, the autodata entry and, with --result_file, the run block and attempt log. - GenerationResult no longer has failure_cause; SkillGen reports planning_failed in its attempt extras, next to planning_time_s. - metric_context() docstrings say what they record; the pose-action entry's columns key is now action_slice. - Docs: the run linkage holds once the run completes. --- autodata_core/algorithms.py | 9 ++- autodata_core/data_generator.py | 8 +-- .../absolute_pose_embodiment_adapter.py | 7 +- .../bimanual_embodiment_adapter.py | 9 ++- .../embodiments/embodiment_adapter.py | 8 ++- .../single_arm_embodiment_adapter.py | 7 +- .../env/isaaclab_env_interface.py | 6 +- autodata_retargeting/runner.py | 4 +- autodata_tests/core/test_algorithms.py | 24 +++++++ autodata_tests/core/test_data_generator.py | 17 +---- autodata_tests/e2e/test_annotation.py | 1 + .../e2e/test_dexmimicgen_data_generation.py | 3 +- .../e2e/test_mimicgen_data_generation.py | 1 + .../e2e/test_skillgen_bin_data_generation.py | 1 + autodata_tests/utils/utils.py | 11 +++ docs/pages/concepts/data_generator.rst | 28 ++++---- scripts/annotate_demos.py | 22 ++++-- scripts/generate_dataset.py | 69 ++++++++++++------- scripts/retarget_dataset.py | 5 ++ .../references/cli-reference.md | 3 +- 20 files changed, 160 insertions(+), 83 deletions(-) diff --git a/autodata_core/algorithms.py b/autodata_core/algorithms.py index 8c969be..e0df4f8 100644 --- a/autodata_core/algorithms.py +++ b/autodata_core/algorithms.py @@ -200,8 +200,12 @@ def __init__(self, motion_planners: dict[int, Any]) -> None: self.motion_planners = motion_planners def attempt_extras(self, eef_states: Iterable[_EEFGenerationState]) -> dict[str, Any]: - """Report the attempt's wall-clock motion-planning time [s] as ``planning_time_s``.""" - return {"planning_time_s": sum(state.planning_time_s for state in eef_states)} + """Report the attempt's wall-clock motion-planning time [s] and whether a motion plan failed.""" + states = list(eef_states) + return { + "planning_time_s": sum(state.planning_time_s for state in states), + "planning_failed": any(state.planning_failed for state in states), + } def plan_subtask_trajectory( self, @@ -261,6 +265,7 @@ def plan_subtask_trajectory( ) eef_state.planning_time_s += time.perf_counter() - start_time if not planning_success: + eef_state.planning_failed = True return None mp_waypoints = self._convert_planned_trajectory_to_waypoints(planner, target_passthrough_action) diff --git a/autodata_core/data_generator.py b/autodata_core/data_generator.py index 59d1e84..3f20fa5 100644 --- a/autodata_core/data_generator.py +++ b/autodata_core/data_generator.py @@ -15,7 +15,7 @@ import torch from copy import deepcopy from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any from isaaclab.managers import TerminationTermCfg @@ -52,15 +52,12 @@ class GenerationResult: Attributes: initial_state: Scene state of every env right after this env's reset. success: Whether the task succeeded at any step of the attempt. - failure_cause: ``"planner"`` when motion planning aborted the attempt, otherwise ``"execution"`` - for a failed attempt and None on success. extras: Algorithm-specific facts about the attempt, from :meth:`~autodata_core.algorithms.GenerationAlgorithm.attempt_extras`. """ initial_state: dict success: bool - failure_cause: Literal["planner", "execution"] | None = None extras: dict[str, Any] = field(default_factory=dict) @@ -88,6 +85,7 @@ class _EEFGenerationState: pending_subtask_trajectory: WaypointTrajectory | None = None is_in_motion_plan: bool = False planning_time_s: float = 0.0 + planning_failed: bool = False class DataGenerator: @@ -584,7 +582,6 @@ async def generate( return GenerationResult( initial_state=initial_state, success=False, - failure_cause="planner", extras=self.algorithm.attempt_extras(eef_states.values()), ) next_trajectory, _is_motion_plan = result @@ -637,7 +634,6 @@ async def generate( return GenerationResult( initial_state=initial_state, success=buffers.success, - failure_cause=None if buffers.success else "execution", extras=self.algorithm.attempt_extras(eef_states.values()), ) diff --git a/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py b/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py index 33cc04c..08016f4 100644 --- a/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py +++ b/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py @@ -78,9 +78,12 @@ def from_dict(cls, data: dict[str, Any]) -> AbsolutePoseEmbodimentAdapter: return cls(**data) def metric_context(self) -> dict[str, Any]: - """Adds the pose-action columns of every EEF that commands a pose; they are not clipped.""" + """Each end effector with a ``pose_action_slice`` commands its target pose there: position, then quaternion. + + These entries are not clipped. + """ pose_action = { - name: {"columns": list(spec["pose_action_slice"]), "bound": None} + name: {"action_slice": list(spec["pose_action_slice"]), "bound": None} for name, spec in self.eefs.items() if spec.get("pose_action_slice") is not None } diff --git a/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py b/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py index ad698e4..e948e3b 100644 --- a/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py +++ b/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py @@ -192,10 +192,13 @@ def action_dim(self) -> int: return max([self.hand_joints_slice[1], *channel_ends]) def metric_context(self) -> dict[str, Any]: - """Adds the pose-action columns of both hands; they are not clipped.""" + """``left_pose_slice`` and ``right_pose_slice`` of the action vector hold the hands' target poses. + + Each holds a position, then a quaternion. These entries are not clipped. + """ pose_action = { - self.left.name: {"columns": list(self.left_pose_slice), "bound": None}, - self.right.name: {"columns": list(self.right_pose_slice), "bound": None}, + self.left.name: {"action_slice": list(self.left_pose_slice), "bound": None}, + self.right.name: {"action_slice": list(self.right_pose_slice), "bound": None}, } return {**super().metric_context(), "pose_action": pose_action} diff --git a/autodata_interfaces/embodiments/embodiment_adapter.py b/autodata_interfaces/embodiments/embodiment_adapter.py index a219c6b..d0b9859 100644 --- a/autodata_interfaces/embodiments/embodiment_adapter.py +++ b/autodata_interfaces/embodiments/embodiment_adapter.py @@ -51,10 +51,12 @@ class EmbodimentAdapter(ABC): raw_config: dict[str, Any] | None = None def metric_context(self) -> dict[str, Any]: - """Facts about this embodiment that the dataset metrics read, stored with every dataset it records. + """Describe this embodiment for the dataset metrics; the description is saved with every dataset it records. - Subclasses add their pose-action layout. The base adds the optional ``hardware_limits`` section of the - embodiment config (datasheet joint limits, keyed by joint-name pattern). + The base returns the optional ``hardware_limits`` section of the embodiment config: datasheet joint limits + keyed by joint-name pattern. Subclasses add ``pose_action``, which says for each end effector which part of + the action vector commands its pose (``action_slice``, a half-open ``[start, end)`` index range) and the + value the adapter clips those entries to (``bound``, or None when they are not clipped). """ hardware_limits = (self.raw_config or {}).get("hardware_limits") return {"hardware_limits": hardware_limits} if hardware_limits else {} diff --git a/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py b/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py index 3b25dec..6cd8df0 100644 --- a/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py +++ b/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py @@ -134,9 +134,12 @@ class DeltaPoseIKSingleArmAdapter(SingleArmEmbodimentAdapter): clip_pose_action_to_unit: bool = True def metric_context(self) -> dict[str, Any]: - """Adds the pose-action columns and their clip bound.""" + """Action entries 0-5 command the end effector's pose change: delta position, then axis-angle rotation. + + They are clipped to ±1 when ``clip_pose_action_to_unit`` is set. + """ bound = 1.0 if self.clip_pose_action_to_unit else None - pose_action = {self.eef_name: {"columns": [0, _DELTA_POSE_ACTION_DIM], "bound": bound}} + pose_action = {self.eef_name: {"action_slice": [0, _DELTA_POSE_ACTION_DIM], "bound": bound}} return {**super().metric_context(), "pose_action": pose_action} @property diff --git a/autodata_interfaces/env/isaaclab_env_interface.py b/autodata_interfaces/env/isaaclab_env_interface.py index 80754f0..9e4ebf6 100644 --- a/autodata_interfaces/env/isaaclab_env_interface.py +++ b/autodata_interfaces/env/isaaclab_env_interface.py @@ -200,6 +200,7 @@ def setup_env_config( recorder_cfg: RecorderManagerBaseCfg | None = None, env_profile: EnvironmentProfile | None = None, enable_cameras: bool = False, + record_step_success: bool = False, ) -> tuple[Any, Any]: """Configure the environment for data generation. @@ -220,6 +221,8 @@ def setup_env_config( reset-event changes) before the generation-specific adjustments below. enable_cameras: Record the task's image observations. Defaults to low-dimensional observations, excluding the cameras used for teleoperation previews. + record_step_success: Also record whether the task's success term holds after each step, as + ``step_success``, for the dataset metrics. Returns: A tuple of the environment configuration and the success termination condition. @@ -265,7 +268,8 @@ def setup_env_config( env_cfg.recorders = ActionStateRecorderManagerCfg() else: env_cfg.recorders = recorder_cfg - add_step_success_recorder(env_cfg.recorders, success_term) + if record_step_success: + add_step_success_recorder(env_cfg.recorders, success_term) env_cfg.recorders.dataset_export_dir_path = output_dir env_cfg.recorders.dataset_filename = output_file_name diff --git a/autodata_retargeting/runner.py b/autodata_retargeting/runner.py index cb82c9b..7621d1c 100644 --- a/autodata_retargeting/runner.py +++ b/autodata_retargeting/runner.py @@ -98,6 +98,7 @@ def run(args, simulation_app) -> int: num_envs=num_envs, device=args.device, generation_policy_params=generation_policy, + record_step_success=getattr(args, "record_metric_inputs", False), ) # Compensate the target env's (auto-detected) arm-action scale *in the encoded action* rather than @@ -122,7 +123,8 @@ def run(args, simulation_app) -> int: autodata_entry = None try: target_adapter.bind_env(env) - autodata_entry = collect_autodata_entry(env, target_adapter, uuid.uuid4().hex) + if getattr(args, "record_metric_inputs", False): + autodata_entry = collect_autodata_entry(env, target_adapter, uuid.uuid4().hex) env.reset() # Snapshot the target's home scene state once; each replay restores its robot from here # (deepcopy so later sim steps cannot mutate the cached tensors). diff --git a/autodata_tests/core/test_algorithms.py b/autodata_tests/core/test_algorithms.py index b3b0867..ae90f59 100644 --- a/autodata_tests/core/test_algorithms.py +++ b/autodata_tests/core/test_algorithms.py @@ -3,6 +3,9 @@ """Unit tests for :mod:`autodata_core.algorithms`.""" +from types import SimpleNamespace +from unittest.mock import Mock + import pytest from autodata_core.algorithms import ( @@ -14,6 +17,7 @@ get_algorithm, iter_algorithms, ) +from autodata_core.data_generator import _EEFGenerationState def test_registry_contents(): @@ -81,3 +85,23 @@ def test_get_algorithm_skillgen_needs_planners(): assert isinstance( get_algorithm("skillgen", motion_planners={0: object()}), SkillGen ) # base implementation must not raise + + +def test_skillgen_reports_a_failed_motion_plan(): + planner = Mock(update_world_and_plan_motion=Mock(return_value=False)) + algorithm = SkillGen(motion_planners={0: planner}) + data_generator = Mock() + data_generator.generate_eef_subtask_trajectory.return_value = [SimpleNamespace(pose=None, passthrough_action=None)] + eef_state = _EEFGenerationState() + + result = algorithm.plan_subtask_trajectory( + data_generator=data_generator, + env_id=0, + eef_name="franka", + eef_state=eef_state, + all_randomized_subtask_boundaries={}, + runtime_subtask_constraints_dict={}, + selected_src_demo_inds={}, + ) + + assert result is None and algorithm.attempt_extras([eef_state])["planning_failed"] is True diff --git a/autodata_tests/core/test_data_generator.py b/autodata_tests/core/test_data_generator.py index 6667ca1..806c0e9 100644 --- a/autodata_tests/core/test_data_generator.py +++ b/autodata_tests/core/test_data_generator.py @@ -196,21 +196,6 @@ def test_navigation_does_not_skip_playback_after_cancellation_or_while_paused(pa assert all(state.subtask_step_index == 4 for state in states.values()) -def test_planning_failure_is_attributed_to_the_planner(monkeypatch): - generator = _navigation_generator(active=False, reached=True) - generator.datastream.env = SimpleNamespace(recorder_manager=Mock()) - monkeypatch.setattr(generator.algorithm, "plan_subtask_trajectory", lambda **kwargs: None) - monkeypatch.setattr(generator, "_reset_environment_for_generation", AsyncMock(return_value=(torch.tensor([0]), {}))) - - result = asyncio.run( - generator.generate( - env_id=0, success_term=None, env_reset_queue=asyncio.Queue(), env_action_queue=asyncio.Queue() - ) - ) - - assert result.success is False and result.failure_cause == "planner" - - def test_unresponsive_navigation_fails_and_finalizes_the_attempt(monkeypatch): generator = _navigation_generator(active=True, reached=False) generator.datastream.generation_policy.max_navigation_hold_steps = 3 @@ -234,7 +219,7 @@ def test_unresponsive_navigation_fails_and_finalizes_the_attempt(monkeypatch): ) ) - assert result.success is False and result.failure_cause == "execution" + assert result.success is False assert execute.await_count == 3 assert recorder.set_success_to_episodes.call_args.args[1].item() is False recorder.export_episodes.assert_called_once() diff --git a/autodata_tests/e2e/test_annotation.py b/autodata_tests/e2e/test_annotation.py index bedc99c..fa93c2e 100644 --- a/autodata_tests/e2e/test_annotation.py +++ b/autodata_tests/e2e/test_annotation.py @@ -36,6 +36,7 @@ def test_automatic_annotation_from_dataset(tmp_path): str(input_file), "--output_file", str(output_file), + "--record_metric_inputs", ]) assert_valid_dataset(str(output_file), min_num_demos=1) entry = assert_recorded_episodes(str(output_file)) diff --git a/autodata_tests/e2e/test_dexmimicgen_data_generation.py b/autodata_tests/e2e/test_dexmimicgen_data_generation.py index fa73a6c..602c954 100644 --- a/autodata_tests/e2e/test_dexmimicgen_data_generation.py +++ b/autodata_tests/e2e/test_dexmimicgen_data_generation.py @@ -8,7 +8,7 @@ from autodata_tests.utils.constants import TestPaths from autodata_tests.utils.subprocess import run_subprocess -from autodata_tests.utils.utils import assert_valid_dataset +from autodata_tests.utils.utils import assert_no_metric_inputs, assert_valid_dataset HEADLESS = True GENERATION_NUM_TRIALS = 1 @@ -47,6 +47,7 @@ def _run_gr1_pick_place_dexmimicgen(num_envs: int, device: str) -> None: run_subprocess(args) assert_valid_dataset(output_file, min_num_demos=GENERATION_NUM_TRIALS) + assert_no_metric_inputs(output_file) def _run_g1_pick_place_dexmimicgen(num_envs: int, device: str) -> None: diff --git a/autodata_tests/e2e/test_mimicgen_data_generation.py b/autodata_tests/e2e/test_mimicgen_data_generation.py index a97271e..0552279 100644 --- a/autodata_tests/e2e/test_mimicgen_data_generation.py +++ b/autodata_tests/e2e/test_mimicgen_data_generation.py @@ -38,6 +38,7 @@ def _run_franka_cube_stack_mimicgen(num_envs: int, device: str) -> None: output_file, "--result_file", result_file, + "--record_metric_inputs", "--generation_num_trials", str(GENERATION_NUM_TRIALS), "--num_envs", diff --git a/autodata_tests/e2e/test_skillgen_bin_data_generation.py b/autodata_tests/e2e/test_skillgen_bin_data_generation.py index 90b5a17..580d086 100644 --- a/autodata_tests/e2e/test_skillgen_bin_data_generation.py +++ b/autodata_tests/e2e/test_skillgen_bin_data_generation.py @@ -50,6 +50,7 @@ def _run_franka_bin_stack_skillgen(num_envs: int, device: str) -> None: output_file, "--result_file", result_file, + "--record_metric_inputs", "--generation_num_trials", str(GENERATION_NUM_TRIALS), "--num_envs", diff --git a/autodata_tests/utils/utils.py b/autodata_tests/utils/utils.py index c300e97..4792836 100644 --- a/autodata_tests/utils/utils.py +++ b/autodata_tests/utils/utils.py @@ -85,3 +85,14 @@ def assert_recorded_run(dataset_file: str, result_file: str) -> list[dict]: run_ids = {entry["run_id"], result["run"]["run_id"], *(attempt["run_id"] for attempt in attempts)} assert len(run_ids) == 1, f"The dataset, result and attempt log carry different run ids: {run_ids}" return attempts + + +def assert_no_metric_inputs(dataset_file: str) -> None: + """Assert a dataset recorded without ``--record_metric_inputs`` has no metric inputs, as before they existed.""" + + with h5py.File(dataset_file, "r") as h5_file: + data = h5_file["data"] + assert "autodata" not in json.loads(data.attrs["env_args"]), "The dataset has an autodata entry." + for name, episode in data.items(): + assert "step_success" not in episode, f"{name} records step_success." + assert "datagen_info" not in episode["obs"], f"{name} records datagen_info." diff --git a/docs/pages/concepts/data_generator.rst b/docs/pages/concepts/data_generator.rst index 6fa9454..244451b 100644 --- a/docs/pages/concepts/data_generator.rst +++ b/docs/pages/concepts/data_generator.rst @@ -123,13 +123,13 @@ Results and Statistics Four layers report what happened: -* **Per step** — every exported episode records ``step_success``, whether the task's success term - held after each step. Generation also records the end-effector pose and its commanded target under - ``obs/datagen_info``; object poses are recorded there only during annotation, for the source pool. -* **Per attempt** — ``GenerationResult``: ``success``, ``failure_cause`` (``planner`` or - ``execution``), ``extras`` from the algorithm's ``attempt_extras()``, and ``initial_state``. With - ``--result_file``, each completed attempt is also written as one line of - ``..attempts.jsonl``. +* **Per step** — annotation records ``obs/datagen_info`` (object, end-effector and commanded + target poses) for the source pool. With ``--record_metric_inputs``, every exported episode also + records ``step_success``, whether the task's success term held after each step, and generation + records the end-effector pose and its commanded target. +* **Per attempt** — ``GenerationResult``: ``success``, ``extras`` from the algorithm's + ``attempt_extras()``, and ``initial_state``. With ``--record_metric_inputs`` and ``--result_file``, + each completed attempt is also written as one line of ``..attempts.jsonl``. * **Per run, in memory** — the ``stats`` counters: ``num_success``, ``num_failures``, ``num_attempts`` (``num_success + num_failures = num_attempts``). * **Per run, on disk** — with ``--result_file``, a JSON sidecar is written atomically at the @@ -168,11 +168,12 @@ Four layers report what happened: applied — the generated dataset itself records only the base env id, so the sidecar is what documents that the scene was modified. -Every exported dataset, whether from generation, annotation or retargeting, also carries an ``autodata`` -entry in its ``env_args`` attribute. The entry holds the ``run_id`` and the number of exported -episodes, the embodiment's pose-action layout, the robot's joint names with their simulator limits, -and the embodiment's optional datasheet ``hardware_limits``. For a dataset generated with -``--result_file``, the ``run_id`` ties it to that run's result file and attempt log. +The ``run`` block is written only with ``--record_metric_inputs``. With that option, every exported +dataset, whether from generation, annotation or retargeting, also carries an ``autodata`` entry in +its ``env_args`` attribute. The entry holds the ``run_id`` and the number of exported episodes, the +embodiment's pose-action layout, the robot's joint names with their simulator limits, and the +embodiment's optional datasheet ``hardware_limits``. For a dataset generated with ``--result_file``, +the ``run_id`` ties it to that run's attempt log and, once the run completes, its result file. The GenerationAlgorithm Plug-In ------------------------------- @@ -203,7 +204,8 @@ The shipped algorithms — ``mimicgen``, ``dexmimicgen``, ``skillgen`` — are d :doc:`algorithms`. An algorithm can also override ``attempt_extras()`` to report facts about each attempt in its -attempt-log line; SkillGen reports its motion-planning time as ``planning_time_s``. +attempt-log line. SkillGen reports its motion-planning time as ``planning_time_s`` and whether a +motion plan failed as ``planning_failed``. The behavioral hook: ``plan_subtask_trajectory`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/scripts/annotate_demos.py b/scripts/annotate_demos.py index 95d370f..ee5434f 100644 --- a/scripts/annotate_demos.py +++ b/scripts/annotate_demos.py @@ -88,6 +88,11 @@ action="store_true", help="Render and record the task's image observations.", ) +parser.add_argument( + "--record_metric_inputs", + action="store_true", + help="Record what the dataset metrics read: per-step task success and run metadata in the dataset. Off by default.", +) AppLauncher.add_app_launcher_args(parser) parser.add_argument("--external_callback", type=callable_path, help="Callable registering environments after startup.") args_cli, environment_args = parser.parse_known_args() @@ -248,6 +253,7 @@ def main() -> int: generation_policy_params=generation_policy, recorder_cfg=DatagenInfoRecorderManagerCfg(), enable_cameras=args_cli.record_images, + record_step_success=args_cli.record_metric_inputs, ) # Only export episodes we explicitly mark successful (i.e. fully annotated). env_cfg.recorders.dataset_export_mode = DatasetExportMode.EXPORT_SUCCEEDED_ONLY @@ -271,7 +277,8 @@ def main() -> int: uses_start_signals=annotate_start_signals, ) PreStepDatagenInfoRecorder.datastream = _datastream - autodata_entry = collect_autodata_entry(env, embodiment_adapter, uuid.uuid4().hex) + if args_cli.record_metric_inputs: + autodata_entry = collect_autodata_entry(env, embodiment_adapter, uuid.uuid4().hex) env.reset() @@ -328,12 +335,13 @@ def main() -> int: f"\nExported {exported_episode_count} (out of {processed_episode_count}) annotated" f" episode{'s' if exported_episode_count != 1 else ''} to {args_cli.output_file}." ) - autodata_entry["annotation"] = { - "input_episodes": len(episode_names), - "processed_episodes": processed_episode_count, - "annotated_episodes": exported_episode_count, - "time_s": time.monotonic() - annotation_start, - } + if autodata_entry is not None: + autodata_entry["annotation"] = { + "input_episodes": len(episode_names), + "processed_episodes": processed_episode_count, + "annotated_episodes": exported_episode_count, + "time_s": time.monotonic() - annotation_start, + } print("Exiting the app.") return exported_episode_count finally: diff --git a/scripts/generate_dataset.py b/scripts/generate_dataset.py index 19791a5..81ebb62 100755 --- a/scripts/generate_dataset.py +++ b/scripts/generate_dataset.py @@ -101,8 +101,16 @@ type=str, default=None, help=( - "Optional JSON path for the final structured generation result. An attempt log with one JSON " - "line per attempt is written next to it (..attempts.jsonl)." + "Optional JSON path for the final structured generation result. With --record_metric_inputs, an attempt " + "log with one JSON line per attempt is written next to it (..attempts.jsonl)." + ), +) +parser.add_argument( + "--record_metric_inputs", + action="store_true", + help=( + "Record what the dataset metrics read: per-step end-effector and target poses and task success, run " + "metadata in the dataset and, with --result_file, a run block and an attempt log. Off by default." ), ) parser.add_argument( @@ -221,7 +229,6 @@ def _attempt_record(env_id: int, result: GenerationResult, start_time: float, en return { "env_id": env_id, "success": result.success, - "failure_cause": result.failure_cause, "start_time_s": start_time, "end_time_s": end_time, "initial_object_positions": { @@ -393,7 +400,12 @@ def main() -> None: num_envs=args_cli.num_envs, device=args_cli.device, generation_policy_params=generation_policy_params, - recorder_cfg=load_callable(args_cli.recorder_cfg)() if args_cli.recorder_cfg else _generation_recorder_cfg(), + recorder_cfg=( + load_callable(args_cli.recorder_cfg)() + if args_cli.recorder_cfg + else _generation_recorder_cfg() if args_cli.record_metric_inputs else None + ), + record_step_success=args_cli.record_metric_inputs, env_profile=env_profile, enable_cameras=args_cli.record_images, ) @@ -415,7 +427,7 @@ def main() -> None: ) PreStepDatagenInfoRecorder.datastream = datastream run_id = uuid.uuid4().hex - autodata_entry = collect_autodata_entry(env, embodiment_adapter, run_id) + autodata_entry = collect_autodata_entry(env, embodiment_adapter, run_id) if args_cli.record_metric_inputs else None # SkillGen needs one curobo planner per env. Mimic/DexMimic take no kwargs. motion_planners: dict | None = None @@ -431,7 +443,7 @@ def main() -> None: alg_kwargs["motion_planners"] = motion_planners algorithm = get_algorithm(args_cli.alg, **alg_kwargs) - attempts_file = attempts_file_for(args_cli.result_file, run_id) if args_cli.result_file else None + attempts_file = attempts_file_for(args_cli.result_file, run_id) if autodata_entry and args_cli.result_file else None attempt_log = AttemptLog(attempts_file, run_id) if attempts_file else None try: async_components = setup_async_generation( @@ -480,28 +492,35 @@ def main() -> None: if env_profile else None ), - run_metadata={ - "run_id": run_id, - "seed": generation_policy_params.seed, - "revision": code_revision(), - "inputs": { - "source": input_record(args_cli.input_file), - "embodiment": input_record(args_cli.embodiment), - "task_descriptor": input_record(args_cli.task_descriptor), - "env_profile": input_record(args_cli.env_profile), - }, - "device": args_cli.device, - **_hardware(args_cli.device), - "num_envs": args_cli.num_envs, - "setup_time_s": generation_start - PROCESS_START, - "generation_time_s": generation_end - generation_start, - "attempts_file": Path(attempts_file).name, - }, + run_metadata=( + None + if autodata_entry is None + else { + "run_id": run_id, + "seed": generation_policy_params.seed, + "revision": code_revision(), + "inputs": { + "source": input_record(args_cli.input_file), + "embodiment": input_record(args_cli.embodiment), + "task_descriptor": input_record(args_cli.task_descriptor), + "env_profile": input_record(args_cli.env_profile), + }, + "device": args_cli.device, + **_hardware(args_cli.device), + "num_envs": args_cli.num_envs, + "setup_time_s": generation_start - PROCESS_START, + "generation_time_s": generation_end - generation_start, + "attempts_file": Path(attempts_file).name, + } + ), ) finally: env.close() - for dataset_file in exported_dataset_files(output_dir, output_file_name, generation_policy_params.keep_failed): - merge_autodata_entry(dataset_file, autodata_entry) + if autodata_entry is not None: + for dataset_file in exported_dataset_files( + output_dir, output_file_name, generation_policy_params.keep_failed + ): + merge_autodata_entry(dataset_file, autodata_entry) if __name__ == "__main__": diff --git a/scripts/retarget_dataset.py b/scripts/retarget_dataset.py index d2906ad..af10815 100755 --- a/scripts/retarget_dataset.py +++ b/scripts/retarget_dataset.py @@ -138,6 +138,11 @@ "expected to be low." ), ) +parser.add_argument( + "--record_metric_inputs", + action="store_true", + help="Record what the dataset metrics read: per-step task success and run metadata in the dataset. Off by default.", +) AppLauncher.add_app_launcher_args(parser) args_cli = parser.parse_args() diff --git a/skills/autodata-generate-datasets/references/cli-reference.md b/skills/autodata-generate-datasets/references/cli-reference.md index 9062e4a..39d00a8 100644 --- a/skills/autodata-generate-datasets/references/cli-reference.md +++ b/skills/autodata-generate-datasets/references/cli-reference.md @@ -25,7 +25,8 @@ YAML, live env, and HDF5 source dataset, then runs the selected generation algor | `--output_file ` | no | `./datasets/output_dataset.hdf5` | Destination for generated demos. | | `--generation_num_trials ` | no | `None` | Number of demos to generate. | | `--num_envs ` | no | `1` | Number of parallel simulation environments. | -| `--result_file ` | no | — | Where to write the generation result summary. An attempt log (`..attempts.jsonl`) is written next to it. | +| `--result_file ` | no | — | Where to write the generation result summary. With `--record_metric_inputs`, an attempt log (`..attempts.jsonl`) is written next to it. | +| `--record_metric_inputs` | no | off | Record what the dataset metrics read: per-step poses and task success, run metadata, and the attempt log. | | `--pause_subtask` | no | off | Pause at subtask boundaries (debugging). | | `--visualize_plan` | no | off | Visualize the motion plan (SkillGen debugging). | From ca4a948666bca47ec86a100c08513c49ef672e74 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Wed, 28 Oct 2026 13:59:23 -0500 Subject: [PATCH 09/13] fix: reject non-finite values, require --report for report options, run metrics tests in CI - The report marks a metric whose value is not a finite number as an error, and the gate counts a missing or non-numeric value as a violation. - validate_dataset.py errors when --result_file, --source, --metrics, --gate or --report_file is given without --report. - failure_share is replaced by planning_failure_rate, read from SkillGen's planning_failed attempt extra. - CI runs autodata_tests/metrics/. - Docs: metrics need --record_metric_inputs at recording time. --- .github/workflows/ci.yml | 2 +- autodata_metrics/report.py | 16 ++++++++++++++- autodata_metrics/viability.py | 18 ++++++++--------- .../e2e/test_mimicgen_data_generation.py | 2 +- .../e2e/test_skillgen_bin_data_generation.py | 10 ++++++++-- autodata_tests/metrics/test_report.py | 20 +++++++++++++++---- autodata_tests/utils/test_validate_dataset.py | 9 +++++++++ docs/pages/references/metrics.rst | 3 ++- scripts/validate_dataset.py | 10 ++++++++++ 9 files changed, 70 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb5051c..20ea111 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: # trials each) are heavy and run in the nightly instead. No marker filter, so # both subprocess and unit tests run. env: - TEST_PATH: "autodata_tests/e2e/ autodata_tests/interfaces/ autodata_tests/core/ autodata_tests/utils/ autodata_tests/retargeting/" + TEST_PATH: "autodata_tests/e2e/ autodata_tests/interfaces/ autodata_tests/core/ autodata_tests/utils/ autodata_tests/retargeting/ autodata_tests/metrics/" steps: - name: nvidia-smi diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index c11ac92..41d7133 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -8,6 +8,8 @@ import contextlib import h5py import json +import math +import numbers import yaml from collections.abc import Mapping from pathlib import Path @@ -92,9 +94,18 @@ def evaluate(spec: MetricSpec, evidence: Evidence) -> dict[str, Any]: return {**row, "status": "not_applicable", "reason": str(exc)} except Exception as exc: # A failing metric must not sink the report of every other metric. return {**row, "status": "error", "reason": f"{type(exc).__name__}: {exc}"} + if not _is_finite(value.value): + return {**row, "status": "error", "reason": f"non-finite value {value.value}"} return {**row, "status": "ok", "value": value.value, "n": value.n, "details": dict(value.details)} +def _is_finite(value: Any) -> bool: + """Whether ``value`` is a finite number, or a mapping of finite numbers.""" + if isinstance(value, Mapping): + return all(_is_finite(item) for item in value.values()) + return isinstance(value, numbers.Real) and math.isfinite(value) + + def build_report( dataset: str, result_file: str | None = None, @@ -150,8 +161,11 @@ def gate_violations(report: Mapping[str, Any], gate_file: str) -> list[str]: violations.append(f"{name}: version {row['version']}, gate written for version {bound['version']}") continue key = bound.get("key", row["primary"]) - value = row["value"][key] if key else row["value"] + value = row["value"].get(key) if key else row["value"] assert not isinstance(value, Mapping), f"Gate entry {name!r} needs a key into {sorted(value)}." + if not _is_finite(value): + violations.append(f"{name}: {value!r} is not a finite number") + continue if "min" in bound and value < bound["min"]: violations.append(f"{name}: {value:.4g} is below {bound['min']}") if "max" in bound and value > bound["max"]: diff --git a/autodata_metrics/viability.py b/autodata_metrics/viability.py index 62e91e9..a8de7a1 100644 --- a/autodata_metrics/viability.py +++ b/autodata_metrics/viability.py @@ -130,21 +130,19 @@ def annotation_pass_rate(ev: Evidence) -> Value: @metric( - "failure_share", + "planning_failure_rate", area="diagnostics", signal="failure causes", scope="run", - unit="fraction of failures", - requires=["attempts"], + unit="fraction of attempts", + better="lower", + requires=["attempts.extras.planning_failed"], ) -def failure_share(ev: Evidence) -> Value: - """Share of failed attempts by cause, ``planner`` or ``execution``.""" +def planning_failure_rate(ev: Evidence) -> Value: + """Share of attempts whose motion planning failed, stop-corrected when the attempt log allows.""" attempts, corrected = _counted_attempts(ev) - causes = Counter(attempt["failure_cause"] for attempt in attempts if not attempt["success"]) - total = sum(causes.values()) - return Value( - {cause: count / total for cause, count in causes.items()}, n=total, details={"stop_corrected": corrected} - ) + failed = sum(attempt["extras"]["planning_failed"] for attempt in attempts) + return Value(failed / len(attempts), n=len(attempts), details={"stop_corrected": corrected}) @metric( diff --git a/autodata_tests/e2e/test_mimicgen_data_generation.py b/autodata_tests/e2e/test_mimicgen_data_generation.py index f65c7fc..08d996f 100644 --- a/autodata_tests/e2e/test_mimicgen_data_generation.py +++ b/autodata_tests/e2e/test_mimicgen_data_generation.py @@ -13,7 +13,7 @@ HEADLESS = True GENERATION_NUM_TRIALS = 1 # Metrics a MimicGen run reported with its result file must compute. -RUN_METRICS = {"generation_success_rate", "demos_per_gpu_hour", "time_to_dataset", "failure_share"} +RUN_METRICS = {"generation_success_rate", "demos_per_gpu_hour", "time_to_dataset"} def _run_franka_cube_stack_mimicgen(num_envs: int, device: str) -> None: diff --git a/autodata_tests/e2e/test_skillgen_bin_data_generation.py b/autodata_tests/e2e/test_skillgen_bin_data_generation.py index b61db5d..b464095 100644 --- a/autodata_tests/e2e/test_skillgen_bin_data_generation.py +++ b/autodata_tests/e2e/test_skillgen_bin_data_generation.py @@ -21,8 +21,14 @@ HEADLESS = True GENERATION_NUM_TRIALS = 1 -# Metrics a SkillGen run reported with its result file must compute; planning time is SkillGen's own. -RUN_METRICS = {"generation_success_rate", "demos_per_gpu_hour", "time_to_dataset", "failure_share", "planning_time"} +# Metrics a SkillGen run reported with its result file must compute; the planning metrics are SkillGen's own. +RUN_METRICS = { + "generation_success_rate", + "demos_per_gpu_hour", + "time_to_dataset", + "planning_time", + "planning_failure_rate", +} def _run_franka_bin_stack_skillgen(num_envs: int, device: str) -> None: diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py index 7986217..69579ac 100644 --- a/autodata_tests/metrics/test_report.py +++ b/autodata_tests/metrics/test_report.py @@ -88,15 +88,27 @@ def test_report_refuses_merged_data(tmp_path): build_report(_dataset(tmp_path / "merged.hdf5", num_episodes=2, entry=entry), metrics={}) -def test_gate_fails_on_bound_inapplicable_metric_and_outdated_version(tmp_path): +def test_gate_fails_on_bound_inapplicable_missing_and_outdated_metric(tmp_path): row = {"status": "ok", "value": 0.4, "primary": None, "version": 2} - report = {"metrics": {"low": row, "old": row, "fine": row, "absent": {"status": "not_applicable"}}} + missing = {"status": "ok", "value": {"p50": 1.0}, "primary": "p95", "version": 1} + report = {"metrics": {"low": row, "old": row, "fine": row, "absent": {"status": "not_applicable"}, "gap": missing}} gate = tmp_path / "gate.yaml" - gate.write_text("low: {min: 0.5}\nold: {version: 1}\nfine: {min: 0.3, max: 0.5, version: 2}\nabsent: {min: 0}\n") + gate.write_text( + "low: {min: 0.5}\nold: {version: 1}\nfine: {min: 0.3, max: 0.5, version: 2}\nabsent: {min: 0}\ngap: {max: 2}\n" + ) violations = gate_violations(report, str(gate)) - assert [violation.split(":")[0] for violation in violations] == ["low", "old", "absent"] + assert [violation.split(":")[0] for violation in violations] == ["low", "old", "absent", "gap"] + + +def test_non_finite_value_is_an_error_and_fails_the_gate(tmp_path): + report = build_report(_dataset(tmp_path / "d.hdf5"), metrics={"m": _spec("m", lambda ev: Value(float("nan"), n=1))}) + gate = tmp_path / "gate.yaml" + gate.write_text("m: {min: 0}\n") + + assert report["metrics"]["m"]["status"] == "error" + assert gate_violations(report, str(gate)) == ["m: error"] def test_metrics_import_without_torch_or_isaaclab(): diff --git a/autodata_tests/utils/test_validate_dataset.py b/autodata_tests/utils/test_validate_dataset.py index 3e1ea73..d109227 100644 --- a/autodata_tests/utils/test_validate_dataset.py +++ b/autodata_tests/utils/test_validate_dataset.py @@ -7,6 +7,8 @@ import json from pathlib import Path +import pytest + from scripts import validate_dataset @@ -45,3 +47,10 @@ def test_main_returns_nonzero_for_missing_file(tmp_path: Path) -> None: """An unreadable input path should produce a failing exit status.""" assert validate_dataset.main([str(tmp_path / "missing.hdf5")]) == 1 + + +def test_report_options_without_report_are_an_error(tmp_path: Path) -> None: + """A gate given without --report would otherwise be silently ignored.""" + + with pytest.raises(SystemExit): + validate_dataset.main([str(tmp_path / "data.hdf5"), "--gate", str(tmp_path / "gate.yaml")]) diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst index a2b8073..b327d61 100644 --- a/docs/pages/references/metrics.rst +++ b/docs/pages/references/metrics.rst @@ -3,7 +3,8 @@ Metrics ``scripts/validate_dataset.py --report`` computes the metrics of a dataset and, given its result file, of the generation run that produced it. Metrics are computed from recorded files and need no -simulator. +simulator. Generate, annotate or retarget with ``--record_metric_inputs`` to record what they read; +without it, most metrics report that their inputs are missing. .. code-block:: bash diff --git a/scripts/validate_dataset.py b/scripts/validate_dataset.py index a341278..fd8aa08 100644 --- a/scripts/validate_dataset.py +++ b/scripts/validate_dataset.py @@ -178,6 +178,16 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) if args.report and len(args.files) != 1: parser.error("--report takes exactly one dataset") + report_options = { + "--result_file": args.result_file, + "--source": args.source, + "--metrics": args.metrics, + "--gate": args.gate, + "--report_file": args.report_file, + } + given = [option for option, value in report_options.items() if value] + if given and not args.report: + parser.error(f"{', '.join(given)} only apply with --report") results = [validate_file(path) for path in args.files] print_results(results) From 9ea6a76ae42092d56bccacdb6648a3e4b796816b Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Wed, 28 Oct 2026 16:37:03 -0500 Subject: [PATCH 10/13] fix: record metric inputs with a custom recorder and without a success term - With --record_metric_inputs, the per-step pose term is added to a custom --recorder_cfg too, unless it already records datagen info. - Without a success term, step_success is not recorded instead of failing on the first step. --- .../env/datagen_info_recorder.py | 22 +++++++++++++-- .../env/test_datagen_info_recorder.py | 28 +++++++++++++++++++ scripts/generate_dataset.py | 24 ++++++---------- 3 files changed, 55 insertions(+), 19 deletions(-) create mode 100644 autodata_tests/interfaces/env/test_datagen_info_recorder.py diff --git a/autodata_interfaces/env/datagen_info_recorder.py b/autodata_interfaces/env/datagen_info_recorder.py index 15bb80b..81d5fa6 100644 --- a/autodata_interfaces/env/datagen_info_recorder.py +++ b/autodata_interfaces/env/datagen_info_recorder.py @@ -69,11 +69,27 @@ class DatagenInfoRecorderManagerCfg(ActionStateRecorderManagerCfg): record_pre_step_datagen_info = PreStepDatagenInfoRecorderCfg() -def add_step_success_recorder(recorder_cfg: object, success_term: TerminationTermCfg) -> None: +def add_step_success_recorder(recorder_cfg: object, success_term: TerminationTermCfg | None) -> None: """Add the ``step_success`` term to any recorder manager configuration. + Nothing is added without a success term, since there is then no success to record. + + Args: + recorder_cfg: Recorder manager configuration the env will be built with. + success_term: The task's success term, or None when the task has none. + """ + if success_term is not None: + recorder_cfg.record_post_step_success = PostStepSuccessRecorderCfg(success_term=success_term) + + +def add_pose_recorder(recorder_cfg: object) -> None: + """Add the per-step end-effector and commanded target pose term to any recorder manager configuration. + + Object poses are left out, as no metric reads them. A configuration that already has the + ``record_pre_step_datagen_info`` term, such as :class:`DatagenInfoRecorderManagerCfg`, keeps its own. + Args: recorder_cfg: Recorder manager configuration the env will be built with. - success_term: The task's success term. """ - recorder_cfg.record_post_step_success = PostStepSuccessRecorderCfg(success_term=success_term) + if not hasattr(recorder_cfg, "record_pre_step_datagen_info"): + recorder_cfg.record_pre_step_datagen_info = PreStepDatagenInfoRecorderCfg(record_object_poses=False) diff --git a/autodata_tests/interfaces/env/test_datagen_info_recorder.py b/autodata_tests/interfaces/env/test_datagen_info_recorder.py new file mode 100644 index 0000000..2929133 --- /dev/null +++ b/autodata_tests/interfaces/env/test_datagen_info_recorder.py @@ -0,0 +1,28 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from isaaclab.envs.mdp.recorders.recorders_cfg import ActionStateRecorderManagerCfg + +from autodata_interfaces.env.datagen_info_recorder import ( + DatagenInfoRecorderManagerCfg, + add_pose_recorder, + add_step_success_recorder, +) + + +def test_metric_recorders_extend_a_custom_configuration(): + custom = ActionStateRecorderManagerCfg() + + add_pose_recorder(custom) + add_step_success_recorder(custom, success_term=None) + + assert custom.record_pre_step_datagen_info.record_object_poses is False + assert not hasattr(custom, "record_post_step_success") + + +def test_pose_recorder_keeps_an_existing_datagen_info_term(): + annotation = DatagenInfoRecorderManagerCfg() + + add_pose_recorder(annotation) + + assert annotation.record_pre_step_datagen_info.record_object_poses is True diff --git a/scripts/generate_dataset.py b/scripts/generate_dataset.py index 81ebb62..2aa3604 100755 --- a/scripts/generate_dataset.py +++ b/scripts/generate_dataset.py @@ -160,6 +160,8 @@ from pathlib import Path # noqa: E402 from typing import Any # noqa: E402 +from isaaclab.envs.mdp.recorders.recorders_cfg import ActionStateRecorderManagerCfg # noqa: E402 + from autodata_core import DataGenerator, get_algorithm # noqa: E402 from autodata_core.algorithms import REGISTERED_ALGORITHMS # noqa: E402 from autodata_core.data_generator import GenerationResult # noqa: E402 @@ -172,10 +174,7 @@ setup_env_config, setup_output_paths, ) -from autodata_interfaces.env.datagen_info_recorder import ( # noqa: E402 - DatagenInfoRecorderManagerCfg, - PreStepDatagenInfoRecorder, -) +from autodata_interfaces.env.datagen_info_recorder import PreStepDatagenInfoRecorder, add_pose_recorder # noqa: E402 from autodata_interfaces.env.registration import load_callable, register_external_environment # noqa: E402 from autodata_utils.dataset_metadata import exported_dataset_files, merge_autodata_entry # noqa: E402 from autodata_utils.generation_result import ( # noqa: E402 @@ -253,13 +252,6 @@ def _hardware(device: str) -> dict[str, str | None]: return {"gpu": gpu, "num_gpus": 1 if gpu else 0, "cpu": cpu or platform.processor() or None} -def _generation_recorder_cfg() -> DatagenInfoRecorderManagerCfg: - """Actions and states, plus the per-step poses the dataset metrics read; object poses are left out.""" - recorder_cfg = DatagenInfoRecorderManagerCfg() - recorder_cfg.record_pre_step_datagen_info.record_object_poses = False - return recorder_cfg - - def build_datastream( env: Any, input_file: str, @@ -393,6 +385,10 @@ def main() -> None: # names the motion-planner profile tuned for the resulting scene. env_profile = EnvironmentProfile.from_yaml(args_cli.env_profile) if args_cli.env_profile else None + recorder_cfg = load_callable(args_cli.recorder_cfg)() if args_cli.recorder_cfg else None + if args_cli.record_metric_inputs: + recorder_cfg = recorder_cfg or ActionStateRecorderManagerCfg() + add_pose_recorder(recorder_cfg) env_cfg, success_term = setup_env_config( env_name=env_name, output_dir=output_dir, @@ -400,11 +396,7 @@ def main() -> None: num_envs=args_cli.num_envs, device=args_cli.device, generation_policy_params=generation_policy_params, - recorder_cfg=( - load_callable(args_cli.recorder_cfg)() - if args_cli.recorder_cfg - else _generation_recorder_cfg() if args_cli.record_metric_inputs else None - ), + recorder_cfg=recorder_cfg, record_step_success=args_cli.record_metric_inputs, env_profile=env_profile, enable_cameras=args_cli.record_images, From 2bd83526b9cf88883c58cb029a44f908c7a8c791 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Wed, 28 Oct 2026 20:29:40 -0500 Subject: [PATCH 11/13] fix: check the attempt log against its result and keep reports valid JSON - The report refuses a result whose attempt and success counts differ from its attempt log. - Run and gate input checks raise ValueError, so they also hold under python -O. - Non-finite numbers in a metric's details are reported as null. - A dataset whose env_args are not a JSON object reads as having none. - An empty metrics mapping computes no metrics instead of all of them. - Docs: how the report treats a result from another run; the example metric gets its own name; joint-limit metrics are not mentioned yet. - The end-to-end tests run the report through assert_ok_metrics. --- autodata_metrics/evidence.py | 26 ++++------ autodata_metrics/report.py | 52 ++++++++++++++----- .../e2e/test_mimicgen_data_generation.py | 13 +---- .../e2e/test_skillgen_bin_data_generation.py | 13 +---- autodata_tests/metrics/test_report.py | 18 +++++-- autodata_tests/utils/utils.py | 22 +++++--- docs/pages/references/metrics.rst | 25 ++++----- scripts/validate_dataset.py | 9 +--- 8 files changed, 94 insertions(+), 84 deletions(-) diff --git a/autodata_metrics/evidence.py b/autodata_metrics/evidence.py index 972b628..5f5a8b2 100644 --- a/autodata_metrics/evidence.py +++ b/autodata_metrics/evidence.py @@ -11,27 +11,21 @@ from collections.abc import Mapping from typing import Any -_MISSING = object() - -def lookup(record: Any, dotted_key: str) -> Any: - """``record[a][b]`` for ``dotted_key`` ``"a.b"``, or a sentinel that :func:`is_present` rejects.""" +def _has_path(record: Any, dotted_key: str) -> bool: + """Whether ``record[a][b]`` exists for ``dotted_key`` ``"a.b"``.""" for part in dotted_key.split("."): if not isinstance(record, Mapping) or part not in record: - return _MISSING + return False record = record[part] - return record + return True def read_env_args(h5_file: h5py.File) -> dict[str, Any]: - """The dataset's ``env_args``; its ``autodata`` entry was written by the producer.""" + """The dataset's ``env_args``, or empty when they are not a JSON object.""" raw_env_args = h5_file["data"].attrs.get("env_args", "{}") - return json.loads(raw_env_args.decode() if isinstance(raw_env_args, bytes) else raw_env_args) - - -def is_present(value: Any) -> bool: - """Whether a :func:`lookup` found its key.""" - return value is not _MISSING + env_args = json.loads(raw_env_args.decode() if isinstance(raw_env_args, bytes) else raw_env_args) + return env_args if isinstance(env_args, dict) else {} class Episode: @@ -88,9 +82,9 @@ def has(self, key: str) -> bool: if head == "source": return self.source is not None and (not rest or self.source.has(rest)) if head == "attempts": - return bool(self.attempts) and (not rest or all(is_present(lookup(line, rest)) for line in self.attempts)) + return bool(self.attempts) and (not rest or all(_has_path(line, rest) for line in self.attempts)) if head == "env_args": - return is_present(lookup(self.env_args, rest)) + return _has_path(self.env_args, rest) if head == "run": - return is_present(lookup(self.run, rest)) + return _has_path(self.run, rest) return bool(self.episodes) and all(key in episode for episode in self.episodes) diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index 41d7133..0a80c06 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -54,23 +54,36 @@ def check_same_run(evidence: Evidence) -> list[str]: Run ids alone are not enough: merging datasets keeps the first file's ``env_args``, so the exported episode count must match too. + + Raises: + ValueError: If the dataset, result and attempt log disagree on their run or their counts. """ num_episodes = len(evidence.episodes) - if evidence.result is not None: - assert ( - num_episodes == evidence.result["num_success"] - ), f"The dataset holds {num_episodes} episodes but the run counts {evidence.result['num_success']} successes." + result = evidence.result + if result is not None and num_episodes != result["num_success"]: + raise ValueError( + f"The dataset holds {num_episodes} episodes but the run counts {result['num_success']} successes." + ) + if result is not None and evidence.attempts is not None: + logged = len(evidence.attempts), sum(line["success"] for line in evidence.attempts) + if logged != (result["num_attempts"], result["num_success"]): + raise ValueError( + f"The attempt log holds {logged[0]} attempts and {logged[1]} successes but the result counts" + f" {result['num_attempts']} and {result['num_success']}." + ) entry = evidence.env_args.get("autodata") if entry is None: return ["the dataset has no autodata entry, so it is matched to its run by episode count only"] - assert entry["exported_episodes"] == num_episodes, ( - f"The dataset holds {num_episodes} episodes but its run exported {entry['exported_episodes']};" - " it was merged or edited after export." - ) + if entry["exported_episodes"] != num_episodes: + raise ValueError( + f"The dataset holds {num_episodes} episodes but its run exported {entry['exported_episodes']};" + " it was merged or edited after export." + ) run_ids = {entry["run_id"]} | {line["run_id"] for line in evidence.attempts or []} if "run_id" in evidence.run: run_ids.add(evidence.run["run_id"]) - assert len(run_ids) == 1, f"The dataset, result and attempt log come from different runs: {sorted(run_ids)}" + if len(run_ids) != 1: + raise ValueError(f"The dataset, result and attempt log come from different runs: {sorted(run_ids)}") return [] @@ -96,7 +109,7 @@ def evaluate(spec: MetricSpec, evidence: Evidence) -> dict[str, Any]: return {**row, "status": "error", "reason": f"{type(exc).__name__}: {exc}"} if not _is_finite(value.value): return {**row, "status": "error", "reason": f"non-finite value {value.value}"} - return {**row, "status": "ok", "value": value.value, "n": value.n, "details": dict(value.details)} + return {**row, "status": "ok", "value": value.value, "n": value.n, "details": _finite_or_none(value.details)} def _is_finite(value: Any) -> bool: @@ -106,6 +119,16 @@ def _is_finite(value: Any) -> bool: return isinstance(value, numbers.Real) and math.isfinite(value) +def _finite_or_none(value: Any) -> Any: + """``value`` with every non-finite number replaced by None, so the report stays valid JSON.""" + value = value.tolist() if hasattr(value, "tolist") else value + if isinstance(value, Mapping): + return {key: _finite_or_none(item) for key, item in value.items()} + if isinstance(value, list | tuple): + return [_finite_or_none(item) for item in value] + return None if isinstance(value, float) and not math.isfinite(value) else value + + def build_report( dataset: str, result_file: str | None = None, @@ -127,7 +150,8 @@ def build_report( source_evidence = Evidence(stack.enter_context(h5py.File(source, "r"))) if source else None evidence = Evidence(h5_file, result, attempts, source_evidence) warnings += check_same_run(evidence) - rows = {name: evaluate(spec, evidence) for name, spec in (metrics or REGISTERED_METRICS).items()} + metrics = REGISTERED_METRICS if metrics is None else metrics + rows = {name: evaluate(spec, evidence) for name, spec in metrics.items()} num_episodes = len(evidence.episodes) return { "schema_version": SCHEMA_VERSION, @@ -150,6 +174,9 @@ def gate_violations(report: Mapping[str, Any], gate_file: str) -> list[str]: The gate maps metric names to ``min``/``max`` bounds, an optional ``key`` into a dict value (default: the metric's primary entry) and an optional ``version``. A gated metric that is missing, not applicable, errored or at another version is a violation. + + Raises: + ValueError: If a gate entry bounds a dict value without a ``key`` and the metric has no primary entry. """ violations = [] for name, bound in yaml.safe_load(Path(gate_file).read_text(encoding="utf-8")).items(): @@ -162,7 +189,8 @@ def gate_violations(report: Mapping[str, Any], gate_file: str) -> list[str]: continue key = bound.get("key", row["primary"]) value = row["value"].get(key) if key else row["value"] - assert not isinstance(value, Mapping), f"Gate entry {name!r} needs a key into {sorted(value)}." + if isinstance(value, Mapping): + raise ValueError(f"Gate entry {name!r} needs a key into {sorted(value)}.") if not _is_finite(value): violations.append(f"{name}: {value!r} is not a finite number") continue diff --git a/autodata_tests/e2e/test_mimicgen_data_generation.py b/autodata_tests/e2e/test_mimicgen_data_generation.py index 08d996f..16b27f1 100644 --- a/autodata_tests/e2e/test_mimicgen_data_generation.py +++ b/autodata_tests/e2e/test_mimicgen_data_generation.py @@ -54,18 +54,7 @@ def _run_franka_cube_stack_mimicgen(num_envs: int, device: str) -> None: assert_valid_dataset(output_file, min_num_demos=GENERATION_NUM_TRIALS) assert_recorded_run(output_file, result_file) - report_file = os.path.join(temp_dir, "report.json") - run_subprocess([ - TestPaths.python_path, - os.path.join(TestPaths.scripts_dir, "validate_dataset.py"), - output_file, - "--report", - "--result_file", - result_file, - "--report_file", - report_file, - ]) - assert_ok_metrics(report_file, RUN_METRICS) + assert_ok_metrics(output_file, result_file, RUN_METRICS) @pytest.mark.with_subprocess diff --git a/autodata_tests/e2e/test_skillgen_bin_data_generation.py b/autodata_tests/e2e/test_skillgen_bin_data_generation.py index b464095..bf45200 100644 --- a/autodata_tests/e2e/test_skillgen_bin_data_generation.py +++ b/autodata_tests/e2e/test_skillgen_bin_data_generation.py @@ -74,18 +74,7 @@ def _run_franka_bin_stack_skillgen(num_envs: int, device: str) -> None: attempts = assert_recorded_run(output_file, result_file) # Every SkillGen attempt plans at least one transit, and the algorithm reports how long it took. assert all(attempt["extras"]["planning_time_s"] > 0 for attempt in attempts) - report_file = os.path.join(temp_dir, "report.json") - run_subprocess([ - TestPaths.python_path, - os.path.join(TestPaths.scripts_dir, "validate_dataset.py"), - output_file, - "--report", - "--result_file", - result_file, - "--report_file", - report_file, - ]) - assert_ok_metrics(report_file, RUN_METRICS) + assert_ok_metrics(output_file, result_file, RUN_METRICS) with open(result_file, encoding="utf-8") as result_handle: result = json.load(result_handle) diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py index 69579ac..1071cb9 100644 --- a/autodata_tests/metrics/test_report.py +++ b/autodata_tests/metrics/test_report.py @@ -82,11 +82,16 @@ def test_interrupted_rerun_is_reported_from_its_own_attempt_log(tmp_path): assert "belongs to another run" in report["warnings"][0] -def test_report_refuses_merged_data(tmp_path): +def test_report_refuses_records_whose_counts_disagree(tmp_path): entry = {"run_id": "run-a", "exported_episodes": 1} - with pytest.raises(AssertionError, match="merged or edited"): + with pytest.raises(ValueError, match="merged or edited"): build_report(_dataset(tmp_path / "merged.hdf5", num_episodes=2, entry=entry), metrics={}) + result_file = _run(tmp_path, "run-a") + _log(tmp_path, "run-a", [True, True]) + with pytest.raises(ValueError, match="attempt log holds 2 attempts"): + build_report(_dataset(tmp_path / "d.hdf5", entry=entry), result_file=result_file, metrics={}) + def test_gate_fails_on_bound_inapplicable_missing_and_outdated_metric(tmp_path): row = {"status": "ok", "value": 0.4, "primary": None, "version": 2} @@ -102,13 +107,18 @@ def test_gate_fails_on_bound_inapplicable_missing_and_outdated_metric(tmp_path): assert [violation.split(":")[0] for violation in violations] == ["low", "old", "absent", "gap"] -def test_non_finite_value_is_an_error_and_fails_the_gate(tmp_path): - report = build_report(_dataset(tmp_path / "d.hdf5"), metrics={"m": _spec("m", lambda ev: Value(float("nan"), n=1))}) +def test_non_finite_numbers_never_reach_the_report(tmp_path): + metrics = { + "m": _spec("m", lambda ev: Value(float("nan"), n=1)), + "d": _spec("d", lambda ev: Value(1.0, n=1, details={"interval_95": [0.5, float("inf")]})), + } + report = build_report(_dataset(tmp_path / "d.hdf5"), metrics=metrics) gate = tmp_path / "gate.yaml" gate.write_text("m: {min: 0}\n") assert report["metrics"]["m"]["status"] == "error" assert gate_violations(report, str(gate)) == ["m: error"] + assert report["metrics"]["d"]["details"] == {"interval_95": [0.5, None]} def test_metrics_import_without_torch_or_isaaclab(): diff --git a/autodata_tests/utils/utils.py b/autodata_tests/utils/utils.py index 8e15b97..14a47b6 100644 --- a/autodata_tests/utils/utils.py +++ b/autodata_tests/utils/utils.py @@ -7,6 +7,9 @@ import json import os +from autodata_tests.utils.constants import TestPaths +from autodata_tests.utils.subprocess import run_subprocess + def assert_valid_dataset(output_file: str, min_num_demos: int) -> None: """Assert the run wrote a dataset containing at least min_num_demos entries.""" @@ -87,21 +90,28 @@ def assert_recorded_run(dataset_file: str, result_file: str) -> list[dict]: return attempts -def assert_ok_metrics(report_file: str, expected_ok: set[str]) -> dict: - """Assert exactly ``expected_ok`` metrics computed a value; any other metric must say why it did not. +def assert_ok_metrics(dataset_file: str, result_file: str, expected_ok: set[str]) -> None: + """Report a generation run's metrics with ``validate_dataset.py`` and assert exactly ``expected_ok`` computed a value. A renamed recorded key or an adapter that stops recording a stream turns a metric not applicable, which this catches. - - Returns: - The report's metric rows. """ + report_file = os.path.join(os.path.dirname(dataset_file), "report.json") + run_subprocess([ + TestPaths.python_path, + os.path.join(TestPaths.scripts_dir, "validate_dataset.py"), + dataset_file, + "--report", + "--result_file", + result_file, + "--report_file", + report_file, + ]) with open(report_file, encoding="utf-8") as report_handle: rows = json.load(report_handle)["metrics"] ok = {name for name, row in rows.items() if row["status"] == "ok"} assert ok == expected_ok, f"Computed {sorted(ok)}, expected {sorted(expected_ok)}: {rows}" - return rows def assert_no_metric_inputs(dataset_file: str) -> None: diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst index b327d61..18a9ff8 100644 --- a/docs/pages/references/metrics.rst +++ b/docs/pages/references/metrics.rst @@ -19,8 +19,10 @@ Each metric reports one of three things: - ``not_applicable``, with the reason, such as a missing recorded input; - ``error``, when it fails. The other metrics are still computed. -The report refuses a dataset and a result file from different runs, because their run ids and exported -episode counts must match. +The report reads the run named by the dataset's run id. A result file from another run, such as the +earlier result of an interrupted rerun, is set aside with a warning, and the run is reported from its own +attempt log. The report refuses a dataset, result file and attempt log whose episode, attempt or success +counts disagree, as they do after datasets are merged. Gating ------ @@ -48,26 +50,22 @@ requires. Without that evidence, the report says which key is missing and does n .. code-block:: python - import numpy as np - from autodata_metrics import Value, metric @metric( - "planning_time", + "slowest_plan", area="diagnostics", signal="generation performance", scope="run", - unit="s per attempt", + unit="s", better="lower", - primary="p95", requires=["attempts.extras.planning_time_s"], ) - def planning_time(ev): - """p50 and p95 motion-planning time per attempt.""" - times = np.array([attempt["extras"]["planning_time_s"] for attempt in ev.attempts]) - p50, p95 = np.percentile(times, [50, 95]) - return Value({"p50": float(p50), "p95": float(p95)}, n=len(times)) + def slowest_plan(ev): + """Longest motion-planning time of any attempt.""" + times = [attempt["extras"]["planning_time_s"] for attempt in ev.attempts] + return Value(max(times), n=len(times)) An evidence key is one of: @@ -83,7 +81,6 @@ When you add an algorithm or an embodiment: - An algorithm reports per-attempt facts by overriding ``attempt_extras()``. Metrics that require those facts then apply to its runs. -- An embodiment's adapter describes its pose actions through ``metric_context()``. Add datasheet - ``hardware_limits`` to its YAML to enable the joint-limit metrics. +- An embodiment's adapter describes its pose actions through ``metric_context()``. - List the workflow's expected metrics in its end-to-end test, so a metric that stops applying fails the test. diff --git a/scripts/validate_dataset.py b/scripts/validate_dataset.py index fd8aa08..8b9bb43 100644 --- a/scripts/validate_dataset.py +++ b/scripts/validate_dataset.py @@ -178,14 +178,7 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) if args.report and len(args.files) != 1: parser.error("--report takes exactly one dataset") - report_options = { - "--result_file": args.result_file, - "--source": args.source, - "--metrics": args.metrics, - "--gate": args.gate, - "--report_file": args.report_file, - } - given = [option for option, value in report_options.items() if value] + given = [f"--{name}" for name in ("result_file", "source", "metrics", "gate", "report_file") if getattr(args, name)] if given and not args.report: parser.error(f"{', '.join(given)} only apply with --report") From 475e598247aaf88d53f4ea9db9970967c7346cef Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Wed, 28 Oct 2026 20:44:44 -0500 Subject: [PATCH 12/13] fix: refuse a source other than the run's and non-finite gate bounds - When the result records the run's source, the report refuses a --source with a different SHA-256, so source-based metrics read the dataset the run started from. - A gate entry whose min or max is not a finite number is an error instead of a bound that can never be violated. --- autodata_metrics/report.py | 15 +++++++++++++-- autodata_tests/metrics/test_report.py | 18 ++++++++++++++---- docs/pages/references/metrics.rst | 5 +++-- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index 0a80c06..d77d285 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -142,6 +142,9 @@ def build_report( result_file: Generation result (``generate_dataset.py --result_file``); its attempt log is read beside it. source: Source dataset the run started from. metrics: Metrics to compute; defaults to every registered metric. + + Raises: + ValueError: If the dataset, result, attempt log and source do not all come from one run. """ with contextlib.ExitStack() as stack: h5_file = stack.enter_context(h5py.File(dataset, "r")) @@ -150,6 +153,10 @@ def build_report( source_evidence = Evidence(stack.enter_context(h5py.File(source, "r"))) if source else None evidence = Evidence(h5_file, result, attempts, source_evidence) warnings += check_same_run(evidence) + source_record = input_record(source) + recorded_source = evidence.run.get("inputs", {}).get("source") + if source_record and recorded_source and source_record["sha256"] != recorded_source["sha256"]: + raise ValueError(f"The source {source} is not the one the run started from, {recorded_source['path']}.") metrics = REGISTERED_METRICS if metrics is None else metrics rows = {name: evaluate(spec, evidence) for name, spec in metrics.items()} num_episodes = len(evidence.episodes) @@ -158,7 +165,7 @@ def build_report( "inputs": { "dataset": input_record(dataset), "result_file": result_file, - "source": input_record(source), + "source": source_record, "run_id": run_id, "revision": code_revision(), }, @@ -176,10 +183,14 @@ def gate_violations(report: Mapping[str, Any], gate_file: str) -> list[str]: at another version is a violation. Raises: - ValueError: If a gate entry bounds a dict value without a ``key`` and the metric has no primary entry. + ValueError: If a gate entry has a bound that is not a finite number, or bounds a dict value without a + ``key`` and the metric has no primary entry. """ violations = [] for name, bound in yaml.safe_load(Path(gate_file).read_text(encoding="utf-8")).items(): + limits = [bound[side] for side in ("min", "max") if side in bound] + if not all(_is_finite(limit) for limit in limits): + raise ValueError(f"Gate entry {name!r} has a bound that is not a finite number: {limits}.") row = report["metrics"].get(name) if row is None or row["status"] != "ok": violations.append(f"{name}: {'not reported' if row is None else row['status']}") diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py index 1071cb9..cb6e87c 100644 --- a/autodata_tests/metrics/test_report.py +++ b/autodata_tests/metrics/test_report.py @@ -43,10 +43,12 @@ def _log(tmp_path: Path, run_id: str, successes: list[bool]) -> None: (tmp_path / f"result.{run_id}.attempts.jsonl").write_text("\n".join(lines) + "\n") -def _run(tmp_path: Path, run_id: str) -> str: +def _run(tmp_path: Path, run_id: str, source_sha256: str | None = None) -> str: """A completed run: its result file and attempt log.""" result_file = tmp_path / "result.json" run = {"run_id": run_id, "attempts_file": f"result.{run_id}.attempts.jsonl"} + if source_sha256 is not None: + run["inputs"] = {"source": {"path": "source.hdf5", "sha256": source_sha256}} result_file.write_text(json.dumps({"num_success": 1, "num_attempts": 1, "run": run})) _log(tmp_path, run_id, [True]) return str(result_file) @@ -82,18 +84,23 @@ def test_interrupted_rerun_is_reported_from_its_own_attempt_log(tmp_path): assert "belongs to another run" in report["warnings"][0] -def test_report_refuses_records_whose_counts_disagree(tmp_path): +def test_report_refuses_inputs_that_disagree_with_their_run(tmp_path): entry = {"run_id": "run-a", "exported_episodes": 1} with pytest.raises(ValueError, match="merged or edited"): build_report(_dataset(tmp_path / "merged.hdf5", num_episodes=2, entry=entry), metrics={}) + dataset = _dataset(tmp_path / "d.hdf5", entry=entry) result_file = _run(tmp_path, "run-a") _log(tmp_path, "run-a", [True, True]) with pytest.raises(ValueError, match="attempt log holds 2 attempts"): - build_report(_dataset(tmp_path / "d.hdf5", entry=entry), result_file=result_file, metrics={}) + build_report(dataset, result_file=result_file, metrics={}) + result_file = _run(tmp_path, "run-a", source_sha256="0" * 64) + with pytest.raises(ValueError, match="not the one the run started from"): + build_report(dataset, result_file=result_file, source=_dataset(tmp_path / "source.hdf5"), metrics={}) -def test_gate_fails_on_bound_inapplicable_missing_and_outdated_metric(tmp_path): + +def test_gate_flags_violations_and_refuses_a_non_finite_bound(tmp_path): row = {"status": "ok", "value": 0.4, "primary": None, "version": 2} missing = {"status": "ok", "value": {"p50": 1.0}, "primary": "p95", "version": 1} report = {"metrics": {"low": row, "old": row, "fine": row, "absent": {"status": "not_applicable"}, "gap": missing}} @@ -105,6 +112,9 @@ def test_gate_fails_on_bound_inapplicable_missing_and_outdated_metric(tmp_path): violations = gate_violations(report, str(gate)) assert [violation.split(":")[0] for violation in violations] == ["low", "old", "absent", "gap"] + gate.write_text("absent: {min: .nan}\n") + with pytest.raises(ValueError, match="not a finite number"): + gate_violations(report, str(gate)) def test_non_finite_numbers_never_reach_the_report(tmp_path): diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst index 18a9ff8..9a933ad 100644 --- a/docs/pages/references/metrics.rst +++ b/docs/pages/references/metrics.rst @@ -22,7 +22,8 @@ Each metric reports one of three things: The report reads the run named by the dataset's run id. A result file from another run, such as the earlier result of an interrupted rerun, is set aside with a warning, and the run is reported from its own attempt log. The report refuses a dataset, result file and attempt log whose episode, attempt or success -counts disagree, as they do after datasets are merged. +counts disagree, as they do after datasets are merged, and a ``--source`` other than the dataset the run +started from. Gating ------ @@ -35,7 +36,7 @@ Gating planning_time: {max: 5.0, key: p95} A gated metric also violates the gate if it is not applicable, fails, or reports a different -``version``. +``version``. A bound that is not a finite number is an error. Built-in metrics ---------------- From 3d4f4350a1276fafd3848c9d0db1c65867d843e1 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Wed, 28 Oct 2026 22:51:05 -0500 Subject: [PATCH 13/13] fix: require scalar gate bounds and qualify the source check in the docs - A gate bound must be an int or float that is finite; YAML booleans and mappings are errors instead of passing or failing the comparison. - Docs: the source is verified only when the result file records it. --- autodata_metrics/report.py | 7 ++++++- autodata_tests/metrics/test_report.py | 7 ++++--- docs/pages/references/metrics.rst | 4 ++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index d77d285..f8724aa 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -189,7 +189,7 @@ def gate_violations(report: Mapping[str, Any], gate_file: str) -> list[str]: violations = [] for name, bound in yaml.safe_load(Path(gate_file).read_text(encoding="utf-8")).items(): limits = [bound[side] for side in ("min", "max") if side in bound] - if not all(_is_finite(limit) for limit in limits): + if not all(_is_bound(limit) for limit in limits): raise ValueError(f"Gate entry {name!r} has a bound that is not a finite number: {limits}.") row = report["metrics"].get(name) if row is None or row["status"] != "ok": @@ -212,6 +212,11 @@ def gate_violations(report: Mapping[str, Any], gate_file: str) -> list[str]: return violations +def _is_bound(limit: Any) -> bool: + """Whether a gate bound is a finite number; YAML reads ``true`` and ``false`` as bools, which are not bounds.""" + return isinstance(limit, int | float) and not isinstance(limit, bool) and math.isfinite(limit) + + def _format_value(value: Any) -> str: if isinstance(value, Mapping): return ", ".join(f"{key}={_format_value(item)}" for key, item in value.items()) or "none" diff --git a/autodata_tests/metrics/test_report.py b/autodata_tests/metrics/test_report.py index cb6e87c..d5a5b90 100644 --- a/autodata_tests/metrics/test_report.py +++ b/autodata_tests/metrics/test_report.py @@ -112,9 +112,10 @@ def test_gate_flags_violations_and_refuses_a_non_finite_bound(tmp_path): violations = gate_violations(report, str(gate)) assert [violation.split(":")[0] for violation in violations] == ["low", "old", "absent", "gap"] - gate.write_text("absent: {min: .nan}\n") - with pytest.raises(ValueError, match="not a finite number"): - gate_violations(report, str(gate)) + for bad_bound in (".nan", "false", "{p95: 1}"): + gate.write_text(f"absent: {{min: {bad_bound}}}\n") + with pytest.raises(ValueError, match="not a finite number"): + gate_violations(report, str(gate)) def test_non_finite_numbers_never_reach_the_report(tmp_path): diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst index 9a933ad..de230ad 100644 --- a/docs/pages/references/metrics.rst +++ b/docs/pages/references/metrics.rst @@ -22,8 +22,8 @@ Each metric reports one of three things: The report reads the run named by the dataset's run id. A result file from another run, such as the earlier result of an interrupted rerun, is set aside with a warning, and the run is reported from its own attempt log. The report refuses a dataset, result file and attempt log whose episode, attempt or success -counts disagree, as they do after datasets are merged, and a ``--source`` other than the dataset the run -started from. +counts disagree, as they do after datasets are merged. When the result file records the run's source, the +report also refuses a different ``--source``; without that record, the source is not verified. Gating ------