diff --git a/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py b/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py index 08016f4..9c00320 100644 --- a/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py +++ b/autodata_interfaces/embodiments/absolute_pose_embodiment_adapter.py @@ -83,7 +83,7 @@ def metric_context(self) -> dict[str, Any]: These entries are not clipped. """ pose_action = { - name: {"action_slice": list(spec["pose_action_slice"]), "bound": None} + name: {"action_slice": list(spec["pose_action_slice"]), "bound": None, "absolute": True} for name, spec in self.eefs.items() if spec.get("pose_action_slice") is not None } diff --git a/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py b/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py index e948e3b..8cce101 100644 --- a/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py +++ b/autodata_interfaces/embodiments/bimanual_embodiment_adapter.py @@ -197,8 +197,8 @@ def metric_context(self) -> dict[str, Any]: Each holds a position, then a quaternion. These entries are not clipped. """ pose_action = { - self.left.name: {"action_slice": list(self.left_pose_slice), "bound": None}, - self.right.name: {"action_slice": list(self.right_pose_slice), "bound": None}, + self.left.name: {"action_slice": list(self.left_pose_slice), "bound": None, "absolute": True}, + self.right.name: {"action_slice": list(self.right_pose_slice), "bound": None, "absolute": True}, } return {**super().metric_context(), "pose_action": pose_action} diff --git a/autodata_interfaces/embodiments/embodiment_adapter.py b/autodata_interfaces/embodiments/embodiment_adapter.py index d0b9859..2b416c0 100644 --- a/autodata_interfaces/embodiments/embodiment_adapter.py +++ b/autodata_interfaces/embodiments/embodiment_adapter.py @@ -55,8 +55,9 @@ def metric_context(self) -> dict[str, Any]: The base returns the optional ``hardware_limits`` section of the embodiment config: datasheet joint limits keyed by joint-name pattern. Subclasses add ``pose_action``, which says for each end effector which part of - the action vector commands its pose (``action_slice``, a half-open ``[start, end)`` index range) and the - value the adapter clips those entries to (``bound``, or None when they are not clipped). + the action vector commands its pose (``action_slice``, a half-open ``[start, end)`` index range), the + value the adapter clips those entries to (``bound``, or None when they are not clipped), and whether they + command an absolute target pose or a change of pose (``absolute``). """ hardware_limits = (self.raw_config or {}).get("hardware_limits") return {"hardware_limits": hardware_limits} if hardware_limits else {} diff --git a/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py b/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py index 6cd8df0..d9f2d4a 100644 --- a/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py +++ b/autodata_interfaces/embodiments/single_arm_embodiment_adapter.py @@ -139,7 +139,7 @@ def metric_context(self) -> dict[str, Any]: They are clipped to ±1 when ``clip_pose_action_to_unit`` is set. """ bound = 1.0 if self.clip_pose_action_to_unit else None - pose_action = {self.eef_name: {"action_slice": [0, _DELTA_POSE_ACTION_DIM], "bound": bound}} + pose_action = {self.eef_name: {"action_slice": [0, _DELTA_POSE_ACTION_DIM], "bound": bound, "absolute": False}} return {**super().metric_context(), "pose_action": pose_action} @property diff --git a/autodata_interfaces/env/isaaclab_env_interface.py b/autodata_interfaces/env/isaaclab_env_interface.py index 9e4ebf6..165104c 100644 --- a/autodata_interfaces/env/isaaclab_env_interface.py +++ b/autodata_interfaces/env/isaaclab_env_interface.py @@ -46,8 +46,8 @@ def collect_autodata_entry(env: Any, embodiment_adapter: EmbodimentAdapter, run_id: str) -> dict[str, Any]: """The ``autodata`` entry of the datasets this env exports, read while the env is alive. - It holds the run id, the embodiment's metric context, and the robot's joint names with env 0's simulator - position limits [rad or m] and velocity limits [rad/s or m/s]. Unbounded limits are stored as None. + It holds the run id, the embodiment's metric context, and the robot's asset name and joint names with env 0's + simulator position limits [rad or m] and velocity limits [rad/s or m/s]. Unbounded limits are stored as None. :func:`~autodata_utils.dataset_metadata.merge_autodata_entry` writes it once the env has closed the file. Args: @@ -59,6 +59,7 @@ def collect_autodata_entry(env: Any, embodiment_adapter: EmbodimentAdapter, run_ robot = env.scene.articulations.get(embodiment_adapter.robot_asset_name) if robot is not None: entry["joints"] = { + "asset": embodiment_adapter.robot_asset_name, "names": list(robot.joint_names), "position_limits": _finite_or_none(as_torch(robot.data.joint_pos_limits)[0].cpu()), "velocity_limits": _finite_or_none(as_torch(robot.data.joint_vel_limits)[0].cpu()), diff --git a/autodata_metrics/evidence.py b/autodata_metrics/evidence.py index 3180c95..93335df 100644 --- a/autodata_metrics/evidence.py +++ b/autodata_metrics/evidence.py @@ -5,6 +5,7 @@ from __future__ import annotations +import copy import h5py import json import numpy as np @@ -118,6 +119,26 @@ def has(self, key: str) -> bool: return _has_path(self.run, rest) return bool(self.episodes) and all(key in episode for episode in self.episodes) + @property + def entry(self) -> Mapping[str, Any]: + """The ``autodata`` entry of ``env_args``, or empty.""" + return self.env_args.get("autodata", {}) + + def as_baseline_for(self, run: Evidence) -> Evidence: + """This source dataset, with the run's ``autodata`` fields where its own entry lacks them, so a baseline + reads the robot layout and limits. + + ``joints`` and ``hardware_limits`` fill in field by field. A ``pose_action`` the source records stays whole, + as a channel it leaves out is not a pose action. + """ + baseline = copy.copy(self) + entry = {**run.entry, **self.entry} + for key in ("joints", "hardware_limits"): + if key in self.entry and key in run.entry: + entry[key] = {**run.entry[key], **self.entry[key]} + baseline.env_args = {**self.env_args, "autodata": entry} + return baseline + @cached_property def camera_paths(self) -> frozenset[str]: """Paths of the camera streams, 4-D ``uint8`` arrays, that any episode records.""" diff --git a/autodata_metrics/records.py b/autodata_metrics/records.py index 35f9e56..5ea1cd1 100644 --- a/autodata_metrics/records.py +++ b/autodata_metrics/records.py @@ -45,9 +45,9 @@ class MapOf: NUMBER = "a finite number" -COUNT = "a non-negative integer" -DURATION = "a non-negative finite number" +NON_NEGATIVE = "a non-negative finite number" POSITIVE = "a positive finite number" +COUNT = "a non-negative integer" INPUT = {"path": str, "sha256": str} RUN = { @@ -60,8 +60,8 @@ class MapOf: "cpu": Maybe(str), "num_gpus": Maybe(COUNT), "num_envs": Maybe(COUNT), - "setup_time_s": Maybe(DURATION), - "generation_time_s": Maybe(DURATION), + "setup_time_s": Maybe(NON_NEGATIVE), + "generation_time_s": Maybe(NON_NEGATIVE), "attempts_file": Maybe(str), } RESULT = {"num_success": COUNT, "num_failures": Maybe(COUNT), "num_attempts": COUNT, "run": Maybe(RUN)} @@ -71,22 +71,25 @@ class MapOf: "success": bool, "start_time_s": NUMBER, "end_time_s": NUMBER, - "extras": Maybe({"planning_time_s": Maybe(DURATION), "planning_failed": Maybe(bool)}), + "extras": Maybe({"planning_time_s": Maybe(NON_NEGATIVE), "planning_failed": Maybe(bool)}), } ANNOTATION = { "mode": Maybe(("auto", "manual")), "input_episodes": COUNT, "processed_episodes": COUNT, "annotated_episodes": COUNT, - "setup_time_s": Maybe(DURATION), - "time_s": DURATION, + "setup_time_s": Maybe(NON_NEGATIVE), + "time_s": NON_NEGATIVE, } ENTRY = { "version": COUNT, "run_id": str, "exported_episodes": COUNT, - "pose_action": Maybe(MapOf({"action_slice": ListOf(int, 2), "bound": Maybe(NUMBER)})), + "pose_action": Maybe(MapOf({"action_slice": ListOf(COUNT, 2), "bound": Maybe(POSITIVE), "absolute": Maybe(bool)})), "annotation": Maybe(ANNOTATION), + "action_noise": Maybe(NUMBER), + "joints": Maybe({"asset": Maybe(str), "names": ListOf(str)}), + "hardware_limits": Maybe({"velocity": Maybe(MapOf(POSITIVE)), "acceleration": Maybe(MapOf(POSITIVE))}), } ENV_ARGS = {"autodata": Maybe(ENTRY), "sim_args": Maybe({"dt": Maybe(POSITIVE), "decimation": Maybe(POSITIVE)})} @@ -94,9 +97,9 @@ class MapOf: def parse(value: Any, shape: Any, where: str) -> Any: """``value`` checked against ``shape``, with its absent or null optional fields left out. - A shape is a type, a tuple of allowed values, :data:`NUMBER`, :data:`COUNT`, :data:`DURATION`, :data:`POSITIVE`, - a :class:`Maybe`, :class:`ListOf` or :class:`MapOf`, or a dict of field shapes; a dict's fields are required - unless their shape is a :class:`Maybe`. + A shape is a type, a tuple of allowed values, :data:`NUMBER`, :data:`NON_NEGATIVE`, :data:`POSITIVE`, + :data:`COUNT`, a :class:`Maybe`, :class:`ListOf` or :class:`MapOf`, or a dict of field shapes; a dict's fields + are required unless their shape is a :class:`Maybe`. Raises: InvalidRecord: Naming ``where`` and the field, if ``value`` does not have ``shape``. @@ -123,8 +126,8 @@ def parse(value: Any, shape: Any, where: str) -> Any: return {key: item for key, item in parsed.items() if item is not None} if isinstance(shape, tuple): _require(value in shape, value, f"one of {list(shape)}", where) - elif shape in (NUMBER, DURATION, POSITIVE): - in_range = is_finite_number(value) and (shape is NUMBER or value > 0 or (shape is DURATION and value == 0)) + elif shape in (NUMBER, NON_NEGATIVE, POSITIVE): + in_range = is_finite_number(value) and (shape is NUMBER or value > 0 or (shape is NON_NEGATIVE and value == 0)) _require(in_range, value, shape, where) elif shape is COUNT: _require(isinstance(value, int) and not isinstance(value, bool) and value >= 0, value, shape, where) @@ -141,6 +144,9 @@ def parse_env_args(env_args: Any, where: str) -> dict[str, Any]: if isinstance(version, int) and version > AUTODATA_ENTRY_VERSION: raise InvalidRecord(f"{where}.autodata was written by a newer AutoData (entry version {version}).") parsed = parse(env_args, ENV_ARGS, where) + for eef, layout in parsed.get("autodata", {}).get("pose_action", {}).items(): + start, end = layout["action_slice"] + _require(start < end, [start, end], "a non-empty range", f"{where}.autodata.pose_action.{eef}.action_slice") annotation = parsed.get("autodata", {}).get("annotation") if annotation is not None: counts = [annotation[name] for name in ("annotated_episodes", "processed_episodes", "input_episodes")] diff --git a/autodata_metrics/report.py b/autodata_metrics/report.py index 8e643d6..82153dc 100644 --- a/autodata_metrics/report.py +++ b/autodata_metrics/report.py @@ -182,7 +182,7 @@ def evaluate(spec: MetricSpec, evidence: Evidence) -> dict[str, Any]: "details": _finite_or_none(value.details), } if spec.baseline and evidence.source is not None: - row["baseline"] = evaluate(spec, evidence.source).get("value") + row["baseline"] = evaluate(spec, evidence.source.as_baseline_for(evidence)).get("value") return row diff --git a/autodata_metrics/validity.py b/autodata_metrics/validity.py index bf48171..e6dd237 100644 --- a/autodata_metrics/validity.py +++ b/autodata_metrics/validity.py @@ -8,6 +8,7 @@ import h5py import math import numpy as np +import re from autodata_metrics.evidence import Episode, Evidence from autodata_metrics.registry import NotApplicable, Value, check, metric @@ -26,6 +27,9 @@ """Annotation's per-step subtask signals, stored as a ``(steps,)`` column or a ``(1, steps)`` row.""" OBSERVED_POSE_PATHS = ("obs/datagen_info/eef_pose/", "obs/datagen_info/object_pose/") """Recorded poses that are observations; commanded targets are measured by ``action_rotation_error`` instead.""" +NEAR_LIMIT = 0.95 +"""Share of a datasheet limit from which an episode counts as running at the limit, which a simulator that clamps +there never exceeds.""" def _image_streams(episode: Episode, ev: Evidence) -> dict[str, h5py.Dataset]: @@ -109,10 +113,17 @@ def valid_rotations(episode: Episode, ev: Evidence) -> str | None: @check("pose_action_bounds", requires=["env_args.autodata.pose_action"]) def pose_action_bounds(episode: Episode, ev: Evidence) -> str | None: - """Fails an episode whose pose actions exceed the bound the embodiment clips them to.""" + """Fails an episode whose pose actions exceed the bound the embodiment clips them to, or lie beyond its actions.""" actions = episode["actions"] + outside = [ + f"{eef} {layout['action_slice']}" + for eef, layout in ev.entry["pose_action"].items() + if layout["action_slice"][1] > actions.shape[-1] + ] + if outside: + return f"pose action slices beyond the {actions.shape[-1]} action columns: {', '.join(outside)}" bad = [] - for eef, layout in ev.env_args["autodata"]["pose_action"].items(): + for eef, layout in ev.entry["pose_action"].items(): if layout.get("bound") is None: continue start, end = layout["action_slice"] @@ -235,12 +246,156 @@ def action_rotation_error(ev: Evidence) -> Value: """p50 and p95 deviation of the commanded target rotations from a valid rotation, over every step and end effector.""" errors = [] for episode in ev.episodes: - for eef in episode.children("obs/datagen_info/target_eef_pose"): + for eef in _pose_channels(ev, episode, "obs/datagen_info/target_eef_pose"): targets = episode[f"obs/datagen_info/target_eef_pose/{eef}"] - # A channel that commands no pose records all-zero targets, which are not rotations. - errors.append(_rotation_errors(targets[np.abs(targets).reshape(len(targets), -1).max(axis=1) > 0])) - errors = np.concatenate(errors) if errors else np.empty(0) - if not len(errors): - raise NotApplicable("no end effector commands a pose") - p50, p95 = np.percentile(errors, [50, 95]) - return Value({"p50": float(p50), "p95": float(p95)}, n=len(errors)) + errors.append(_rotation_errors(targets[_commanded(targets)])) + return _percentiles(errors, "no end effector commands a pose") + + +@metric( + "tracking_error", + area="validity", + signal="motion feasibility", + scope="dataset", + unit="mm", + sample="poses", + better="lower", + primary="p95", + requires=["env_args.autodata.pose_action", "obs/datagen_info/eef_pose", "obs/datagen_info/target_eef_pose"], + baseline=True, +) +def tracking_error(ev: Evidence) -> Value: + """p50 and p95 distance from each absolute end-effector target to the end effector one step later.""" + absolute = [eef for eef, layout in ev.entry["pose_action"].items() if layout.get("absolute")] + distances = [] + for episode in ev.episodes: + for eef in absolute: + target, observed = f"obs/datagen_info/target_eef_pose/{eef}", f"obs/datagen_info/eef_pose/{eef}" + if target not in episode or observed not in episode: + continue + targets, reached = episode[target][:-1], episode[observed][1:] + commanded = _commanded(targets) + distances.append(np.linalg.norm(targets[commanded, :3, 3] - reached[commanded, :3, 3], axis=1) * 1000) + return _percentiles(distances, "no pose action is recorded as absolute") + + +@metric( + "action_saturation", + area="validity", + signal="motion feasibility", + scope="dataset", + unit="fraction", + sample="steps", + better="lower", + requires=["env_args.autodata.pose_action"], + baseline=True, +) +def action_saturation(ev: Evidence) -> Value: + """Share of steps at which a pose action sits at the bound the embodiment clips it to.""" + layouts = [layout for layout in ev.entry["pose_action"].values() if "bound" in layout] + if not layouts: + raise NotApplicable("no pose action is clipped") + saturated = [] + for episode in ev.episodes: + actions = np.abs(episode["actions"]) + at_bound = [ + actions[:, slice(*layout["action_slice"])].max(axis=1, initial=0.0) >= layout["bound"] * (1 - 1e-6) + for layout in layouts + ] + saturated.append(np.any(at_bound, axis=0)) + saturated = np.concatenate(saturated) + return Value(float(saturated.mean()), n=len(saturated)) + + +@metric( + "motion_smoothness", + area="validity", + signal="motion quality", + scope="dataset", + unit="m/s³", + sample="poses", + better="lower", + primary="p95", + requires=["obs/datagen_info/eef_pose", "env_args.sim_args.dt", "env_args.sim_args.decimation"], + baseline=True, +) +def motion_smoothness(ev: Evidence) -> Value: + """p50 and p95 end-effector jerk, labelled with the run's largest action-noise scale.""" + step = ev.env_args["sim_args"]["dt"] * ev.env_args["sim_args"]["decimation"] + jerks = [ + np.linalg.norm(np.diff(episode[f"obs/datagen_info/eef_pose/{eef}"][:, :3, 3], n=3, axis=0), axis=1) / step**3 + for episode in ev.episodes + for eef in _pose_channels(ev, episode, "obs/datagen_info/eef_pose") + ] + noise = {"action_noise": ev.entry.get("action_noise")} + return _percentiles(jerks, "no end effector that commands a pose records four steps", noise) + + +@metric( + "joint_limit_ratio", + area="validity", + signal="motion feasibility", + scope="dataset", + unit="fraction of episodes", + sample="episodes", + better="lower", + primary="velocity", + requires=["env_args.autodata.joints.asset", "env_args.sim_args.dt", "env_args.sim_args.decimation"], + baseline=True, +) +def joint_limit_ratio(ev: Evidence) -> Value: + """Share of episodes that reach ``NEAR_LIMIT`` of a datasheet velocity or acceleration limit.""" + joints, datasheet = ev.entry["joints"], ev.entry.get("hardware_limits", {}) + limits = {kind: _datasheet(joints["names"], datasheet.get(kind, {})) for kind in ("velocity", "acceleration")} + limits = {kind: limit for kind, limit in limits.items() if np.isfinite(limit).any()} + if not limits: + raise NotApplicable("the embodiment config gives no hardware_limits for these joints") + path = f"states/articulation/{joints['asset']}/joint_velocity" + if not all(path in episode for episode in ev.episodes): + raise NotApplicable(f"not every episode records {path}") + step = ev.env_args["sim_args"]["dt"] * ev.env_args["sim_args"]["decimation"] + peaks = {kind: [] for kind in limits} + for episode in ev.episodes: + velocity = episode[path] + if velocity.shape[1:] != (len(joints["names"]),): + raise ValueError( + f"{episode.name} records joint velocities of shape {velocity.shape} for {len(joints['names'])} joints" + ) + motion = {"velocity": velocity, "acceleration": np.diff(velocity, axis=0) / step} + for kind, limit in limits.items(): + peaks[kind].append((np.abs(motion[kind]) / limit).max(axis=0, initial=0.0)) + value, details = {}, {} + for kind, per_episode in peaks.items(): + per_episode = np.array(per_episode) + value[kind] = float((per_episode.max(axis=1) >= NEAR_LIMIT).mean()) + details[f"{kind}_peak_ratio"] = float(per_episode.max()) + per_joint = np.where(np.isfinite(limits[kind]), per_episode.max(axis=0), -1.0) + details[f"{kind}_worst_joint"] = joints["names"][int(per_joint.argmax())] + return Value(value, n=len(ev.episodes), details=details) + + +def _pose_channels(ev: Evidence, episode: Episode, group: str) -> list[str]: + """The end effectors under ``group`` that command a pose action, or every one when the entry does not say.""" + commanding = ev.entry.get("pose_action") + return [eef for eef in episode.children(group) if commanding is None or eef in commanding] + + +def _commanded(targets: np.ndarray) -> np.ndarray: + """Which steps of ``targets`` hold a pose: all-zero targets mark steps or channels that command none.""" + return np.abs(targets).max(axis=tuple(range(1, targets.ndim))) > 0 + + +def _percentiles(samples: list[np.ndarray], empty: str, details: dict | None = None) -> Value: + values = np.concatenate(samples) if samples else np.empty(0) + if not len(values): + raise NotApplicable(empty) + p50, p95 = np.percentile(values, [50, 95]) + return Value({"p50": float(p50), "p95": float(p95)}, n=len(values), details=details or {}) + + +def _datasheet(names: list[str], limits: dict[str, float]) -> np.ndarray: + """Each joint's limit from the first pattern its name fully matches, infinite where none does.""" + return np.array( + [next((limit for pattern, limit in limits.items() if re.fullmatch(pattern, name)), np.inf) for name in names], + dtype=float, + ) diff --git a/autodata_metrics/viability.py b/autodata_metrics/viability.py index 7f94094..25dc0a0 100644 --- a/autodata_metrics/viability.py +++ b/autodata_metrics/viability.py @@ -87,7 +87,7 @@ def demos_per_gpu_hour(ev: Evidence) -> Value: if ev.run["num_gpus"] == 0: raise NotApplicable("the run used no GPU") gpu_hours = ev.run["generation_time_s"] / 3600 * ev.run["num_gpus"] - exported = ev.env_args["autodata"]["exported_episodes"] + exported = ev.entry["exported_episodes"] details = {key: ev.run.get(key) for key in ("gpu", "cpu", "device", "num_envs")} return Value(exported / gpu_hours, n=exported, details=details) @@ -108,7 +108,7 @@ def time_to_dataset(ev: Evidence) -> Value: Annotation that was manual, or recorded without its mode, is reported in ``uncounted_annotation_s`` instead. """ - annotation = ev.source.env_args.get("autodata", {}).get("annotation", {}) if ev.source else {} + annotation = ev.source.entry.get("annotation", {}) if ev.source else {} parts = {"setup_s": ev.run["setup_time_s"], "generation_s": ev.run["generation_time_s"]} uncounted = None if annotation: @@ -131,7 +131,7 @@ def time_to_dataset(ev: Evidence) -> Value: ) def annotation_pass_rate(ev: Evidence) -> Value: """Share of processed recordings that annotation exported into the source dataset.""" - annotation = ev.source.env_args["autodata"]["annotation"] + annotation = ev.source.entry["annotation"] if annotation["processed_episodes"] == 0: raise NotApplicable("annotation processed no episodes") rate = annotation["annotated_episodes"] / annotation["processed_episodes"] diff --git a/autodata_tests/e2e/test_mimicgen_data_generation.py b/autodata_tests/e2e/test_mimicgen_data_generation.py index e5584cc..dc7fd34 100644 --- a/autodata_tests/e2e/test_mimicgen_data_generation.py +++ b/autodata_tests/e2e/test_mimicgen_data_generation.py @@ -20,6 +20,9 @@ "correctness_pass_rate", "final_success_rate", "action_rotation_error", + "action_saturation", + "motion_smoothness", + "joint_limit_ratio", } diff --git a/autodata_tests/e2e/test_skillgen_bin_data_generation.py b/autodata_tests/e2e/test_skillgen_bin_data_generation.py index 8e6ec57..ef75446 100644 --- a/autodata_tests/e2e/test_skillgen_bin_data_generation.py +++ b/autodata_tests/e2e/test_skillgen_bin_data_generation.py @@ -31,6 +31,9 @@ "correctness_pass_rate", "final_success_rate", "action_rotation_error", + "action_saturation", + "motion_smoothness", + "joint_limit_ratio", } diff --git a/autodata_tests/metrics/test_records.py b/autodata_tests/metrics/test_records.py index 50bc4f0..b142a0c 100644 --- a/autodata_tests/metrics/test_records.py +++ b/autodata_tests/metrics/test_records.py @@ -29,17 +29,18 @@ def _no_git(monkeypatch): monkeypatch.setattr(report_module, "code_revision", lambda: None) -def _write_dataset(path: Path, entry: dict, num_episodes: int = 2) -> str: - pose = np.tile(np.eye(4), (3, 1, 1)) +def _write_dataset(path: Path, env_args: dict, num_episodes: int = 2) -> str: + pose = np.tile(np.eye(4), (4, 1, 1)) with h5py.File(path, "w") as h5_file: data = h5_file.create_group("data") - data.attrs["env_args"] = json.dumps({"env_name": "Test-v0", "autodata": entry}) + data.attrs["env_args"] = json.dumps(env_args) for index in range(num_episodes): episode = data.create_group(f"demo_{index}") - episode.create_dataset("actions", data=np.zeros((3, 7))) - episode.create_dataset("step_success", data=[False, True, True]) + episode.create_dataset("actions", data=np.zeros((4, 7))) + episode.create_dataset("step_success", data=[False, False, True, True]) episode.create_dataset("obs/datagen_info/eef_pose/arm", data=pose) episode.create_dataset("obs/datagen_info/target_eef_pose/arm", data=pose) + episode.create_dataset("states/articulation/robot/joint_velocity", data=np.zeros((4, 2))) return str(path) @@ -53,6 +54,7 @@ def _samples(tmp_path: Path) -> dict[str, dict]: "setup_time_s": 10.0, "time_s": 20.0, } + sim_args = {"dt": 0.01, "decimation": 5} source_entry = {"version": 1, "run_id": "annotation", "exported_episodes": 2, "annotation": annotation} run = { "run_id": "run-a", @@ -77,14 +79,18 @@ def _samples(tmp_path: Path) -> dict[str, dict]: "initial_object_positions": {"cube": [0.1, 0.2]}, "extras": {"planning_time_s": 0.5, "planning_failed": False}, } + entry = { + "version": 1, + "run_id": "run-a", + "exported_episodes": 2, + "pose_action": {"arm": {"action_slice": [0, 6], "bound": 1.0, "absolute": True}}, + "action_noise": 0.03, + "joints": {"asset": "robot", "names": ["j1", "j2"]}, + "hardware_limits": {"velocity": {"j[12]": 2.0}, "acceleration": {"j1": 10.0}}, + } return { - "entry": { - "version": 1, - "run_id": "run-a", - "exported_episodes": 2, - "pose_action": {"arm": {"action_slice": [0, 6], "bound": 1.0}}, - }, - "source_entry": source_entry, + "env_args": {"env_name": "Test-v0", "sim_args": sim_args, "autodata": entry}, + "source_env_args": {"env_name": "Test-v0", "sim_args": sim_args, "autodata": source_entry}, "result": {"algorithm": "mimicgen", "num_success": 2, "num_failures": 1, "num_attempts": 3, "run": run}, "attempt": attempt, } @@ -112,8 +118,8 @@ def _replaced(record: dict, path: tuple, value: object) -> dict: def _report_or_refusal(tmp_path: Path, records: dict) -> dict | None: """Build, summarize and serialize the report of these records; None when the report refuses them.""" - dataset = _write_dataset(tmp_path / "data.hdf5", records["entry"]) - source = _write_dataset(tmp_path / "source.hdf5", records["source_entry"]) + dataset = _write_dataset(tmp_path / "data.hdf5", records["env_args"]) + source = _write_dataset(tmp_path / "source.hdf5", records["source_env_args"]) result_file = tmp_path / "result.json" result_file.write_text( json.dumps(records["result"]).replace(json.dumps(SOURCE_RECORD), json.dumps(input_record(source))) @@ -141,7 +147,7 @@ def test_every_malformed_record_field_is_refused_or_reported(tmp_path): negative = {**samples["attempt"], "extras": {"planning_time_s": -5.0, "planning_failed": False}} assert _report_or_refusal(tmp_path, {**samples, "attempt": negative}) is None crashes = [] - for kind in ("entry", "source_entry", "result", "attempt"): + for kind in ("env_args", "source_env_args", "result", "attempt"): for path in _paths(samples[kind]): for wrong in WRONG_VALUES: records = {**samples, kind: _replaced(samples[kind], path, wrong)} diff --git a/autodata_tests/metrics/test_validity.py b/autodata_tests/metrics/test_validity.py index ab7a045..8264081 100644 --- a/autodata_tests/metrics/test_validity.py +++ b/autodata_tests/metrics/test_validity.py @@ -141,3 +141,130 @@ def test_action_rotation_error_measures_commanded_rotations(tmp_path): row = report["metrics"]["action_rotation_error"] assert row["value"]["p95"] == pytest.approx(1.01**2 - 1) and row["n"] == 4 assert report["num_valid_episodes"] == 1 + + +def _poses(positions: np.ndarray) -> np.ndarray: + poses = np.tile(np.eye(4), (len(positions), 1, 1)) + poses[:, :3, 3] = positions + return poses + + +def test_tracking_error_measures_absolute_targets_against_the_next_step(tmp_path): + reached = _poses(np.array([[0.01 * step, 0.0, 0.0] for step in range(5)])) + targets = _poses(np.vstack([reached[1:, :3, 3] + [0.0, 0.02, 0.0], [[0.0, 0.0, 0.0]]])) + targets[-1] = 0 + pose_action = { + "arm": {"action_slice": [0, 6], "absolute": True}, + "wrist": {"action_slice": [6, 7], "absolute": False}, + } + groups = ("obs/datagen_info/eef_pose", "obs/datagen_info/target_eef_pose") + episode = {"actions": np.zeros((5, 7)), f"{groups[0]}/arm": reached, f"{groups[1]}/arm": targets} + # A relative action and a channel that commands no pose action are not tracked. + episode.update({f"{group}/{eef}": _poses(np.zeros((5, 3))) for group in groups for eef in ("wrist", "body")}) + one_step = {key: array[:1] for key, array in episode.items()} + dataset = _write( + tmp_path / "d.hdf5", [episode, one_step], {"run_id": "r", "exported_episodes": 2, "pose_action": pose_action} + ) + + row = build_report(dataset, metrics=_metrics("tracking_error"))["metrics"]["tracking_error"] + + assert row["value"]["p50"] == pytest.approx(20.0) and row["n"] == 4 + + +@pytest.mark.parametrize( + "source_entry, baseline", [(None, 0.25), ({"run_id": "a", "exported_episodes": 1, "pose_action": {}}, None)] +) +def test_action_saturation_counts_steps_at_the_bound_and_borrows_a_missing_source_layout( + tmp_path, source_entry, baseline +): + # A source without a layout borrows the run's; one that records no pose actions keeps that. + actions = np.zeros((4, 7)) + actions[1, 2] = -1.0 + entry = {"run_id": "r", "exported_episodes": 1, "pose_action": {"arm": {"action_slice": [0, 6], "bound": 1.0}}} + source = _write(tmp_path / "source.hdf5", [{"actions": actions}], source_entry) + + report = build_report( + _write(tmp_path / "d.hdf5", [{"actions": actions}], entry), source=source, metrics=_metrics("action_saturation") + ) + + row = report["metrics"]["action_saturation"] + assert (row["value"], row["baseline"]) == (0.25, baseline) + + +def test_motion_smoothness_is_the_jerk_of_the_hand_labelled_with_the_noise(tmp_path): + step = 0.01 * 5 + positions = np.array([[2.0 * (index * step) ** 3, 0.0, 0.0] for index in range(6)]) + entry = {"run_id": "r", "exported_episodes": 1, "action_noise": 0.03} + episode = {"actions": np.zeros((6, 7)), "obs/datagen_info/eef_pose/arm": _poses(positions)} + + row = build_report(_write(tmp_path / "d.hdf5", [episode], entry), metrics=_metrics("motion_smoothness"))["metrics"][ + "motion_smoothness" + ] + + assert row["value"]["p50"] == pytest.approx(12.0) and row["details"]["action_noise"] == 0.03 + + +@pytest.mark.parametrize("action_slice, failed", [([1, 7], 0), ([6, 8], 1)]) +def test_pose_action_slices_must_lie_within_the_actions(tmp_path, action_slice, failed): + entry = { + "run_id": "r", + "exported_episodes": 1, + "pose_action": {"arm": {"action_slice": action_slice, "bound": 1.0}}, + } + + report = build_report(_write(tmp_path / "d.hdf5", [{"actions": np.zeros((3, 7))}], entry), metrics={}) + + assert report["check_failures"]["pose_action_bounds"]["failed"] == failed + + +def test_joint_limit_ratio_baseline_fills_what_its_source_entry_lacks_from_the_run(tmp_path): + # The source's own velocity limit applies to it, and the run fills in the acceleration limit and the robot asset + # the source's entry lacks. + joints = {"asset": "robot", "names": ["j1"]} + run = { + "run_id": "r", + "exported_episodes": 1, + "joints": joints, + "hardware_limits": {"velocity": {"j1": 2.0}, "acceleration": {"j1": 10.0}}, + } + source = { + "run_id": "a", + "exported_episodes": 1, + "joints": {"names": ["j1"]}, + "hardware_limits": {"velocity": {"j1": 1.0}}, + } + episode = {"actions": np.zeros((3, 7)), "states/articulation/robot/joint_velocity": np.array([[0.0], [1.0], [1.0]])} + + report = build_report( + _write(tmp_path / "d.hdf5", [episode], run), + source=_write(tmp_path / "s.hdf5", [episode], source), + metrics=_metrics("joint_limit_ratio"), + ) + + row = report["metrics"]["joint_limit_ratio"] + assert (row["value"], row["baseline"]) == ( + {"velocity": 0.0, "acceleration": 1.0}, + {"velocity": 1.0, "acceleration": 1.0}, + ) + + +def test_joint_limit_ratio_compares_velocity_and_acceleration_with_the_datasheet(tmp_path): + entry = { + "run_id": "r", + "exported_episodes": 3, + "joints": {"asset": "robot", "names": ["j1", "j2"]}, + "hardware_limits": {"velocity": {"j[12]": 2.0}, "acceleration": {"j1": 10.0}}, + } + # The first episode doubles j2's velocity limit and j1's acceleration limit; the others run at 96% and 94% of j1's. + velocities = [[[0.0, 4.0], [1.0, 0.0], [1.0, 0.0]], [[1.92, 0.0]] * 3, [[1.88, 0.0]] * 3] + episodes = [ + {"actions": np.zeros((3, 7)), "states/articulation/robot/joint_velocity": np.array(velocity)} + for velocity in velocities + ] + + row = build_report(_write(tmp_path / "d.hdf5", episodes, entry), metrics=_metrics("joint_limit_ratio"))["metrics"][ + "joint_limit_ratio" + ] + + assert row["value"] == pytest.approx({"velocity": 2 / 3, "acceleration": 1 / 3}) + assert (row["details"]["acceleration_peak_ratio"], row["details"]["velocity_worst_joint"]) == (2.0, "j2") diff --git a/autodata_tests/metrics/test_viability.py b/autodata_tests/metrics/test_viability.py index c0afed0..b996058 100644 --- a/autodata_tests/metrics/test_viability.py +++ b/autodata_tests/metrics/test_viability.py @@ -34,9 +34,7 @@ def test_time_to_dataset_counts_only_annotation_recorded_as_automatic(): run = {"setup_time_s": 20.0, "generation_time_s": 80.0} def measured(annotation: dict): - return time_to_dataset( - SimpleNamespace(source=SimpleNamespace(env_args={"autodata": {"annotation": annotation}}), run=run) - ) + return time_to_dataset(SimpleNamespace(source=SimpleNamespace(entry={"annotation": annotation}), run=run)) automatic = measured({"mode": "auto", "setup_time_s": 15.0, "time_s": 60.0}) manual = measured({"mode": "manual", "setup_time_s": 15.0, "time_s": 600.0}) diff --git a/autodata_tests/utils/test_validate_dataset.py b/autodata_tests/utils/test_validate_dataset.py index eee0200..e9599a5 100644 --- a/autodata_tests/utils/test_validate_dataset.py +++ b/autodata_tests/utils/test_validate_dataset.py @@ -26,12 +26,13 @@ def _create_dataset(path: Path, *, include_actions: bool = True) -> None: def test_main_returns_zero_for_valid_dataset(tmp_path: Path) -> None: - """A structurally valid dataset should produce a successful exit status.""" + """A structurally valid dataset should produce a successful exit status, with or without the report.""" dataset_path = tmp_path / "valid.hdf5" _create_dataset(dataset_path) assert validate_dataset.main([str(dataset_path)]) == 0 + assert validate_dataset.main([str(dataset_path), "--report"]) == 0 def test_main_returns_nonzero_for_invalid_dataset_by_default(tmp_path: Path) -> None: diff --git a/docs/pages/concepts/data_generator.rst b/docs/pages/concepts/data_generator.rst index 525100d..0abb004 100644 --- a/docs/pages/concepts/data_generator.rst +++ b/docs/pages/concepts/data_generator.rst @@ -171,11 +171,12 @@ Four layers report what happened: The ``run`` block is written only with ``--record_metric_inputs``. With that option, every exported dataset, whether from generation, annotation or retargeting, also carries an ``autodata`` entry in its ``env_args`` attribute. The entry holds the ``run_id`` and the number of exported episodes, the -embodiment's pose-action layout, the robot's joint names with their simulator limits, and the -embodiment's optional datasheet ``hardware_limits``. An annotated dataset's entry also records the -annotation mode (automatic or manual), its episode counts, and its setup and annotation times. For a -dataset generated with ``--result_file``, the ``run_id`` ties it to that run's attempt log and, once the -run completes, its result file. +embodiment's pose-action layout, the robot's asset and joint names with their simulator limits, and +the embodiment's optional datasheet ``hardware_limits``. A generated dataset's entry also records the +largest noise scale generation applies to actions (``action_noise``). An annotated dataset's entry also +records the annotation mode (automatic or manual), its episode counts, and its setup and annotation +times. For a dataset generated with ``--result_file``, the ``run_id`` ties it to that run's attempt log +and, once the run completes, its result file. The GenerationAlgorithm Plug-In ------------------------------- diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst index e4d451e..761369a 100644 --- a/docs/pages/references/metrics.rst +++ b/docs/pages/references/metrics.rst @@ -128,6 +128,7 @@ When you add an algorithm or an embodiment: - An algorithm reports per-attempt facts by overriding ``attempt_extras()``. Metrics that require those facts then apply to its runs. -- An embodiment's adapter describes its pose actions through ``metric_context()``. +- An embodiment's adapter describes its pose actions through ``metric_context()``. Add datasheet + ``hardware_limits`` to its YAML to compute ``joint_limit_ratio``. - List the workflow's expected metrics in its end-to-end test, so a metric that stops applying fails the test. diff --git a/scripts/generate_dataset.py b/scripts/generate_dataset.py index 2aa3604..93bb778 100755 --- a/scripts/generate_dataset.py +++ b/scripts/generate_dataset.py @@ -419,8 +419,6 @@ def main() -> None: ) PreStepDatagenInfoRecorder.datastream = datastream run_id = uuid.uuid4().hex - autodata_entry = collect_autodata_entry(env, embodiment_adapter, run_id) if args_cli.record_metric_inputs else None - # SkillGen needs one curobo planner per env. Mimic/DexMimic take no kwargs. motion_planners: dict | None = None alg_kwargs: dict = {} @@ -434,6 +432,14 @@ def main() -> None: ) alg_kwargs["motion_planners"] = motion_planners algorithm = get_algorithm(args_cli.alg, **alg_kwargs) + autodata_entry = None + if args_cli.record_metric_inputs: + noises = [spec.action_noise for specs in task_descriptor.subtasks.values() for spec in specs] + noises += [getattr(planner.config, "motion_noise_scale", 0.0) for planner in (motion_planners or {}).values()] + autodata_entry = { + **collect_autodata_entry(env, embodiment_adapter, run_id), + "action_noise": max(noises, default=0.0), + } attempts_file = attempts_file_for(args_cli.result_file, run_id) if autodata_entry and args_cli.result_file else None attempt_log = AttemptLog(attempts_file, run_id) if attempts_file else None