From f21e276c318c2ff353feb435453ad13893dddc40 Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Thu, 5 Nov 2026 17:18:38 -0600 Subject: [PATCH 1/2] Add the start layout and workspace diversity metrics start_layout_spread averages, over pairs of episodes, the root-sum-square of each rigid, articulated and deformable object's horizontal start distance, so an object that never moves adds nothing and pinning one can only lower it. workspace_spread averages how far one episode's hands pass from where the other's went: each recorded end-effector path is sampled every 2.5 mm between recorded steps and keeps one point per 5 mm cell, so retracing a path adds nothing, and a hand that moves more than 8 cm in one recorded step makes it not applicable. Both read at most 1,000 episodes in an order their actions set, with names ordering episodes whose actions repeat exactly, and the workspace measures at most 2,000 pairs. An object or a hand missing from an episode is named. The end-to-end metric checks expect both spreads once a run exports two episodes. --- autodata_metrics/__init__.py | 2 +- autodata_metrics/diversity.py | 183 +++++++++++++++++++++++ autodata_tests/metrics/test_diversity.py | 178 ++++++++++++++++++++++ autodata_tests/utils/utils.py | 7 +- 4 files changed, 367 insertions(+), 3 deletions(-) create mode 100644 autodata_metrics/diversity.py create mode 100644 autodata_tests/metrics/test_diversity.py diff --git a/autodata_metrics/__init__.py b/autodata_metrics/__init__.py index 8897644..98ee6f5 100644 --- a/autodata_metrics/__init__.py +++ b/autodata_metrics/__init__.py @@ -7,7 +7,7 @@ Importing this package registers the built-in metrics. """ -from autodata_metrics import validity, viability +from autodata_metrics import diversity, validity, viability from autodata_metrics.evidence import Episode, Evidence from autodata_metrics.registry import ( AREAS, diff --git a/autodata_metrics/diversity.py b/autodata_metrics/diversity.py new file mode 100644 index 0000000..1bea0bf --- /dev/null +++ b/autodata_metrics/diversity.py @@ -0,0 +1,183 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Diversity metrics: how different the demonstrations in a dataset are from one another. + +Each spread averages a distance over pairs of episodes: + +- ``start_layout_spread``: how far apart two episodes' starting scenes are [mm]. +- ``workspace_spread``: how far one episode's hands pass from where the other's went [mm]. + +An average over pairs estimates how different two demonstrations from the same producer are, so it does not grow with +the number of episodes. The metrics reference page defines each metric and when two values compare. +""" + +from __future__ import annotations + +import hashlib +import numpy as np + +from autodata_metrics.evidence import Episode, Evidence +from autodata_metrics.registry import NotApplicable, Value, metric + +MAX_EPISODES = 1000 +"""Episodes a spread reads at most.""" +MAX_WORKSPACE_PAIRS = 2000 +"""Episode pairs the workspace measures at most; the start layout uses every pair.""" +WORKSPACE_GRID_M = 0.005 +"""Grid in which a hand's path keeps one point per cell, so a path weighs the space it visits, not its length [m].""" +MAX_WORKSPACE_STEP_M = 0.08 +"""A hand's move in one recorded step above which the workspace does not apply [m].""" +START_GROUPS = ("rigid_object", "articulation", "deformable_object") +"""``initial_state`` groups whose entities make up a start layout.""" + + +@metric( + "start_layout_spread", + area="diversity", + signal="start layouts", + scope="dataset", + unit="mm", + sample="episodes", + requires=["actions", "initial_state"], + baseline=True, +) +def start_layout_spread(ev: Evidence) -> Value: + """Average distance between two episodes' starting scenes, counting every object. + + For each pair of episodes, each object's horizontal start distance is combined with the others' as a + root-sum-square: an object that never moves adds nothing, one that moves more adds more. Robots and deformable + objects count by their root and their nodes' centroid. 0 means every object starts at the same xy in every + episode. + """ + episodes = _sample(ev) + names, layouts = _start_layouts(episodes) + entities = {name: _pair_mean(1000 * layouts[:, k]) for k, name in enumerate(names)} + return Value(_pair_mean(1000 * layouts), n=len(episodes), details={"entities_mm": entities}) + + +@metric( + "workspace_spread", + area="diversity", + signal="hand workspace", + scope="dataset", + unit="mm", + sample="episodes", + requires=["actions", "obs/datagen_info/eef_pose"], + baseline=True, +) +def workspace_spread(ev: Evidence) -> Value: + """Average distance from where one episode's hands pass to where the other's went. + + A hand is each recorded end effector, followed by the origin of its frame as the embodiment records it, from the + episode's first recorded step. Its path keeps one point per :data:`WORKSPACE_GRID_M` cell, so retracing a path adds + nothing. Each kept point's distance to the nearest kept point of the other episode's path is averaged, both ways, + and halved; hands are averaged. 0 means the hands pass through the same space. + """ + episodes = _sample(ev) + hands = sorted(set().union(*(episode.children("obs/datagen_info/eef_pose") for episode in episodes))) + if not hands: + raise NotApplicable("records no end-effector poses") + hand_paths = [_hand_paths(episode, hands) for episode in episodes] + for k, hand in enumerate(hands): + move = max(np.linalg.norm(np.diff(paths[k], axis=0), axis=1).max(initial=0.0) for paths in hand_paths) + if move > MAX_WORKSPACE_STEP_M: + raise NotApplicable( + f"{hand} moves {100 * move:g} cm in one recorded step, over the {100 * MAX_WORKSPACE_STEP_M:g} cm limit" + ) + kept = [[_grid_points(path) for path in paths] for paths in hand_paths] + first, second = _workspace_pairs(len(episodes)) + gaps = 1000 * np.array([[_mean_gap(a, b) for a, b in zip(kept[i], kept[j])] for i, j in zip(first, second)]) + hands_mm = {hand: float(gaps[:, k].mean()) for k, hand in enumerate(hands)} + return Value(float(gaps.mean()), n=len(episodes), details={"pairs": len(first), "hands_mm": hands_mm}) + + +def _digest(episode: Episode) -> bytes: + actions = np.ascontiguousarray(episode["actions"]) + return hashlib.sha256(f"{actions.dtype.str}{actions.shape}".encode() + actions.tobytes()).digest() + + +def _sample(ev: Evidence) -> list[Episode]: + """The episodes a spread reads, at least two: the first :data:`MAX_EPISODES` in a ranking by a digest of each + episode's actions and how many episodes before it, by name, repeat them. Repeats spread through the ranking, and + file order matters only between episodes whose actions repeat exactly.""" + copies: dict[bytes, int] = {} + ranked = [] + for episode in sorted(ev.episodes, key=lambda episode: episode.name): + digest = _digest(episode) + copies[digest] = copies.get(digest, -1) + 1 + ranked.append((hashlib.sha256(digest + copies[digest].to_bytes(8, "little")).digest(), episode)) + episodes = [episode for _, episode in sorted(ranked, key=lambda item: item[0])[:MAX_EPISODES]] + if len(episodes) < 2: + raise NotApplicable("needs two episodes") + return episodes + + +def _pair_mean(rows: np.ndarray) -> float: + """The mean Euclidean distance between two rows, over every pair of rows.""" + rows = rows.reshape(len(rows), -1) + return float(np.concatenate([np.linalg.norm(rows[k + 1 :] - rows[k], axis=1) for k in range(len(rows) - 1)]).mean()) + + +def _starts(episode: Episode) -> dict[str, np.ndarray]: + """Each scene entity's start xy in the episode: rigid objects and articulations by their root, deformable objects + by their nodes' centroid [m].""" + found = {} + for group in START_GROUPS: + prefix = f"initial_state/{group}" + for name in episode.children(prefix) if prefix in episode else (): + if group == "deformable_object": + found[f"{group}/{name}"] = episode[f"{prefix}/{name}/nodal_position"][0].mean(axis=0)[:2] + else: + found[f"{group}/{name}"] = episode[f"{prefix}/{name}/root_pose"][0, :2] + return found + + +def _start_layouts(episodes: list[Episode]) -> tuple[list[str], np.ndarray]: + """The entities with a start pose, and each episode's start xy of each [m].""" + starts = [_starts(episode) for episode in episodes] + names = sorted(set().union(*starts)) + if not names: + raise NotApplicable("no start poses recorded") + for episode, found in zip(episodes, starts): + if missing := sorted(set(names) - set(found)): + raise NotApplicable(f"{missing[0]} has no start pose in episode {episode.name}") + return names, np.array([[found[name] for name in names] for found in starts], dtype=float) + + +def _hand_paths(episode: Episode, hands: list[str]) -> list[np.ndarray]: + """Each hand's position at each recorded step [m].""" + paths = [] + for hand in hands: + key = f"obs/datagen_info/eef_pose/{hand}" + if key not in episode: + raise NotApplicable(f"episode {episode.name} records no pose of {hand}") + paths.append(np.asarray(episode[key], dtype=float)[:, :3, 3]) + return paths + + +def _grid_points(path: np.ndarray) -> np.ndarray: + """The path sampled every half :data:`WORKSPACE_GRID_M` along its segments, keeping the first sample in each grid + cell, so every kept point lies on the path.""" + steps = np.diff(path, axis=0) + counts = np.maximum(1, np.ceil(np.linalg.norm(steps, axis=1) / (WORKSPACE_GRID_M / 2))).astype(int) + segment = np.repeat(np.arange(len(steps)), counts) + fraction = (np.arange(counts.sum()) - np.repeat(np.cumsum(counts) - counts, counts)) / np.repeat(counts, counts) + samples = np.concatenate([path[segment] + fraction[:, None] * steps[segment], path[-1:]]) + _, first = np.unique(np.floor(samples / WORKSPACE_GRID_M).astype(np.int64), axis=0, return_index=True) + return samples[np.sort(first)] + + +def _mean_gap(a: np.ndarray, b: np.ndarray) -> float: + """Half the sum of each path's mean distance from its kept points to the other's nearest kept point [m].""" + distances = np.linalg.norm(a[:, None] - b[None], axis=2) + return 0.5 * float(distances.min(axis=1).mean() + distances.min(axis=0).mean()) + + +def _workspace_pairs(count: int) -> tuple[np.ndarray, np.ndarray]: + """Index pairs of distinct episodes: all, or :data:`MAX_WORKSPACE_PAIRS` drawn with a fixed seed.""" + first, second = np.triu_indices(count, 1) + if len(first) <= MAX_WORKSPACE_PAIRS: + return first, second + chosen = np.sort(np.random.default_rng(0).choice(len(first), MAX_WORKSPACE_PAIRS, replace=False)) + return first[chosen], second[chosen] diff --git a/autodata_tests/metrics/test_diversity.py b/autodata_tests/metrics/test_diversity.py new file mode 100644 index 0000000..d64ac25 --- /dev/null +++ b/autodata_tests/metrics/test_diversity.py @@ -0,0 +1,178 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Known-answer tests of the diversity metrics.""" + +import h5py +import json +import numpy as np +from pathlib import Path + +import pytest + +from autodata_metrics import REGISTERED_METRICS, build_report +from autodata_metrics.diversity import _digest, _sample +from autodata_metrics.evidence import Evidence + + +def _write(path: Path, episodes: list[dict[str, np.ndarray]]) -> str: + with h5py.File(path, "w") as h5_file: + data = h5_file.create_group("data") + data.attrs["env_args"] = json.dumps({"env_name": "Test-v0"}) + for index, arrays in enumerate(episodes): + for key, array in arrays.items(): + data.create_dataset(f"demo_{index}/{key}", data=array) + return str(path) + + +def _row(dataset: str, name: str) -> dict: + return build_report(dataset, metrics={name: REGISTERED_METRICS[name]})["metrics"][name] + + +def _starts(**positions: list[float]) -> dict[str, np.ndarray]: + """An episode whose objects start at the given xy, or xyz.""" + poses = { + f"initial_state/rigid_object/{name}/root_pose": [[*position, *[0.0] * (3 - len(position)), 1.0, 0.0, 0.0, 0.0]] + for name, position in positions.items() + } + return {"actions": np.zeros((2, 7)), **poses} + + +def _poses(positions: np.ndarray) -> np.ndarray: + poses = np.tile(np.eye(4), (len(positions), 1, 1)) + poses[:, :3, 3] = positions + return poses + + +def _moving(**tracks: np.ndarray) -> dict[str, np.ndarray]: + """An episode whose hands follow the given tracks.""" + first = next(iter(tracks.values())) + return { + "actions": np.pad(first, ((0, 0), (0, 4))), + **{f"obs/datagen_info/eef_pose/{hand}": _poses(track) for hand, track in tracks.items()}, + } + + +def test_start_layout_spread_counts_every_object_so_pinning_one_lowers_it(tmp_path): + # x moves 2 cm and rises 10 cm, which does not count, and y moves 10 cm per episode while the table stays; a pair's + # distance is the root-sum-square of the objects' horizontal ones. + varied = [_starts(x=[0.02 * i, 0.0, 0.1 * i], y=[0.0, 0.1 * i], table=[0.5, 0.5]) for i in range(4)] + pinned = [_starts(x=[0.0, 0.0], y=[0.0, 0.1 * i], table=[0.5, 0.5]) for i in range(4)] + mean_step = np.mean([j - i for i in range(4) for j in range(i + 1, 4)]) + + varied_row = _row(_write(tmp_path / "varied.hdf5", varied), "start_layout_spread") + pinned_row = _row(_write(tmp_path / "pinned.hdf5", pinned), "start_layout_spread") + + assert varied_row["value"] == pytest.approx(1000 * np.hypot(0.02, 0.1) * mean_step) + assert pinned_row["value"] == pytest.approx(1000 * 0.1 * mean_step) + + +def test_a_deformable_object_starts_at_its_nodes_centroid(tmp_path): + # A rope's first node stays at the origin while its second lies 0, 10 and 20 cm away: its centroid moves 5 cm a step. + episodes = [ + { + "actions": np.zeros((2, 7)), + "initial_state/deformable_object/rope/nodal_position": [[[0.0] * 3, [0.1 * i, 0, 0]]], + } + for i in range(3) + ] + + row = _row(_write(tmp_path / "d.hdf5", episodes), "start_layout_spread") + + assert row["value"] == pytest.approx((50.0 + 100.0 + 50.0) / 3) + + +_LINE = np.linspace([0.0, 0.0, 0.0], [0.1, 0.0, 0.0], 11) +_WITH_BALL, _WITHOUT_BALL = _starts(cube=[0.0, 0.0], ball=[0.1, 0.0]), _starts(cube=[0.1, 0.0]) +_TWO_HANDS, _ONE_HAND = _moving(left=_LINE, right=_LINE), _moving(left=_LINE) + + +@pytest.mark.parametrize( + "name, episodes, reason", + [ + ("start_layout_spread", [_WITH_BALL], "needs two episodes"), + ("start_layout_spread", [_WITH_BALL, _WITHOUT_BALL], "rigid_object/ball has no start pose in episode demo_1"), + ("start_layout_spread", [_WITHOUT_BALL, _WITH_BALL], "rigid_object/ball has no start pose in episode demo_0"), + ("workspace_spread", [_TWO_HANDS, _ONE_HAND], "episode demo_1 records no pose of right"), + ("workspace_spread", [_ONE_HAND, _TWO_HANDS], "episode demo_0 records no pose of right"), + ], +) +def test_a_spread_says_why_it_does_not_apply(tmp_path, name, episodes, reason): + # Two episodes that hold the same actions are read in the opposite order once reversed, so in one of the two orders + # a spread reads first the episode missing what the other records. + row = _row(_write(tmp_path / "d.hdf5", episodes), name) + + assert row["status"] == "not_applicable" and row["reason"] == reason + + +def test_repeated_episodes_spread_through_the_episodes_a_spread_reads(tmp_path): + # 40 distinct episodes, 30 copies each: the 1,000 read include every one. + episodes = [{"actions": np.full((2, 7), float(k))} for k in range(40) for _ in range(30)] + path = _write(tmp_path / "d.hdf5", episodes) + + with h5py.File(path, "r") as h5_file: + sample = _sample(Evidence(h5_file)) + distinct = {_digest(episode) for episode in sample} + + assert len(sample) == 1000 and len(distinct) == 40 + + +def test_the_workspace_refuses_a_hand_that_moves_more_than_8_cm_in_one_step(tmp_path): + # The right hand moves 1 mm a step but once 9 cm, 0.5% of its steps; the left, which sorts first, stands still. + steps = np.array([0.001] * 199 + [0.09]) + episodes = [ + _moving(right=np.stack([np.concatenate([[0.0], np.cumsum(steps)]), np.full(201, side), np.zeros(201)], axis=1)) + | _moving(left=np.zeros((201, 3))) + for side in (0.0, 0.1) + ] + + row = _row(_write(tmp_path / "d.hdf5", episodes), "workspace_spread") + + assert row["status"] == "not_applicable" and "right moves 9 cm in one recorded step" in row["reason"] + + +def test_workspace_counts_a_retraced_path_once(tmp_path): + # Against a 30 cm line, the line with a 10 cm branch and the line traced five times with the same branch visit the + # same space. + line = np.linspace([0.0, 0.0, 0.0], [0.3, 0.0, 0.0], 31) + branch = np.linspace([0.3, 0.0, 0.0], [0.3, 0.1, 0.0], 11)[1:] + once = np.concatenate([line, branch]) + retraced = np.concatenate([line, line[-2::-1], line[1:], line[-2::-1], line[1:], branch]) + + values = [ + _row(_write(tmp_path / f"{i}.hdf5", [_moving(arm=line), _moving(arm=other)]), "workspace_spread") + for i, other in enumerate((once, retraced)) + ] + + assert values[0]["value"] == pytest.approx(values[1]["value"], abs=0.01) + + +def test_the_workspace_is_how_far_apart_two_episodes_hands_run(tmp_path): + # In one episode both hands sweep 10 cm along x, recorded at 0, 5 and 10 cm; in the other, the left hand rests + # 10 cm beyond the sweep's end and the right 3 cm beside its middle. An offset of 1.25 mm keeps the sweep's 2.5 mm + # samples off the 5 mm cell edges, so it keeps a point every 5 mm. The left sweep's points lie 150 mm from the + # resting hand on average, which lies 100 mm from the nearest of them: 125 mm. The right sweep's points at x [mm] lie + # hypot(30, x - 52.5) mm from the resting hand, which lies hypot(30, 2.5) mm from the nearest of them. + offset = np.full(3, 0.00125) + sweep = offset + [[0.0, 0.0, 0.0], [0.05, 0.0, 0.0], [0.1, 0.0, 0.0]] + beyond = np.tile(offset + [0.2, 0.0, 0.0], (3, 1)) + beside = np.tile(offset + [0.0525, 0.03, 0.0], (3, 1)) + right = 0.5 * (np.mean([np.hypot(30.0, x - 52.5) for x in range(0, 101, 5)]) + np.hypot(30.0, 2.5)) + episodes = [_moving(left=sweep, right=sweep), _moving(left=beyond, right=beside)] + + row = _row(_write(tmp_path / "d.hdf5", episodes), "workspace_spread") + + assert row["value"] == pytest.approx(0.5 * (125.0 + right), rel=1e-9) + + +def test_the_workspace_does_not_depend_on_the_order_of_episodes(tmp_path): + # 64 episodes make 2,016 pairs, more than the workspace measures; which ones depends on content, not order. + starts = np.random.default_rng(0).uniform(-0.3, 0.3, (64, 3)) + episodes = [_moving(arm=np.linspace(start, start + [0.1, 0.0, 0.0], 6)) for start in starts] + + values = [ + _row(_write(tmp_path / f"{label}.hdf5", ordered), "workspace_spread")["value"] + for label, ordered in (("forward", episodes), ("reversed", episodes[::-1])) + ] + + assert values[0] == pytest.approx(values[1], rel=1e-12) diff --git a/autodata_tests/utils/utils.py b/autodata_tests/utils/utils.py index 42bd977..efc2419 100644 --- a/autodata_tests/utils/utils.py +++ b/autodata_tests/utils/utils.py @@ -94,7 +94,8 @@ def assert_recorded_run(dataset_file: str, result_file: str) -> list[dict]: def assert_ok_metrics(dataset_file: str, result_file: str, expected_ok: set[str]) -> None: """Report a generation run's metrics with ``validate_dataset.py``; assert every episode passes the checks and - exactly ``expected_ok`` computed a value. + exactly ``expected_ok`` computed a value, with ``start_layout_spread`` and ``workspace_spread`` too when the run + exported two episodes or more. A renamed recorded key or an adapter that stops recording a stream turns a metric not applicable, which this catches. @@ -118,7 +119,9 @@ def assert_ok_metrics(dataset_file: str, result_file: str, expected_ok: set[str] ), f"{report['num_episodes'] - report['num_valid_episodes']} episodes failed checks: {report['check_failures']}" rows = report["metrics"] ok = {name for name, row in rows.items() if row["status"] == "ok"} - assert ok == expected_ok, f"Computed {sorted(ok)}, expected {sorted(expected_ok)}: {rows}" + spreads = {"start_layout_spread", "workspace_spread"} if report["num_valid_episodes"] >= 2 else set() + expected = expected_ok | spreads + assert ok == expected, f"Computed {sorted(ok)}, expected {sorted(expected)}: {rows}" def assert_no_metric_inputs(dataset_file: str) -> None: From 731349822b107b8b5a7db1a36580b5e58cdca85c Mon Sep 17 00:00:00 2001 From: Billy Okal Date: Thu, 5 Nov 2026 17:18:38 -0600 Subject: [PATCH 2/2] Document the diversity metrics and when they compare The metrics reference defines both spreads and says when two values compare: same task and objects, same hands recorded in the same frames. It gives the measured effect of the end-effector frame, the first recorded step and the recording rate on the test datasets and runs, and its console example is a real report that shows both. --- docs/pages/references/metrics.rst | 63 ++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst index 761369a..366f7eb 100644 --- a/docs/pages/references/metrics.rst +++ b/docs/pages/references/metrics.rst @@ -11,28 +11,36 @@ without it, most metrics report that their inputs are missing. python scripts/validate_dataset.py datasets/generated.hdf5 --report \ --result_file datasets/generated.json \ --source datasets/annotated_datasets/dataset_franka_annotated.hdf5 \ + --gate datasets/gate.yaml \ --report_file datasets/generated_report.json The command prints a summary; ``--report_file`` writes the full report, with every detail, as JSON. .. code-block:: text - AutoData metrics · gr1.hdf5 · run e79e37b1 · source dataset_gr1_annotated.hdf5 - 6/6 episodes valid · 7 computed · 3 not computed · gate failed on 2 of 4 + AutoData metrics · generated.hdf5 · run cc273b3f · source dataset_franka_annotated.hdf5 + 20/20 episodes valid · 11 computed · 5 not computed · gate failed on 1 of 4 - Value Source Based on + Value Source Based on VALIDITY - ✗ action_rotation_error p95 0.0345 3.58e-7 1,612 poses - ✓ correctness_pass_rate 100% 6 episodes - ✓ final_success_rate 100% 6 episodes - image_stream_pass_rate 100% 6 episodes + action_rotation_error p95 8.34e-7 4.77e-7 4,803 poses + action_saturation 4.06% 0% 4,803 steps + ✓ correctness_pass_rate 100% 20 episodes + final_success_rate 95% 20 episodes + joint_limit_ratio velocity 40% 10% 20 episodes + motion_smoothness p95 142 m/s³ 40 m/s³ 4,743 poses + DIVERSITY + ✓ start_layout_spread 217 mm 222 mm 20 episodes + ✓ workspace_spread 46.9 mm 44.7 mm 20 episodes VIABILITY - demos_per_gpu_hour 65.8 demos/GPU-h 6 demos - ✗ generation_success_rate 21.4% 28 attempts · 95% CI 10.2–39.5% - time_to_dataset 6 min 11 s 1 run + demos_per_gpu_hour 863 demos/GPU-h 20 demos + ✗ generation_success_rate 40% 50 attempts · 95% CI 27.6–53.8% + time_to_dataset 105 s 1 run NOT COMPUTED not reported by this algorithm: planning_failure_rate, planning_time + no camera streams: image_stream_pass_rate + no pose action is recorded as absolute: tracking_error needs a --source annotated with --record_metric_inputs: annotation_pass_rate - The second line is the verdict: valid episodes, how many metrics were computed, and, with ``--gate``, @@ -88,6 +96,41 @@ Built-in metrics :autodata_metrics_table: +Diversity metrics +----------------- + +The diversity metrics describe how different the demonstrations in a dataset are from one another. Each +averages a distance over pairs of episodes, so it needs at least two: + +- ``start_layout_spread`` [mm]: how far apart two episodes' starting scenes are. Each object's horizontal + start distance is combined with the others' as a root-sum-square, so an object that never moves adds nothing + and one that moves more adds more. Robots count by their root, deformable objects by their nodes' centroid. +- ``workspace_spread`` [mm]: how far one episode's hands pass from where the other's went. A hand is each + recorded end effector, followed by the origin of its frame as the embodiment records it (a whole-body + controller records its hands relative to the robot's body), from the episode's first recorded step. Its + path, sampled every 2.5 mm between recorded steps, keeps one point per 5 mm cell, so retracing a path adds + nothing; the mean distance from one path's kept points to the other's nearest kept points is averaged over + the two directions, then over hands. Beyond 63 episodes, it measures 2,000 pairs drawn with a fixed seed + among the episodes it reads. + +An average over pairs estimates how different two demonstrations from the same producer are, so it does not +grow with the number of episodes. The spreads read episodes in a ranking by a digest of each episode's actions +and how many earlier episodes repeat them, at most the first 1,000, so the order of episodes in the file does +not matter unless two episodes repeat each other's actions exactly. + +Comparing datasets +~~~~~~~~~~~~~~~~~~ + +Two spreads compare only on the same task and scene objects, and two workspaces only with the same hands +recorded in the same frames; each metric's details list its objects or hands. On the Franka stack's test +datasets and runs, moving a hand's frame between its MimicGen and SkillGen frames, 10.34 cm along the gripper, +moved the workspace by up to 3.3%, and the same distance across the gripper by up to 42.2%. Workspace values +assume each path starts at its episode's first recorded step, the reset pose. On the test datasets and +generated runs, leaving that step out moved them by up to 7.1%, and halving the recording rate moved them by +under 1%, except for the GR1 pick-and-place seeds and run and the fridge seeds, which it made not applicable. +A hand that moves far between recorded steps blurs its path, so the workspace is not applicable when any hand +moves more than 8 cm in one recorded step. + Adding a metric ---------------