diff --git a/autodata_metrics/__init__.py b/autodata_metrics/__init__.py index 8897644..98ee6f5 100644 --- a/autodata_metrics/__init__.py +++ b/autodata_metrics/__init__.py @@ -7,7 +7,7 @@ Importing this package registers the built-in metrics. """ -from autodata_metrics import validity, viability +from autodata_metrics import diversity, validity, viability from autodata_metrics.evidence import Episode, Evidence from autodata_metrics.registry import ( AREAS, diff --git a/autodata_metrics/diversity.py b/autodata_metrics/diversity.py new file mode 100644 index 0000000..49fb71e --- /dev/null +++ b/autodata_metrics/diversity.py @@ -0,0 +1,200 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Diversity metrics: how different the demonstrations in a dataset are from one another. + +Each spread averages a distance over pairs of episodes: + +- ``start_layout_spread``: how far apart two episodes' starting scenes are [mm]. +- ``workspace_spread``: how far one episode's hands pass from where the other's went [mm]. + +An average over pairs estimates how different two demonstrations from the same producer are, so it does not grow with +the number of episodes. The metrics reference page defines each metric and when two values compare. +""" + +from __future__ import annotations + +import hashlib +import numpy as np +import weakref + +from autodata_metrics.evidence import Episode, Evidence +from autodata_metrics.registry import NotApplicable, Value, metric + +MAX_EPISODES = 1000 +"""Episodes a spread reads at most: the first in a ranking by a digest of each episode's actions and how many earlier +episodes repeat them, so file order matters only between episodes whose actions repeat exactly.""" +MAX_WORKSPACE_PAIRS = 2000 +"""Episode pairs the workspace measures at most, drawn with a fixed seed among the episodes it reads; the start layout +uses every pair.""" +WORKSPACE_GRID_M = 0.005 +"""Grid in which a hand's path keeps one point per cell, so a path weighs the space it visits, not its length [m].""" +MAX_WORKSPACE_STEP_M = 0.08 +"""A hand's 99th-percentile move per recorded step above which the workspace does not apply [m].""" +START_GROUPS = ("rigid_object", "articulation", "deformable_object") +"""``initial_state`` groups whose entities make up a start layout.""" + + +@metric( + "start_layout_spread", + area="diversity", + signal="start layouts", + scope="dataset", + unit="mm", + sample="episodes", + requires=["actions", "initial_state"], + baseline=True, +) +def start_layout_spread(ev: Evidence) -> Value: + """Average distance between two episodes' starting scenes, counting every object. + + For each pair of episodes, each object's horizontal start distance is combined with the others' as a + root-sum-square: an object that never moves adds nothing, one that moves more adds more. Robots and deformable + objects count by their root and their nodes' centroid. 0 means every object starts at the same place in every + episode. + """ + episodes = _sample(ev) + names, layouts = _start_layouts(ev) + entities = {name: _pair_mean(1000 * _pairwise(layouts[:, k])) for k, name in enumerate(names)} + value = _pair_mean(1000 * _pairwise(layouts.reshape(len(layouts), -1))) + return Value(value, n=len(episodes), details={"entities_mm": entities}) + + +@metric( + "workspace_spread", + area="diversity", + signal="hand workspace", + scope="dataset", + unit="mm", + sample="episodes", + requires=["actions", "obs/datagen_info/eef_pose"], + baseline=True, +) +def workspace_spread(ev: Evidence) -> Value: + """Average distance from where one episode's hands pass to where the other's went. + + A hand is each recorded end effector, followed by the origin of its frame as the embodiment records it, from the + episode's first recorded step. Its path keeps one point per :data:`WORKSPACE_GRID_M` cell, so retracing a path adds + nothing. Each kept point's distance to the nearest kept point of the other episode's path is averaged, both ways, + and halved; hands are averaged. 0 means the hands pass through the same space. + """ + episodes = _sample(ev) + hands = sorted(set().union(*(episode.children("obs/datagen_info/eef_pose") for episode in episodes))) + if not hands: + raise NotApplicable("records no end-effector poses") + hand_paths = [_hand_paths(episode, hands) for episode in episodes] + for k, hand in enumerate(hands): + moves = np.concatenate([np.linalg.norm(np.diff(paths[k], axis=0), axis=1) for paths in hand_paths]) + move = float(np.percentile(moves, 99)) if len(moves) else 0.0 + if move > MAX_WORKSPACE_STEP_M: + raise NotApplicable( + f"{hand} moves {100 * move:g} cm per step (99th percentile), over the " + f"{100 * MAX_WORKSPACE_STEP_M:g} cm limit" + ) + kept = [[_grid_points(path) for path in paths] for paths in hand_paths] + first, second = _workspace_pairs(len(episodes)) + gaps = 1000 * np.array([[_mean_gap(a, b) for a, b in zip(kept[i], kept[j])] for i, j in zip(first, second)]) + hands_mm = {hand: float(gaps[:, k].mean()) for k, hand in enumerate(hands)} + return Value(float(gaps.mean()), n=len(episodes), details={"pairs": len(first), "hands_mm": hands_mm}) + + +_SAMPLES: weakref.WeakKeyDictionary[Evidence, list[Episode]] = weakref.WeakKeyDictionary() + + +def _digest(episode: Episode) -> bytes: + actions = np.ascontiguousarray(episode["actions"]) + return hashlib.sha256(f"{actions.dtype.str}{actions.shape}".encode() + actions.tobytes()).digest() + + +def _sample(ev: Evidence) -> list[Episode]: + """The episodes a spread reads, at least two: the first :data:`MAX_EPISODES` in a ranking by a digest of each + episode's actions and how many episodes before it, by name, repeat them, so repeats spread through the ranking.""" + if ev not in _SAMPLES: + copies: dict[bytes, int] = {} + ranked = [] + for episode in sorted(ev.episodes, key=lambda episode: episode.name): + digest = _digest(episode) + copies[digest] = copies.get(digest, -1) + 1 + ranked.append((hashlib.sha256(digest + copies[digest].to_bytes(8, "little")).digest(), episode)) + _SAMPLES[ev] = [episode for _, episode in sorted(ranked, key=lambda item: item[0])[:MAX_EPISODES]] + if len(_SAMPLES[ev]) < 2: + raise NotApplicable("needs two episodes") + return _SAMPLES[ev] + + +def _pairwise(rows: np.ndarray) -> np.ndarray: + """Euclidean distances between the rows, centred first, since expanding the squares of rows far from the origin loses + precision.""" + rows = rows - rows.mean(axis=0) + squares = (rows * rows).sum(axis=1) + return np.sqrt(np.maximum(squares[:, None] + squares[None] - 2 * rows @ rows.T, 0.0)) + + +def _pair_mean(distances: np.ndarray) -> float: + """The mean of a symmetric matrix of pair distances over its pairs.""" + return float(distances[np.triu_indices(len(distances), 1)].mean()) + + +def _starts(episode: Episode) -> dict[str, np.ndarray]: + """Each scene entity's start xy in the episode: rigid objects and articulations by their root, deformable objects + by their nodes' centroid [m].""" + found = {} + for group in START_GROUPS: + prefix = f"initial_state/{group}" + for name in episode.children(prefix) if prefix in episode else (): + root, nodes = f"{prefix}/{name}/root_pose", f"{prefix}/{name}/nodal_position" + if root in episode and len(pose := episode[root]): + found[f"{group}/{name}"] = pose[0, :2] + elif nodes in episode and len(positions := episode[nodes]): + found[f"{group}/{name}"] = positions[0].mean(axis=0)[:2] + return found + + +def _start_layouts(ev: Evidence) -> tuple[list[str], np.ndarray]: + """The entities with a start pose, and each episode's start xy of each [m].""" + starts = [_starts(episode) for episode in _sample(ev)] + names = sorted(set().union(*starts)) + if not names: + raise NotApplicable("no start poses recorded") + for episode, found in zip(_sample(ev), starts): + if missing := sorted(set(names) - set(found)): + raise NotApplicable(f"{missing[0]} has no start pose in episode {episode.name}") + return names, np.array([[found[name] for name in names] for found in starts], dtype=float) + + +def _hand_paths(episode: Episode, hands: list[str]) -> list[np.ndarray]: + """Each hand's position at each recorded step [m].""" + paths = [] + for hand in hands: + key = f"obs/datagen_info/eef_pose/{hand}" + if key not in episode: + raise NotApplicable(f"episode {episode.name} records no pose of {hand}") + paths.append(np.asarray(episode[key], dtype=float)[:, :3, 3]) + return paths + + +def _grid_points(path: np.ndarray) -> np.ndarray: + """The path sampled every half :data:`WORKSPACE_GRID_M` along its segments, keeping the first sample in each grid + cell, so every kept point lies on the path.""" + steps = np.diff(path, axis=0) + counts = np.maximum(1, np.ceil(np.linalg.norm(steps, axis=1) / (WORKSPACE_GRID_M / 2))).astype(int) + segment = np.repeat(np.arange(len(steps)), counts) + fraction = (np.arange(counts.sum()) - np.repeat(np.cumsum(counts) - counts, counts)) / np.repeat(counts, counts) + samples = np.concatenate([path[segment] + fraction[:, None] * steps[segment], path[-1:]]) + _, first = np.unique(np.floor(samples / WORKSPACE_GRID_M).astype(np.int64), axis=0, return_index=True) + return samples[np.sort(first)] + + +def _mean_gap(a: np.ndarray, b: np.ndarray) -> float: + """Half the sum of each path's mean distance from its kept points to the other's nearest kept point [m].""" + distances = np.linalg.norm(a[:, None] - b[None], axis=2) + return 0.5 * float(distances.min(axis=1).mean() + distances.min(axis=0).mean()) + + +def _workspace_pairs(count: int) -> tuple[np.ndarray, np.ndarray]: + """Index pairs of distinct episodes: all, or :data:`MAX_WORKSPACE_PAIRS` drawn with a fixed seed.""" + first, second = np.triu_indices(count, 1) + if len(first) <= MAX_WORKSPACE_PAIRS: + return first, second + chosen = np.sort(np.random.default_rng(0).choice(len(first), MAX_WORKSPACE_PAIRS, replace=False)) + return first[chosen], second[chosen] diff --git a/autodata_tests/metrics/test_diversity.py b/autodata_tests/metrics/test_diversity.py new file mode 100644 index 0000000..293ce48 --- /dev/null +++ b/autodata_tests/metrics/test_diversity.py @@ -0,0 +1,179 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Known-answer tests of the diversity metrics.""" + +import h5py +import json +import numpy as np +from pathlib import Path + +import pytest + +from autodata_metrics import REGISTERED_METRICS, build_report +from autodata_metrics.diversity import _digest, _sample +from autodata_metrics.evidence import Evidence + + +def _write(path: Path, episodes: list[dict[str, np.ndarray]]) -> str: + with h5py.File(path, "w") as h5_file: + data = h5_file.create_group("data") + data.attrs["env_args"] = json.dumps({"env_name": "Test-v0"}) + for index, arrays in enumerate(episodes): + for key, array in arrays.items(): + data.create_dataset(f"demo_{index}/{key}", data=array) + return str(path) + + +def _row(dataset: str, name: str) -> dict: + return build_report(dataset, metrics={name: REGISTERED_METRICS[name]})["metrics"][name] + + +def _starts(**positions: list[float]) -> dict[str, np.ndarray]: + """An episode whose objects start at the given xy, or xyz.""" + poses = { + f"initial_state/rigid_object/{name}/root_pose": [[*position, *[0.0] * (3 - len(position)), 1.0, 0.0, 0.0, 0.0]] + for name, position in positions.items() + } + return {"actions": np.zeros((2, 7)), **poses} + + +def _poses(positions: np.ndarray) -> np.ndarray: + poses = np.tile(np.eye(4), (len(positions), 1, 1)) + poses[:, :3, 3] = positions + return poses + + +def _moving(**tracks: np.ndarray) -> dict[str, np.ndarray]: + """An episode whose hands follow the given tracks.""" + first = next(iter(tracks.values())) + return { + "actions": np.pad(first, ((0, 0), (0, 4))), + **{f"obs/datagen_info/eef_pose/{hand}": _poses(track) for hand, track in tracks.items()}, + } + + +def test_start_layout_spread_counts_every_object_so_pinning_one_lowers_it(tmp_path): + # x moves 2 cm and rises 10 cm, which does not count, and y moves 10 cm per episode while the table stays; a pair's + # distance is the root-sum-square of the objects' horizontal ones. + varied = [_starts(x=[0.02 * i, 0.0, 0.1 * i], y=[0.0, 0.1 * i], table=[0.5, 0.5]) for i in range(4)] + pinned = [_starts(x=[0.0, 0.0], y=[0.0, 0.1 * i], table=[0.5, 0.5]) for i in range(4)] + mean_step = np.mean([j - i for i in range(4) for j in range(i + 1, 4)]) + + varied_row = _row(_write(tmp_path / "varied.hdf5", varied), "start_layout_spread") + pinned_row = _row(_write(tmp_path / "pinned.hdf5", pinned), "start_layout_spread") + + assert varied_row["value"] == pytest.approx(1000 * np.hypot(0.02, 0.1) * mean_step) + assert pinned_row["value"] == pytest.approx(1000 * 0.1 * mean_step) + + +def test_a_deformable_object_starts_at_its_nodes_centroid(tmp_path): + # A rope's first node stays at the origin while its second lies 0, 10 and 20 cm away: its centroid moves 5 cm a step. + episodes = [ + { + "actions": np.zeros((2, 7)), + "initial_state/deformable_object/rope/nodal_position": [[[0.0] * 3, [0.1 * i, 0, 0]]], + } + for i in range(3) + ] + + row = _row(_write(tmp_path / "d.hdf5", episodes), "start_layout_spread") + + assert row["value"] == pytest.approx((50.0 + 100.0 + 50.0) / 3) + + +_LINE = np.linspace([0.0, 0.0, 0.0], [0.1, 0.0, 0.0], 11) +_WITH_BALL, _WITHOUT_BALL = _starts(cube=[0.0, 0.0], ball=[0.1, 0.0]), _starts(cube=[0.1, 0.0]) +_TWO_HANDS, _ONE_HAND = _moving(left=_LINE, right=_LINE), _moving(left=_LINE) + + +@pytest.mark.parametrize( + "name, episodes, reason", + [ + ("start_layout_spread", [_WITH_BALL], "needs two episodes"), + ("start_layout_spread", [_WITH_BALL, _WITHOUT_BALL], "rigid_object/ball has no start pose in episode demo_1"), + ("start_layout_spread", [_WITHOUT_BALL, _WITH_BALL], "rigid_object/ball has no start pose in episode demo_0"), + ("workspace_spread", [_TWO_HANDS, _ONE_HAND], "episode demo_1 records no pose of right"), + ("workspace_spread", [_ONE_HAND, _TWO_HANDS], "episode demo_0 records no pose of right"), + ], +) +def test_a_spread_says_why_it_does_not_apply(tmp_path, name, episodes, reason): + # Two episodes that hold the same actions are read in the opposite order once reversed, so in one of the two orders + # a spread reads first the episode missing what the other records. + row = _row(_write(tmp_path / "d.hdf5", episodes), name) + + assert row["status"] == "not_applicable" and row["reason"] == reason + + +def test_repeated_episodes_spread_through_the_episodes_a_spread_reads(tmp_path): + # 40 distinct episodes, 30 copies each: the 1,000 read include every one. + episodes = [{"actions": np.full((2, 7), float(k))} for k in range(40) for _ in range(30)] + path = _write(tmp_path / "d.hdf5", episodes) + + with h5py.File(path, "r") as h5_file: + sample = _sample(Evidence(h5_file)) + distinct = {_digest(episode) for episode in sample} + + assert len(sample) == 1000 and len(distinct) == 40 + + +def test_the_workspace_refuses_a_hand_that_moves_more_than_8_cm_per_step(tmp_path): + # The right hand moves 1 mm a step but for 3 steps of 9 cm, 1.5% of its own; the left, which sorts first, stands + # still. + steps = np.array([0.001] * 197 + [0.09] * 3) + episodes = [ + _moving(right=np.stack([np.concatenate([[0.0], np.cumsum(steps)]), np.full(201, side), np.zeros(201)], axis=1)) + | _moving(left=np.zeros((201, 3))) + for side in (0.0, 0.1) + ] + + row = _row(_write(tmp_path / "d.hdf5", episodes), "workspace_spread") + + assert row["status"] == "not_applicable" and "right moves 9 cm per step" in row["reason"] + + +def test_workspace_counts_a_retraced_path_once(tmp_path): + # Against a 30 cm line, the line with a 10 cm branch and the line traced five times with the same branch visit the + # same space. + line = np.linspace([0.0, 0.0, 0.0], [0.3, 0.0, 0.0], 31) + branch = np.linspace([0.3, 0.0, 0.0], [0.3, 0.1, 0.0], 11)[1:] + once = np.concatenate([line, branch]) + retraced = np.concatenate([line, line[-2::-1], line[1:], line[-2::-1], line[1:], branch]) + + values = [ + _row(_write(tmp_path / f"{i}.hdf5", [_moving(arm=line), _moving(arm=other)]), "workspace_spread") + for i, other in enumerate((once, retraced)) + ] + + assert values[0]["value"] == pytest.approx(values[1]["value"], abs=0.01) + + +def test_the_workspace_is_how_far_apart_two_episodes_hands_run(tmp_path): + # In one episode both hands sweep 10 cm along x, recorded at 0, 5 and 10 cm; in the other, the left hand rests + # 10 cm beyond the sweep's end and the right 3 cm beside its middle. An offset of 1.25 mm keeps the sweep's 2.5 mm + # samples off the 5 mm cell edges, so it keeps a point every 5 mm. The left sweep's points lie 150 mm from the + # resting hand on average, which lies 100 mm from the nearest of them: 125 mm. The right sweep's points at x [mm] lie + # hypot(30, x - 52.5) mm from the resting hand, which lies hypot(30, 2.5) mm from the nearest of them. + offset = np.full(3, 0.00125) + sweep = offset + [[0.0, 0.0, 0.0], [0.05, 0.0, 0.0], [0.1, 0.0, 0.0]] + beyond = np.tile(offset + [0.2, 0.0, 0.0], (3, 1)) + beside = np.tile(offset + [0.0525, 0.03, 0.0], (3, 1)) + right = 0.5 * (np.mean([np.hypot(30.0, x - 52.5) for x in range(0, 101, 5)]) + np.hypot(30.0, 2.5)) + episodes = [_moving(left=sweep, right=sweep), _moving(left=beyond, right=beside)] + + row = _row(_write(tmp_path / "d.hdf5", episodes), "workspace_spread") + + assert row["value"] == pytest.approx(0.5 * (125.0 + right), rel=1e-9) + + +def test_spreads_do_not_depend_on_the_order_of_episodes(tmp_path): + # 64 episodes make 2,016 pairs, more than the workspace measures; which ones depends on content, not order. + starts = np.random.default_rng(0).uniform(-0.3, 0.3, (64, 3)) + episodes = [_moving(arm=np.linspace(start, start + [0.1, 0.0, 0.0], 6)) for start in starts] + + values = [ + _row(_write(tmp_path / f"{label}.hdf5", ordered), "workspace_spread")["value"] + for label, ordered in (("forward", episodes), ("reversed", episodes[::-1])) + ] + + assert values[0] == pytest.approx(values[1], rel=1e-12) diff --git a/autodata_tests/utils/utils.py b/autodata_tests/utils/utils.py index 42bd977..efc2419 100644 --- a/autodata_tests/utils/utils.py +++ b/autodata_tests/utils/utils.py @@ -94,7 +94,8 @@ def assert_recorded_run(dataset_file: str, result_file: str) -> list[dict]: def assert_ok_metrics(dataset_file: str, result_file: str, expected_ok: set[str]) -> None: """Report a generation run's metrics with ``validate_dataset.py``; assert every episode passes the checks and - exactly ``expected_ok`` computed a value. + exactly ``expected_ok`` computed a value, with ``start_layout_spread`` and ``workspace_spread`` too when the run + exported two episodes or more. A renamed recorded key or an adapter that stops recording a stream turns a metric not applicable, which this catches. @@ -118,7 +119,9 @@ def assert_ok_metrics(dataset_file: str, result_file: str, expected_ok: set[str] ), f"{report['num_episodes'] - report['num_valid_episodes']} episodes failed checks: {report['check_failures']}" rows = report["metrics"] ok = {name for name, row in rows.items() if row["status"] == "ok"} - assert ok == expected_ok, f"Computed {sorted(ok)}, expected {sorted(expected_ok)}: {rows}" + spreads = {"start_layout_spread", "workspace_spread"} if report["num_valid_episodes"] >= 2 else set() + expected = expected_ok | spreads + assert ok == expected, f"Computed {sorted(ok)}, expected {sorted(expected)}: {rows}" def assert_no_metric_inputs(dataset_file: str) -> None: diff --git a/docs/pages/references/metrics.rst b/docs/pages/references/metrics.rst index 761369a..236caf4 100644 --- a/docs/pages/references/metrics.rst +++ b/docs/pages/references/metrics.rst @@ -11,28 +11,36 @@ without it, most metrics report that their inputs are missing. python scripts/validate_dataset.py datasets/generated.hdf5 --report \ --result_file datasets/generated.json \ --source datasets/annotated_datasets/dataset_franka_annotated.hdf5 \ + --gate datasets/gate.yaml \ --report_file datasets/generated_report.json The command prints a summary; ``--report_file`` writes the full report, with every detail, as JSON. .. code-block:: text - AutoData metrics · gr1.hdf5 · run e79e37b1 · source dataset_gr1_annotated.hdf5 - 6/6 episodes valid · 7 computed · 3 not computed · gate failed on 2 of 4 + AutoData metrics · generated.hdf5 · run cc273b3f · source dataset_franka_annotated.hdf5 + 20/20 episodes valid · 11 computed · 5 not computed · gate failed on 1 of 4 - Value Source Based on + Value Source Based on VALIDITY - ✗ action_rotation_error p95 0.0345 3.58e-7 1,612 poses - ✓ correctness_pass_rate 100% 6 episodes - ✓ final_success_rate 100% 6 episodes - image_stream_pass_rate 100% 6 episodes + action_rotation_error p95 8.34e-7 4.77e-7 4,803 poses + action_saturation 4.06% 0% 4,803 steps + ✓ correctness_pass_rate 100% 20 episodes + final_success_rate 95% 20 episodes + joint_limit_ratio velocity 40% 10% 20 episodes + motion_smoothness p95 142 m/s³ 40 m/s³ 4,743 poses + DIVERSITY + ✓ start_layout_spread 217 mm 222 mm 20 episodes + ✓ workspace_spread 46.9 mm 44.7 mm 20 episodes VIABILITY - demos_per_gpu_hour 65.8 demos/GPU-h 6 demos - ✗ generation_success_rate 21.4% 28 attempts · 95% CI 10.2–39.5% - time_to_dataset 6 min 11 s 1 run + demos_per_gpu_hour 863 demos/GPU-h 20 demos + ✗ generation_success_rate 40% 50 attempts · 95% CI 27.6–53.8% + time_to_dataset 105 s 1 run NOT COMPUTED not reported by this algorithm: planning_failure_rate, planning_time + no camera streams: image_stream_pass_rate + no pose action is recorded as absolute: tracking_error needs a --source annotated with --record_metric_inputs: annotation_pass_rate - The second line is the verdict: valid episodes, how many metrics were computed, and, with ``--gate``, @@ -88,6 +96,40 @@ Built-in metrics :autodata_metrics_table: +Diversity metrics +----------------- + +The diversity metrics describe how different the demonstrations in a dataset are from one another. Each +averages a distance over pairs of episodes: + +- ``start_layout_spread`` [mm]: how far apart two episodes' starting scenes are. Each object's horizontal + start distance is combined with the others' as a root-sum-square, so an object that never moves adds nothing + and one that moves more adds more. Robots count by their root, deformable objects by their nodes' centroid. +- ``workspace_spread`` [mm]: how far one episode's hands pass from where the other's went. A hand is each + recorded end effector, followed by the origin of its frame as the embodiment records it (a whole-body + controller records its hands relative to the robot's body), from the episode's first recorded step. Its path, + sampled every 2.5 mm between recorded steps, keeps one point per 5 mm cell, so retracing a path adds nothing; + each kept point's distance to the nearest kept point of the other episode's path is averaged, both ways. + Beyond 63 episodes, it measures 2,000 pairs drawn with a fixed seed among the episodes it reads. + +An average over pairs estimates how different two demonstrations from the same producer are, so it does not +grow with the number of episodes; below about ten episodes it is a rough guide. The spreads read episodes in a +ranking by a digest of each episode's actions and how many earlier episodes repeat them, at most the first +1,000, so the order of episodes in the file does not matter unless two episodes repeat each other's actions +exactly. + +Comparing datasets +~~~~~~~~~~~~~~~~~~ + +Two spreads compare only on the same task and scene objects, and two workspaces only with the same hands; each +metric's details list its objects or hands. A hand's path follows its end effector's recorded frame, which its +embodiment defines; two embodiments that record different frames on one task still compare (the Franka stack's +MimicGen and SkillGen frames sit 10.34 cm apart along the gripper and move its workspace by about 3%). Values +assume each path starts at its episode's first recorded step, the reset pose. On the test datasets and +generated runs, leaving that step out moved them by up to 7%, and halving the recording rate moved the +workspace by under 1%. A hand that moves far between recorded steps still blurs its path, so the workspace is +not applicable when any hand moves more than 8 cm per recorded step (the 99th percentile of its own steps). + Adding a metric ---------------