From f17934f84f28832ca58626bdfff67b58ac7fc6fb Mon Sep 17 00:00:00 2001 From: Hualiang Xie Date: Mon, 20 Jul 2026 11:31:04 +0800 Subject: [PATCH 1/3] Add two-ONNX comparison to winml eval --mode compare Introduce a --reference option so --mode compare can diff two ONNX files directly instead of only comparing an ONNX candidate against its HF PyTorch reference. Both models run as raw ORT sessions on identical RandomDataset inputs, so --model-id and --task are not required and no torch/transformers reference is loaded. - config: add reference_path field (to_dict/from_dict) - cli: --reference option, _resolve_reference validation/normalization, allow bare ONNX candidate without --model-id, require --mode compare - evaluate: skip task resolution and return no model for the two-ONNX path; print reference in config summary - evaluator: _ONNXSessionModel wrapper; build both raw sessions when reference_path is set, reusing the existing similarity metrics - docs + unit tests --- docs/commands/eval.md | 9 +- src/winml/modelkit/commands/eval.py | 72 ++++++++++++- src/winml/modelkit/eval/config.py | 11 +- src/winml/modelkit/eval/evaluate.py | 21 +++- .../eval/tensor_similarity_evaluator.py | 58 +++++++++- tests/unit/commands/test_eval.py | 100 ++++++++++++++++-- tests/unit/eval/test_eval.py | 61 +++++++++++ .../eval/test_tensor_similarity_evaluator.py | 95 ++++++++++++++++- 8 files changed, 399 insertions(+), 28 deletions(-) diff --git a/docs/commands/eval.md b/docs/commands/eval.md index f31a5df28..39a74e1b2 100644 --- a/docs/commands/eval.md +++ b/docs/commands/eval.md @@ -35,7 +35,8 @@ $ winml eval [options] | `--label-mapping` | | `PATH` | — | Path to a JSON file mapping dataset label names to the integer class IDs the model emits: `{"label_name": id}`. | | `--output` | `-o` | `PATH` | — | Output JSON file path for the evaluation results. | | `--schema` | | flag | `false` | Print the expected dataset schema for the given `--task` and exit. Does not run evaluation. | -| `--mode` | | `onnx\|compare` | `onnx` | Evaluation mode. `onnx` evaluates the ONNX candidate on a dataset. `compare` runs the ONNX candidate and the HuggingFace reference on identical random inputs and reports per-tensor similarity metrics — no dataset required. | +| `--mode` | | `onnx\|compare` | `onnx` | Evaluation mode. `onnx` evaluates the ONNX candidate on a dataset. `compare` runs the ONNX candidate and a reference on identical random inputs and reports per-tensor similarity metrics — no dataset required. The reference is the HuggingFace model from `--model-id` by default, or a second ONNX file when `--reference` is given. | +| `--reference` | | `TEXT` | — | Reference `.onnx` file to compare the candidate against (used with `--mode compare`). Compares two ONNX models on identical random inputs; `--model-id` and `--task` are not required in this mode. Both models run on the same `--device` / `--ep`. | ## How it works @@ -71,6 +72,12 @@ Evaluate a BERT model on the MRPC paraphrase task with column remapping: $ winml eval -m Intel/bert-base-uncased-mrpc --dataset nyu-mll/glue --dataset-name mrpc --column input_column=sentence1 --column second_input_column=sentence2 --samples 500 ``` +Compare two ONNX files directly (e.g. an fp32 baseline vs a quantized build), reporting per-output tensor-similarity metrics on identical random inputs — no `--model-id` or dataset needed: + +```bash +$ winml eval --mode compare -m quantized.onnx --reference baseline.onnx +``` + Check what dataset columns are expected before running, then remap them to match your dataset: ```bash diff --git a/src/winml/modelkit/commands/eval.py b/src/winml/modelkit/commands/eval.py index f97f72294..7c97efc26 100644 --- a/src/winml/modelkit/commands/eval.py +++ b/src/winml/modelkit/commands/eval.py @@ -160,6 +160,17 @@ "random inputs and report tensor-similarity metrics per output tensor." ), ) +@click.option( + "--reference", + "reference", + type=str, + default=None, + help=( + "Reference ONNX file to compare the candidate against (use with " + "--mode compare). Compares two ONNX models on identical random inputs; " + "--model-id / --task are not required in this mode." + ), +) @cli_utils.skip_build_option() @cli_utils.format_option() @cli_utils.build_config_option() @@ -199,6 +210,7 @@ def eval( mode: EvalMode, config_file: Path | None, skip_build: bool, + reference: str | None, ) -> None: r"""Evaluate a model for a task. @@ -207,6 +219,8 @@ def eval( winml eval -m model.onnx --model-id microsoft/resnet-50 + winml eval --mode compare -m candidate.onnx --reference baseline.onnx + Run `winml eval --schema --task ` to see the dataset columns and options expected by each task. """ @@ -240,7 +254,10 @@ def eval( cfg = _build_eval_config(ctx, config_file, column, label_mapping_path) # ── 2. Resolve in place ── - _resolve_model(cfg, model, model_id) + if cfg.reference_path is not None and cfg.mode != "compare": + raise click.UsageError("--reference is only valid with --mode compare.") + _resolve_model(cfg, model, model_id, allow_missing_model_id=cfg.reference_path is not None) + _resolve_reference(cfg) _resolve_device(cfg) _resolve_label_mapping(cfg) _run_dataset_script(cfg, trust_remote_code) @@ -361,13 +378,54 @@ def _resolve_model( cfg: WinMLEvaluationConfig, model: tuple[str, ...], model_id: str | None, + *, + allow_missing_model_id: bool = False, ) -> None: """Resolve ``-m`` / ``--model-id`` into ``cfg.model_path`` / ``cfg.model_id``.""" - model_path, resolved_id = _resolve_model_path(model=model, model_id=model_id) + model_path, resolved_id = _resolve_model_path( + model=model, model_id=model_id, allow_missing_model_id=allow_missing_model_id + ) cfg.model_path = model_path cfg.model_id = resolved_id +def _resolve_reference(cfg: WinMLEvaluationConfig) -> None: + """Validate and normalize ``cfg.reference_path`` for two-ONNX compare. + + Requires the candidate (``-m``) to be a single ONNX file (composite + ``role=path`` candidates and build-from-id are not supported with + ``--reference`` yet). Resolves Hub-hosted ONNX refs to local paths. + """ + if cfg.reference_path is None: + return + + if not isinstance(cfg.model_path, str): + raise click.UsageError( + "--reference requires the candidate (-m) to be a single ONNX file. " + "Composite (role=path) candidates and build-from-id are not " + "supported with --reference." + ) + + ref = cfg.reference_path + if Path(ref).suffix.lower() != ".onnx": + raise click.BadParameter( + f"--reference must be an .onnx file, got: {ref}", + param_hint="--reference", + ) + try: + ref = cli_utils.normalize_model_arg(ref) or ref + except Exception as e: + raise click.ClickException( + f"Failed to resolve Hub-hosted reference ONNX path {ref!r}: {e}" + ) from e + if not Path(ref).exists(): + raise click.BadParameter( + f"Reference ONNX file not found: {ref}", + param_hint="--reference", + ) + cfg.reference_path = ref + + def _resolve_device(cfg: WinMLEvaluationConfig) -> None: """Resolve ``'auto'`` → concrete device string on *cfg* in place.""" if cfg.device and cfg.device.lower() != "auto": @@ -451,8 +509,14 @@ def _resolve_model_path( *, model: tuple[str, ...], model_id: str | None, + allow_missing_model_id: bool = False, ) -> tuple[str | dict[str, str] | None, str | None]: - """Turn repeated -m values + --model-id into (model_path, model_id).""" + """Turn repeated -m values + --model-id into (model_path, model_id). + + When ``allow_missing_model_id`` is set (two-ONNX ``--mode compare``), a + plain ``-m .onnx`` is accepted without ``--model-id`` because the + candidate runs as a raw ORT session with no HF config resolution. + """ if not model: if model_id is not None: return None, model_id @@ -528,6 +592,8 @@ def _resolve_model_path( param_hint="-m/--model", ) if model_id is None: + if allow_missing_model_id: + return value, None raise click.UsageError( "When using an ONNX file, --model-id is required " "for preprocessor and config resolution." diff --git a/src/winml/modelkit/eval/config.py b/src/winml/modelkit/eval/config.py index 923c0474d..b523c6c5d 100644 --- a/src/winml/modelkit/eval/config.py +++ b/src/winml/modelkit/eval/config.py @@ -88,6 +88,10 @@ class WinMLEvaluationConfig: model_path: Path to .onnx model file, or a ``{role: path}`` dict for composite models (e.g. ``{"image-encoder": "...", "text-encoder": "..."}``). None = build from model_id. + reference_path: Path to a second ``.onnx`` file used as the reference in + ``--mode compare``. When set, both ``model_path`` and ``reference_path`` + run as raw ORT sessions and their output tensors are compared directly, + so no ``model_id`` / ``task`` / HF reference is needed. task: HF pipeline task. Auto-detected from model_id if omitted. device: Target device for inference. ep: Explicit execution provider (e.g., "qnn", "dml"). Overrides @@ -100,7 +104,8 @@ class WinMLEvaluationConfig: labeled dataset. - ``"compare"``: compare ONNX vs HF reference output tensors on identical random inputs and report tensor-similarity - metrics per output tensor. + metrics per output tensor. When ``reference_path`` is set, + the reference is a second ONNX file instead of the HF model. Usage: config = WinMLEvaluationConfig( @@ -111,6 +116,7 @@ class WinMLEvaluationConfig: model_id: str | None = None model_path: str | dict[str, str] | None = None + reference_path: str | None = field(default=None, metadata={"cli_name": "reference"}) task: str | None = None device: str = "auto" precision: str = "auto" @@ -134,6 +140,8 @@ def to_dict(self) -> dict: result["model_id"] = self.model_id if self.model_path is not None: result["model_path"] = self.model_path + if self.reference_path is not None: + result["reference_path"] = self.reference_path if self.task is not None: result["task"] = self.task result["device"] = self.device @@ -181,6 +189,7 @@ def from_dict(cls, data: dict) -> WinMLEvaluationConfig: return cls( model_id=data.get("model_id"), model_path=data.get("model_path"), + reference_path=data.get("reference_path"), task=data.get("task"), device=data.get("device", "auto"), precision=data.get("precision", "auto"), diff --git a/src/winml/modelkit/eval/evaluate.py b/src/winml/modelkit/eval/evaluate.py index 74863970f..47a4d231f 100644 --- a/src/winml/modelkit/eval/evaluate.py +++ b/src/winml/modelkit/eval/evaluate.py @@ -231,6 +231,11 @@ def _load_model(config: WinMLEvaluationConfig) -> WinMLPreTrainedModel | WinMLCo from ..models import WinMLAutoModel from ..utils import cli as cli_utils + # Two-ONNX compare: the evaluator builds both raw ORT sessions directly from + # config.model_path / config.reference_path — no WinMLAutoModel / HF config. + if config.mode == "compare" and config.reference_path is not None: + return None + quant_override: Any = None if not config.quant: from ..config import WinMLBuildConfig @@ -327,8 +332,14 @@ def evaluate(config: WinMLEvaluationConfig) -> EvalResult: mode = config.mode if config.mode is not None else "onnx" if mode not in EVAL_MODES: raise ValueError(f"Invalid mode {mode!r}; expected one of {EVAL_MODES} or None.") + # Two-ONNX compare: both candidate and reference run as raw ORT sessions, so + # HF task resolution / model_id are not required — keep task as-is. + onnx_compare = mode == "compare" and config.reference_path is not None config = replace( - config, mode=mode, task=_resolve_task(config), dataset=deepcopy(config.dataset) + config, + mode=mode, + task=config.task if onnx_compare else _resolve_task(config), + dataset=deepcopy(config.dataset), ) if config.mode != "compare" and config.dataset.path is None: default = _DEFAULT_DATASETS.get(config.task) if config.task is not None else None @@ -390,10 +401,14 @@ def print_config(config: WinMLEvaluationConfig) -> None: """Print effective evaluation config to the console (quantize.py style).""" ds = config.dataset output_console = Console() - output_console.print(f"[bold blue]Model:[/bold blue] {config.model_id}") + if config.model_id is not None: + output_console.print(f"[bold blue]Model:[/bold blue] {config.model_id}") if config.model_path is not None: output_console.print(f"[bold blue]Model path:[/bold blue] {config.model_path}") - output_console.print(f"[bold blue]Task:[/bold blue] {config.task}") + if config.reference_path is not None: + output_console.print(f"[bold blue]Reference:[/bold blue] {config.reference_path}") + if config.task is not None: + output_console.print(f"[bold blue]Task:[/bold blue] {config.task}") output_console.print(f"[bold blue]Device:[/bold blue] {config.device}") if config.ep is not None: output_console.print(f"[bold blue]EP:[/bold blue] {config.ep}") diff --git a/src/winml/modelkit/eval/tensor_similarity_evaluator.py b/src/winml/modelkit/eval/tensor_similarity_evaluator.py index 1b3ee7d84..1aa2af56e 100644 --- a/src/winml/modelkit/eval/tensor_similarity_evaluator.py +++ b/src/winml/modelkit/eval/tensor_similarity_evaluator.py @@ -5,10 +5,14 @@ """Tensor-similarity evaluator. -Runs an ONNX candidate and an HF PyTorch reference on identical random -inputs (drawn from :class:`RandomDataset` over the candidate's ONNX I/O) -and reports per-output tensor-parity metrics (SQNR, PSNR, cosine, MSE, -max absolute diff) via :class:`TensorSimilarityMetric`. +Runs an ONNX candidate and a reference on identical random inputs (drawn +from :class:`RandomDataset` over the candidate's ONNX I/O) and reports +per-output tensor-parity metrics (SQNR, PSNR, cosine, MSE, max absolute +diff) via :class:`TensorSimilarityMetric`. + +The reference is an HF PyTorch model resolved from ``model_id`` by default. +When ``config.reference_path`` is set, the reference is instead a second +ONNX file and both sides run as raw ORT sessions (no HF config / task). No labeled dataset, no HF pipeline, no preprocessor — any divergence reflects the build pipeline (optimize / quantize / compile) only. @@ -29,6 +33,37 @@ logger = logging.getLogger(__name__) +class _ONNXSessionModel: + """Minimal raw-ORT wrapper for two-ONNX ``--mode compare``. + + Exposes just the slice of the :class:`WinMLPreTrainedModel` surface that + the tensor-similarity loop needs — ``onnx_path``, ``io_config`` and a + callable returning named ``torch`` tensors — without any HF config or + task-specific output renaming, so both sides compare on their raw ONNX + output names. + """ + + def __init__(self, onnx_path: str, device: str = "auto", ep: Any | None = None) -> None: + from pathlib import Path + + from ..session.session import WinMLSession + + self.onnx_path = Path(onnx_path) + self._session = WinMLSession(onnx_path=self.onnx_path, device=device, ep=ep) + + @property + def io_config(self) -> dict: + """ONNX I/O metadata (delegated to the session).""" + return self._session.io_config + + def __call__(self, **inputs: Any) -> dict[str, Any]: + """Run one sample and return raw outputs as named ``torch`` tensors.""" + import torch + + outputs = self._session.run(inputs) + return {name: torch.from_numpy(arr) for name, arr in outputs.items()} + + class TensorSimilarityEvaluator: """Per-output tensor parity between an ONNX candidate and an HF reference.""" @@ -39,6 +74,20 @@ def __init__( ) -> None: from ..models.winml.composite_model import WinMLCompositeModel + self.config = config + + # Two-ONNX compare: build both raw ORT sessions directly, bypassing the + # HF PyTorch reference. ``model`` is None here (see evaluate._load_model). + if config.reference_path is not None: + self.model = _ONNXSessionModel( + str(config.model_path), device=config.device, ep=config.ep + ) + self.reference_model = _ONNXSessionModel( + str(config.reference_path), device=config.device, ep=config.ep + ) + self.data = self.prepare_data() + return + # Composite models must be split into their sub-components before # tensor-similarity comparison — the union param keeps this runtime # guard live for type checkers. @@ -50,7 +99,6 @@ def __init__( "Example: winml eval --mode compare --task " f"--model --model-id {config.model_id}" ) - self.config = config self.model = model self.reference_model = self._load_reference_model() self.data = self.prepare_data() diff --git a/tests/unit/commands/test_eval.py b/tests/unit/commands/test_eval.py index a73b9f520..8aadadc3e 100644 --- a/tests/unit/commands/test_eval.py +++ b/tests/unit/commands/test_eval.py @@ -14,7 +14,8 @@ import pytest from click.testing import CliRunner -from winml.modelkit.commands.eval import _resolve_model_path +from winml.modelkit.commands.eval import _resolve_model_path, _resolve_reference +from winml.modelkit.eval import WinMLEvaluationConfig # --------------------------------------------------------------------------- @@ -101,6 +102,14 @@ def test_plain_onnx_without_model_id_raises(self, onnx_file): with pytest.raises(click.UsageError, match="--model-id is required"): _resolve_model_path(model=(str(onnx_file),), model_id=None) + def test_plain_onnx_without_model_id_allowed_for_compare(self, onnx_file): + """allow_missing_model_id (two-ONNX compare) accepts a bare ONNX path.""" + path, mid = _resolve_model_path( + model=(str(onnx_file),), model_id=None, allow_missing_model_id=True + ) + assert path == str(onnx_file) + assert mid is None + def test_plain_onnx_missing_file_raises(self, tmp_path): missing = tmp_path / "does-not-exist.onnx" with pytest.raises(click.BadParameter, match="ONNX file not found"): @@ -234,13 +243,8 @@ def test_composite_hub_refs_resolved(self, tmp_path): enc_local.write_bytes(b"") dec_local = tmp_path / "prompt_encoder_mask_decoder_int8.onnx" dec_local.write_bytes(b"") - enc_ref = ( - "onnx-community/sam3-tracker-ONNX/onnx/vision_encoder_int8.onnx" - ) - dec_ref = ( - "onnx-community/sam3-tracker-ONNX/onnx/" - "prompt_encoder_mask_decoder_int8.onnx" - ) + enc_ref = "onnx-community/sam3-tracker-ONNX/onnx/vision_encoder_int8.onnx" + dec_ref = "onnx-community/sam3-tracker-ONNX/onnx/prompt_encoder_mask_decoder_int8.onnx" # Map each Hub ref to its (different) local cache location. def fake_resolve(ref, **kwargs): @@ -271,9 +275,7 @@ def test_composite_mixed_hub_and_local(self, onnx_vision, tmp_path): """One role is a Hub ref, the other is a local path -- both work.""" dec_local = tmp_path / "decoder.onnx" dec_local.write_bytes(b"") - enc_ref = ( - "onnx-community/sam3-tracker-ONNX/onnx/vision_encoder_int8.onnx" - ) + enc_ref = "onnx-community/sam3-tracker-ONNX/onnx/vision_encoder_int8.onnx" # ``resolve_hf_onnx_path`` is the underlying downloader; the # unified classifier+resolver only calls it for hub_onnx inputs. @@ -334,6 +336,82 @@ def test_model_help_mentions_onnx_model_id_and_role_path(self, runner: CliRunner assert "requires --model-id" in result.output assert "role=path" in result.output + def test_help_mentions_reference(self, runner: CliRunner): + from winml.modelkit.commands.eval import eval as eval_cmd + + result = runner.invoke(eval_cmd, ["--help"]) + + assert result.exit_code == 0, result.output + assert "--reference" in result.output + + +class TestResolveReference: + def test_none_is_noop(self): + cfg = WinMLEvaluationConfig(model_path="m.onnx", mode="compare") + _resolve_reference(cfg) + assert cfg.reference_path is None + + def test_happy_path(self, onnx_file, onnx_vision): + cfg = WinMLEvaluationConfig( + model_path=str(onnx_file), + reference_path=str(onnx_vision), + mode="compare", + ) + _resolve_reference(cfg) + assert cfg.reference_path == str(onnx_vision) + + def test_requires_onnx_candidate(self): + cfg = WinMLEvaluationConfig( + model_path=None, + reference_path="ref.onnx", + mode="compare", + ) + with pytest.raises(click.UsageError, match="single ONNX file"): + _resolve_reference(cfg) + + def test_composite_candidate_rejected(self, onnx_vision): + cfg = WinMLEvaluationConfig( + model_path={"encoder": "a.onnx"}, + reference_path=str(onnx_vision), + mode="compare", + ) + with pytest.raises(click.UsageError, match="single ONNX file"): + _resolve_reference(cfg) + + def test_non_onnx_suffix_raises(self, onnx_file, tmp_path): + bad = tmp_path / "ref.txt" + bad.write_bytes(b"") + cfg = WinMLEvaluationConfig( + model_path=str(onnx_file), + reference_path=str(bad), + mode="compare", + ) + with pytest.raises(click.BadParameter, match=r"must be an \.onnx file"): + _resolve_reference(cfg) + + def test_missing_file_raises(self, onnx_file, tmp_path): + missing = tmp_path / "missing.onnx" + cfg = WinMLEvaluationConfig( + model_path=str(onnx_file), + reference_path=str(missing), + mode="compare", + ) + with pytest.raises(click.BadParameter, match="not found"): + _resolve_reference(cfg) + + +class TestReferenceModeGuard: + def test_reference_requires_compare_mode(self, runner: CliRunner, onnx_file): + from winml.modelkit.commands.eval import eval as eval_cmd + + result = runner.invoke( + eval_cmd, + ["-m", str(onnx_file), "--reference", str(onnx_file)], + obj={"debug": False}, + ) + assert result.exit_code != 0 + assert "--reference is only valid with --mode compare" in result.output + @pytest.fixture def eval_config_file(tmp_path): diff --git a/tests/unit/eval/test_eval.py b/tests/unit/eval/test_eval.py index 1fdbf024e..b40cfa313 100644 --- a/tests/unit/eval/test_eval.py +++ b/tests/unit/eval/test_eval.py @@ -55,6 +55,23 @@ def test_dataset_config_revision_default_is_none(self): assert ds.revision is None assert "revision" not in ds.to_dict() + def test_reference_path_default_is_none(self): + """reference_path defaults to None and is omitted from to_dict.""" + config = WinMLEvaluationConfig(model_id="test/model") + assert config.reference_path is None + assert "reference_path" not in config.to_dict() + + def test_config_roundtrip_preserves_reference_path(self): + """reference_path survives to_dict/from_dict roundtrip.""" + config = WinMLEvaluationConfig( + model_path="cand.onnx", + reference_path="ref.onnx", + mode="compare", + ) + restored = WinMLEvaluationConfig.from_dict(config.to_dict()) + assert restored.reference_path == "ref.onnx" + assert restored.mode == "compare" + def test_eval_result_to_dict(self): config = WinMLEvaluationConfig( model_id="test/model", @@ -221,6 +238,50 @@ def test_none_mode_normalizes_to_onnx(self): result = eval_mod.evaluate(config) assert result.config.mode == "onnx" + def test_onnx_compare_skips_task_resolution_and_dataset(self): + """Two-ONNX compare skips HF task resolution and default-dataset lookup.""" + import importlib + import sys + + eval_mod = sys.modules.get( + "winml.modelkit.eval.evaluate", + ) or importlib.import_module("winml.modelkit.eval.evaluate") + + config = WinMLEvaluationConfig( + model_path="cand.onnx", + reference_path="ref.onnx", + mode="compare", + ) + + evaluator = MagicMock() + evaluator.compute.return_value = {"cosine_mean": {"logits": 1.0}} + with ( + patch.object( + eval_mod, + "_resolve_task", + side_effect=AssertionError("task resolution must be skipped"), + ), + patch.object(eval_mod, "_load_model", return_value=None) as load_model, + patch.object(eval_mod, "get_evaluator_class", return_value=lambda *_a, **_k: evaluator), + ): + result = eval_mod.evaluate(config) + + assert result.config.mode == "compare" + assert result.config.task is None + assert result.metrics == {"cosine_mean": {"logits": 1.0}} + load_model.assert_called_once() + + def test_load_model_returns_none_for_onnx_compare(self): + """_load_model short-circuits (no model_id needed) for two-ONNX compare.""" + from winml.modelkit.eval.evaluate import _load_model + + config = WinMLEvaluationConfig( + model_path="cand.onnx", + reference_path="ref.onnx", + mode="compare", + ) + assert _load_model(config) is None + def test_no_dataset_no_default_raises(self): """Tasks without a default dataset raise ValueError.""" import importlib diff --git a/tests/unit/eval/test_tensor_similarity_evaluator.py b/tests/unit/eval/test_tensor_similarity_evaluator.py index 61bffb962..7903bc534 100644 --- a/tests/unit/eval/test_tensor_similarity_evaluator.py +++ b/tests/unit/eval/test_tensor_similarity_evaluator.py @@ -22,7 +22,10 @@ from transformers.modeling_outputs import BaseModelOutput from winml.modelkit.eval import DatasetConfig, WinMLEvaluationConfig -from winml.modelkit.eval.tensor_similarity_evaluator import TensorSimilarityEvaluator +from winml.modelkit.eval.tensor_similarity_evaluator import ( + TensorSimilarityEvaluator, + _ONNXSessionModel, +) from winml.modelkit.models.winml.composite_model import WinMLCompositeModel @@ -30,6 +33,7 @@ # _inference_model # --------------------------------------------------------------------------- + class _EchoModel: """Minimal stand-in that returns a BaseModelOutput from the inputs.""" @@ -77,6 +81,7 @@ def test_returns_numpy_dict_only_for_tensor_fields(self): # composite-model guard in __init__ # --------------------------------------------------------------------------- + class _FakeCompositeModel(WinMLCompositeModel): _SUB_MODEL_CONFIG: ClassVar[dict[str, str]] = { "encoder": "image-feature-extraction", @@ -86,9 +91,7 @@ class _FakeCompositeModel(WinMLCompositeModel): class TestCompositeGuard: def test_rejects_composite_with_helpful_message(self): - composite = _FakeCompositeModel( - sub_models={}, config=PretrainedConfig() - ) + composite = _FakeCompositeModel(sub_models={}, config=PretrainedConfig()) config = WinMLEvaluationConfig( model_id="Salesforce/blip-image-captioning-base", task="image-to-text", @@ -102,3 +105,87 @@ def test_rejects_composite_with_helpful_message(self): assert "image-feature-extraction" in msg assert "text-generation" in msg assert "Salesforce/blip-image-captioning-base" in msg + + +# --------------------------------------------------------------------------- +# Two-ONNX compare (reference_path set) +# --------------------------------------------------------------------------- + + +class _FakeSession: + """Stand-in for WinMLSession that records construction and echoes outputs.""" + + created: ClassVar[list[tuple[str, str, object]]] = [] + + def __init__(self, onnx_path, device="auto", ep=None): + _FakeSession.created.append((str(onnx_path), device, ep)) + self.io_config = {"input_names": ["input"]} + + def run(self, inputs): + return {"logits": np.arange(3.0, dtype=np.float32).reshape(1, 3)} + + +class _FakeRandomDataset: + def __init__(self, **kwargs): + self.kwargs = kwargs + + def __len__(self): + return 0 + + +class TestONNXReferenceInit: + def test_builds_two_raw_sessions_honoring_device(self, monkeypatch): + import winml.modelkit.datasets.random_dataset as rd_mod + import winml.modelkit.session.session as session_mod + + _FakeSession.created = [] + monkeypatch.setattr(session_mod, "WinMLSession", _FakeSession) + monkeypatch.setattr(rd_mod, "RandomDataset", _FakeRandomDataset) + + config = WinMLEvaluationConfig( + model_path="cand.onnx", + reference_path="ref.onnx", + mode="compare", + device="cpu", + ep="dml", + dataset=DatasetConfig(samples=5, seed=1), + ) + + # ``model`` is None in this path (evaluate._load_model returns None). + evaluator = TensorSimilarityEvaluator(config, None) # type: ignore[arg-type] + + assert isinstance(evaluator.model, _ONNXSessionModel) + assert isinstance(evaluator.reference_model, _ONNXSessionModel) + # Candidate first, reference second; both honor --device / --ep. + assert _FakeSession.created[0][0].endswith("cand.onnx") + assert _FakeSession.created[1][0].endswith("ref.onnx") + assert [c[1] for c in _FakeSession.created] == ["cpu", "cpu"] + assert [c[2] for c in _FakeSession.created] == ["dml", "dml"] + # RandomDataset is built over the candidate ONNX I/O. + assert evaluator.data.kwargs["model_path"].endswith("cand.onnx") + assert evaluator.data.kwargs["max_samples"] == 5 + assert evaluator.data.kwargs["seed"] == 1 + + +class TestONNXSessionModel: + def test_call_returns_named_torch_tensors(self, monkeypatch): + import winml.modelkit.session.session as session_mod + + _FakeSession.created = [] + monkeypatch.setattr(session_mod, "WinMLSession", _FakeSession) + + model = _ONNXSessionModel("x.onnx", device="cpu") + out = model(input=torch.zeros(1, 3)) + + assert set(out) == {"logits"} + assert isinstance(out["logits"], torch.Tensor) + assert out["logits"].shape == (1, 3) + + def test_io_config_delegates_to_session(self, monkeypatch): + import winml.modelkit.session.session as session_mod + + _FakeSession.created = [] + monkeypatch.setattr(session_mod, "WinMLSession", _FakeSession) + + model = _ONNXSessionModel("x.onnx") + assert model.io_config["input_names"] == ["input"] From 579cce84483fb59d2399eb0dcb21e7ea0e9a3095 Mon Sep 17 00:00:00 2001 From: Hualiang Xie Date: Mon, 20 Jul 2026 14:23:40 +0800 Subject: [PATCH 2/3] Add --input-data to winml eval --mode compare Support comparing a candidate and reference on real input tensors from a .npz archive instead of randomly generated ones, mirroring winml perf's --input-data. The whole archive is treated as a single sample and its keys are validated/cast against the candidate's inputs. Only valid with --mode compare (accuracy modes need labels). - datasets/input_data.py: shared load_input_data (moved from perf) plus a single-sample InputDataDataset wrapper - perf: load_input_data now delegates to the shared module (lazy import, same public symbol) - config: add input_data field (to_dict/from_dict) - cli: --input-data option, compare-mode guard, example; report header shows Reference / Input data / Samples: 1 - evaluator: prepare_data builds InputDataDataset when input_data is set - evaluate: print the input-data path in the config summary - docs + unit tests --- docs/commands/eval.md | 9 +- src/winml/modelkit/commands/eval.py | 31 +++- src/winml/modelkit/commands/perf.py | 84 +--------- src/winml/modelkit/datasets/input_data.py | 153 ++++++++++++++++++ src/winml/modelkit/eval/config.py | 10 ++ src/winml/modelkit/eval/evaluate.py | 2 + .../eval/tensor_similarity_evaluator.py | 23 ++- tests/unit/commands/test_eval.py | 26 +++ tests/unit/commands/test_perf_cli.py | 2 +- tests/unit/datasets/test_input_data.py | 95 +++++++++++ tests/unit/eval/test_eval.py | 17 ++ .../eval/test_tensor_similarity_evaluator.py | 36 ++++- 12 files changed, 400 insertions(+), 88 deletions(-) create mode 100644 src/winml/modelkit/datasets/input_data.py create mode 100644 tests/unit/datasets/test_input_data.py diff --git a/docs/commands/eval.md b/docs/commands/eval.md index 39a74e1b2..17724bb46 100644 --- a/docs/commands/eval.md +++ b/docs/commands/eval.md @@ -35,8 +35,9 @@ $ winml eval [options] | `--label-mapping` | | `PATH` | — | Path to a JSON file mapping dataset label names to the integer class IDs the model emits: `{"label_name": id}`. | | `--output` | `-o` | `PATH` | — | Output JSON file path for the evaluation results. | | `--schema` | | flag | `false` | Print the expected dataset schema for the given `--task` and exit. Does not run evaluation. | -| `--mode` | | `onnx\|compare` | `onnx` | Evaluation mode. `onnx` evaluates the ONNX candidate on a dataset. `compare` runs the ONNX candidate and a reference on identical random inputs and reports per-tensor similarity metrics — no dataset required. The reference is the HuggingFace model from `--model-id` by default, or a second ONNX file when `--reference` is given. | +| `--mode` | | `onnx\|compare` | `onnx` | Evaluation mode. `onnx` evaluates the ONNX candidate on a dataset. `compare` runs the ONNX candidate and a reference on identical inputs and reports per-tensor similarity metrics — no dataset required. The reference is the HuggingFace model from `--model-id` by default, or a second ONNX file when `--reference` is given. | | `--reference` | | `TEXT` | — | Reference `.onnx` file to compare the candidate against (used with `--mode compare`). Compares two ONNX models on identical random inputs; `--model-id` and `--task` are not required in this mode. Both models run on the same `--device` / `--ep`. | +| `--input-data` | | `PATH` | — | Path to a `.npz` file of real input tensors to compare with instead of randomly generated ones (used with `--mode compare`). Keys must match the candidate model's input names; the whole archive is treated as a single sample, so `--samples` / `--seed` are ignored. | ## How it works @@ -78,6 +79,12 @@ Compare two ONNX files directly (e.g. an fp32 baseline vs a quantized build), re $ winml eval --mode compare -m quantized.onnx --reference baseline.onnx ``` +Compare on real input tensors instead of random ones by passing a `.npz` archive whose keys match the candidate's input names (the whole archive is one sample): + +```bash +$ winml eval --mode compare -m quantized.onnx --reference baseline.onnx --input-data inputs.npz +``` + Check what dataset columns are expected before running, then remap them to match your dataset: ```bash diff --git a/src/winml/modelkit/commands/eval.py b/src/winml/modelkit/commands/eval.py index 7c97efc26..73a8de1c6 100644 --- a/src/winml/modelkit/commands/eval.py +++ b/src/winml/modelkit/commands/eval.py @@ -171,6 +171,18 @@ "--model-id / --task are not required in this mode." ), ) +@click.option( + "--input-data", + "input_data", + type=click.Path(exists=True, dir_okay=False), + default=None, + help=( + "Path to a .npz file of real input tensors to compare with instead of " + "randomly generated ones (use with --mode compare). Keys must match the " + "candidate model's input names; the whole archive is one sample, so " + "--samples / --seed are ignored." + ), +) @cli_utils.skip_build_option() @cli_utils.format_option() @cli_utils.build_config_option() @@ -211,6 +223,7 @@ def eval( config_file: Path | None, skip_build: bool, reference: str | None, + input_data: str | None, ) -> None: r"""Evaluate a model for a task. @@ -221,6 +234,8 @@ def eval( winml eval --mode compare -m candidate.onnx --reference baseline.onnx + winml eval --mode compare -m cand.onnx --reference base.onnx --input-data inputs.npz + Run `winml eval --schema --task ` to see the dataset columns and options expected by each task. """ @@ -256,6 +271,8 @@ def eval( # ── 2. Resolve in place ── if cfg.reference_path is not None and cfg.mode != "compare": raise click.UsageError("--reference is only valid with --mode compare.") + if cfg.input_data is not None and cfg.mode != "compare": + raise click.UsageError("--input-data is only valid with --mode compare.") _resolve_model(cfg, model, model_id, allow_missing_model_id=cfg.reference_path is not None) _resolve_reference(cfg) _resolve_device(cfg) @@ -642,9 +659,17 @@ def display_eval_report(result: EvalResult, console: Console) -> None: console.print() console.print(f"[dim]Task:[/dim] {cfg.task}") console.print(f"[dim]Device:[/dim] {cfg.device}") - if ds.path: - console.print(f"[dim]Dataset:[/dim] {ds.path}") - console.print(f"[dim]Samples:[/dim] {ds.samples}") + if cfg.reference_path: + console.print(f"[dim]Reference:[/dim] {cfg.reference_path}") + if cfg.input_data: + # A .npz archive is a single sample (one batch); the dataset sample + # count does not apply on this path. + console.print(f"[dim]Input data:[/dim] {cfg.input_data}") + console.print("[dim]Samples:[/dim] 1") + else: + if ds.path: + console.print(f"[dim]Dataset:[/dim] {ds.path}") + console.print(f"[dim]Samples:[/dim] {ds.samples}") if cfg.model_path: console.print(f"[dim]ONNX:[/dim] {cfg.model_path}") diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 30d533699..006115d0e 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -381,85 +381,15 @@ def load_input_data( ) -> dict[str, np.ndarray]: """Load benchmark inputs from a ``.npz`` file, validated against the model. - Lets ``winml perf`` profile with real input tensors instead of randomly - generated ones. Only ``.npz`` (a named-array archive) is supported today; - a single-array ``.npy`` carries no input names to bind against and is - rejected with guidance to repackage as ``.npz``. - - Validation: - - * the archive's keys must exactly match the model's input names -- any - missing or unexpected key is an error (an unexpected key is usually a - typo that would otherwise leave a required input silently unset); - * an array whose dtype differs from the model's expected input dtype is - cast to the expected dtype with a warning, matching the silent casting - ``WinMLSession._prepare_inputs`` does on a normal run (e.g. numpy's - default int64 literals binding to an int32 input). - - Shapes are taken from the arrays as-is; correctness beyond dtype (e.g. a - static dimension the data violates) surfaces as a runtime error from the - inference session. - - Args: - path: Path to the ``.npz`` file. - io_config: Model I/O configuration (``input_names``, ``input_types``). - - Returns: - Dictionary of ``input_name -> numpy array``. - - Raises: - click.UsageError: On a non-``.npz`` file or a key mismatch. + Thin wrapper over the shared + :func:`winml.modelkit.datasets.input_data.load_input_data`, which is also + used by ``winml eval --mode compare --input-data``. Imported lazily so + ``winml perf`` startup does not pull in the datasets package unless + ``--input-data`` is actually used. """ - if path.suffix.lower() == ".npy": - raise click.UsageError( - f"--input-data does not support .npy files ({path.name}). A single " - f"array carries no input names; save your inputs as a named .npz " - f"archive instead (e.g. np.savez('inputs.npz', input_ids=..., " - f"attention_mask=...))." - ) - if path.suffix.lower() != ".npz": - raise click.UsageError( - f"--input-data must be a .npz file, got '{path.suffix or path.name}'." - ) - - try: - with np.load(path, allow_pickle=False) as archive: - provided = {name: archive[name] for name in archive.files} - except Exception as exc: - raise click.UsageError(f"Could not read --input-data file {path}: {exc}") from exc - - expected_names = list(io_config["input_names"]) - expected_types = list(io_config["input_types"]) - - missing = [name for name in expected_names if name not in provided] - unexpected = [name for name in provided if name not in expected_names] - if missing or unexpected: - parts = [] - if missing: - parts.append(f"missing {missing}") - if unexpected: - parts.append(f"unexpected {unexpected}") - raise click.UsageError( - f"--input-data keys do not match the model inputs ({', '.join(parts)}). " - f"Expected exactly: {expected_names}." - ) - - # Cast dtype mismatches instead of failing, mirroring the session's - # _prepare_inputs, so inputs that would run fine on a normal invocation - # (e.g. int64 literals against an int32 input) don't hard-error here. - for name, expected_dtype in zip(expected_names, expected_types, strict=True): - want = np.dtype(expected_dtype) - got = provided[name].dtype - if got != want: - logger.warning( - "--input-data dtype for '%s' is %s; casting to the model's expected %s.", - name, - got, - want, - ) - provided[name] = provided[name].astype(want) + from ..datasets.input_data import load_input_data as _load_input_data - return provided + return _load_input_data(path, io_config) def effective_batch_size( diff --git a/src/winml/modelkit/datasets/input_data.py b/src/winml/modelkit/datasets/input_data.py new file mode 100644 index 000000000..ccaf368e4 --- /dev/null +++ b/src/winml/modelkit/datasets/input_data.py @@ -0,0 +1,153 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- + +"""Real input tensors loaded from a ``.npz`` archive. + +Shared by ``winml perf`` (benchmark on real tensors instead of random ones) +and ``winml eval --mode compare`` (compare a candidate and reference on the +same real inputs). :func:`load_input_data` validates and dtype-casts the +archive against a model's I/O config; :class:`InputDataDataset` wraps the +loaded archive as a single-sample torch dataset the compare loop can iterate. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import click +import numpy as np + + +if TYPE_CHECKING: + import torch + +logger = logging.getLogger(__name__) + + +def load_input_data( + path: Path, + io_config: dict[str, Any], +) -> dict[str, np.ndarray]: + """Load model inputs from a ``.npz`` file, validated against the model. + + Lets ``winml perf`` / ``winml eval`` run with real input tensors instead + of randomly generated ones. Only ``.npz`` (a named-array archive) is + supported today; a single-array ``.npy`` carries no input names to bind + against and is rejected with guidance to repackage as ``.npz``. + + Validation: + + * the archive's keys must exactly match the model's input names -- any + missing or unexpected key is an error (an unexpected key is usually a + typo that would otherwise leave a required input silently unset); + * an array whose dtype differs from the model's expected input dtype is + cast to the expected dtype with a warning, matching the silent casting + ``WinMLSession._prepare_inputs`` does on a normal run (e.g. numpy's + default int64 literals binding to an int32 input). + + Shapes are taken from the arrays as-is; correctness beyond dtype (e.g. a + static dimension the data violates) surfaces as a runtime error from the + inference session. + + Args: + path: Path to the ``.npz`` file. + io_config: Model I/O configuration (``input_names``, ``input_types``). + + Returns: + Dictionary of ``input_name -> numpy array``. + + Raises: + click.UsageError: On a non-``.npz`` file or a key mismatch. + """ + path = Path(path) + if path.suffix.lower() == ".npy": + raise click.UsageError( + f"--input-data does not support .npy files ({path.name}). A single " + f"array carries no input names; save your inputs as a named .npz " + f"archive instead (e.g. np.savez('inputs.npz', input_ids=..., " + f"attention_mask=...))." + ) + if path.suffix.lower() != ".npz": + raise click.UsageError( + f"--input-data must be a .npz file, got '{path.suffix or path.name}'." + ) + + try: + with np.load(path, allow_pickle=False) as archive: + provided = {name: archive[name] for name in archive.files} + except Exception as exc: + raise click.UsageError(f"Could not read --input-data file {path}: {exc}") from exc + + expected_names = list(io_config["input_names"]) + expected_types = list(io_config["input_types"]) + + missing = [name for name in expected_names if name not in provided] + unexpected = [name for name in provided if name not in expected_names] + if missing or unexpected: + parts = [] + if missing: + parts.append(f"missing {missing}") + if unexpected: + parts.append(f"unexpected {unexpected}") + raise click.UsageError( + f"--input-data keys do not match the model inputs ({', '.join(parts)}). " + f"Expected exactly: {expected_names}." + ) + + # Cast dtype mismatches instead of failing, mirroring the session's + # _prepare_inputs, so inputs that would run fine on a normal invocation + # (e.g. int64 literals against an int32 input) don't hard-error here. + for name, expected_dtype in zip(expected_names, expected_types, strict=True): + want = np.dtype(expected_dtype) + got = provided[name].dtype + if got != want: + logger.warning( + "--input-data dtype for '%s' is %s; casting to the model's expected %s.", + name, + got, + want, + ) + provided[name] = provided[name].astype(want) + + return provided + + +class InputDataDataset: + """Single-sample dataset backed by a validated ``.npz`` of real tensors. + + Loads the archive once via :func:`load_input_data` (keys and dtypes + validated/cast against ``io_config``) and exposes it as a one-sample + torch dataset, so ``--mode compare`` can run the candidate and reference + on identical real inputs. The whole archive is treated as a single batch + (one sample), mirroring how ``winml perf`` consumes ``--input-data``. + + Args: + path: Path to the ``.npz`` file of real input tensors. + io_config: Candidate model I/O config (``input_names``, ``input_types``). + """ + + TASK_TYPE = "input_data" + + def __init__(self, path: str | Path, io_config: dict[str, Any]) -> None: + import torch + + arrays = load_input_data(Path(path), io_config) + # np.load arrays are owned/writable; ascontiguousarray avoids the + # non-contiguous from_numpy warning without an extra copy when possible. + self._sample: dict[str, torch.Tensor] = { + name: torch.from_numpy(np.ascontiguousarray(arr)) for name, arr in arrays.items() + } + + def __len__(self) -> int: + """A ``.npz`` archive is a single sample (one batch).""" + return 1 + + def __getitem__(self, idx: int) -> dict[str, torch.Tensor]: + """Return the one sample; any index other than 0 is out of range.""" + if idx != 0: + raise IndexError(f"InputDataDataset has a single sample; got index {idx}.") + return self._sample diff --git a/src/winml/modelkit/eval/config.py b/src/winml/modelkit/eval/config.py index b523c6c5d..e5b476989 100644 --- a/src/winml/modelkit/eval/config.py +++ b/src/winml/modelkit/eval/config.py @@ -92,6 +92,10 @@ class WinMLEvaluationConfig: ``--mode compare``. When set, both ``model_path`` and ``reference_path`` run as raw ORT sessions and their output tensors are compared directly, so no ``model_id`` / ``task`` / HF reference is needed. + input_data: Path to a ``.npz`` archive of real input tensors for + ``--mode compare``. When set, the candidate and reference are compared + on these tensors (validated against the candidate's inputs) instead of + randomly generated ones. The whole archive is treated as a single sample. task: HF pipeline task. Auto-detected from model_id if omitted. device: Target device for inference. ep: Explicit execution provider (e.g., "qnn", "dml"). Overrides @@ -106,6 +110,8 @@ class WinMLEvaluationConfig: on identical random inputs and report tensor-similarity metrics per output tensor. When ``reference_path`` is set, the reference is a second ONNX file instead of the HF model. + When ``input_data`` is set, real tensors from a ``.npz`` are + used instead of random inputs. Usage: config = WinMLEvaluationConfig( @@ -117,6 +123,7 @@ class WinMLEvaluationConfig: model_id: str | None = None model_path: str | dict[str, str] | None = None reference_path: str | None = field(default=None, metadata={"cli_name": "reference"}) + input_data: str | None = None task: str | None = None device: str = "auto" precision: str = "auto" @@ -142,6 +149,8 @@ def to_dict(self) -> dict: result["model_path"] = self.model_path if self.reference_path is not None: result["reference_path"] = self.reference_path + if self.input_data is not None: + result["input_data"] = self.input_data if self.task is not None: result["task"] = self.task result["device"] = self.device @@ -190,6 +199,7 @@ def from_dict(cls, data: dict) -> WinMLEvaluationConfig: model_id=data.get("model_id"), model_path=data.get("model_path"), reference_path=data.get("reference_path"), + input_data=data.get("input_data"), task=data.get("task"), device=data.get("device", "auto"), precision=data.get("precision", "auto"), diff --git a/src/winml/modelkit/eval/evaluate.py b/src/winml/modelkit/eval/evaluate.py index 47a4d231f..c5f2827e7 100644 --- a/src/winml/modelkit/eval/evaluate.py +++ b/src/winml/modelkit/eval/evaluate.py @@ -407,6 +407,8 @@ def print_config(config: WinMLEvaluationConfig) -> None: output_console.print(f"[bold blue]Model path:[/bold blue] {config.model_path}") if config.reference_path is not None: output_console.print(f"[bold blue]Reference:[/bold blue] {config.reference_path}") + if config.input_data is not None: + output_console.print(f"[bold blue]Input data:[/bold blue] {config.input_data}") if config.task is not None: output_console.print(f"[bold blue]Task:[/bold blue] {config.task}") output_console.print(f"[bold blue]Device:[/bold blue] {config.device}") diff --git a/src/winml/modelkit/eval/tensor_similarity_evaluator.py b/src/winml/modelkit/eval/tensor_similarity_evaluator.py index 1aa2af56e..52018f253 100644 --- a/src/winml/modelkit/eval/tensor_similarity_evaluator.py +++ b/src/winml/modelkit/eval/tensor_similarity_evaluator.py @@ -5,14 +5,16 @@ """Tensor-similarity evaluator. -Runs an ONNX candidate and a reference on identical random inputs (drawn -from :class:`RandomDataset` over the candidate's ONNX I/O) and reports -per-output tensor-parity metrics (SQNR, PSNR, cosine, MSE, max absolute -diff) via :class:`TensorSimilarityMetric`. +Runs an ONNX candidate and a reference on identical inputs (random by +default, drawn from :class:`RandomDataset` over the candidate's ONNX I/O) +and reports per-output tensor-parity metrics (SQNR, PSNR, cosine, MSE, max +absolute diff) via :class:`TensorSimilarityMetric`. The reference is an HF PyTorch model resolved from ``model_id`` by default. When ``config.reference_path`` is set, the reference is instead a second ONNX file and both sides run as raw ORT sessions (no HF config / task). +When ``config.input_data`` is set, both sides run on real tensors from a +``.npz`` archive instead of random inputs. No labeled dataset, no HF pipeline, no preprocessor — any divergence reflects the build pipeline (optimize / quantize / compile) only. @@ -127,7 +129,18 @@ def _load_reference_model(self) -> Any: ).eval() def prepare_data(self) -> Any: - """Build a RandomDataset over the candidate ONNX's I/O spec.""" + """Build the compare dataset over the candidate ONNX's I/O spec. + + Uses real tensors from ``config.input_data`` (wrapped as a + single-sample :class:`InputDataDataset`, validated against the + candidate's inputs) when provided, otherwise a :class:`RandomDataset` + of synthetic inputs sized by ``config.dataset``. + """ + if self.config.input_data is not None: + from ..datasets.input_data import InputDataDataset + + return InputDataDataset(self.config.input_data, self.model.io_config) + from ..datasets.random_dataset import RandomDataset ds = self.config.dataset diff --git a/tests/unit/commands/test_eval.py b/tests/unit/commands/test_eval.py index 8aadadc3e..dbfb6e585 100644 --- a/tests/unit/commands/test_eval.py +++ b/tests/unit/commands/test_eval.py @@ -344,6 +344,14 @@ def test_help_mentions_reference(self, runner: CliRunner): assert result.exit_code == 0, result.output assert "--reference" in result.output + def test_help_mentions_input_data(self, runner: CliRunner): + from winml.modelkit.commands.eval import eval as eval_cmd + + result = runner.invoke(eval_cmd, ["--help"]) + + assert result.exit_code == 0, result.output + assert "--input-data" in result.output + class TestResolveReference: def test_none_is_noop(self): @@ -413,6 +421,24 @@ def test_reference_requires_compare_mode(self, runner: CliRunner, onnx_file): assert "--reference is only valid with --mode compare" in result.output +class TestInputDataModeGuard: + def test_input_data_requires_compare_mode(self, runner: CliRunner, onnx_file, tmp_path): + import numpy as np + + from winml.modelkit.commands.eval import eval as eval_cmd + + npz = tmp_path / "inputs.npz" + np.savez(npz, x=np.zeros((1, 4), dtype=np.float32)) + + result = runner.invoke( + eval_cmd, + ["-m", str(onnx_file), "--input-data", str(npz)], + obj={"debug": False}, + ) + assert result.exit_code != 0 + assert "--input-data is only valid with --mode compare" in result.output + + @pytest.fixture def eval_config_file(tmp_path): config = { diff --git a/tests/unit/commands/test_perf_cli.py b/tests/unit/commands/test_perf_cli.py index be4984977..65826ef6e 100644 --- a/tests/unit/commands/test_perf_cli.py +++ b/tests/unit/commands/test_perf_cli.py @@ -1052,7 +1052,7 @@ def test_dtype_cast_with_warning(self, tmp_path, caplog) -> None: } path = self._write_npz(tmp_path, input_ids=np.zeros((1, 8), dtype=np.int64)) - with caplog.at_level(logging.WARNING, logger="winml.modelkit.commands.perf"): + with caplog.at_level(logging.WARNING, logger="winml.modelkit.datasets.input_data"): inputs = load_input_data(path, io) assert inputs["input_ids"].dtype == np.int32 diff --git a/tests/unit/datasets/test_input_data.py b/tests/unit/datasets/test_input_data.py new file mode 100644 index 000000000..b944de07c --- /dev/null +++ b/tests/unit/datasets/test_input_data.py @@ -0,0 +1,95 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- + +"""Unit tests for winml.modelkit.datasets.input_data. + +Covers the shared ``.npz`` loader (also exercised via ``winml perf``) and the +single-sample :class:`InputDataDataset` used by ``winml eval --mode compare``. +""" + +from __future__ import annotations + +import logging +from typing import ClassVar + +import click +import numpy as np +import pytest +import torch + +from winml.modelkit.datasets.input_data import InputDataDataset, load_input_data + + +class TestLoadInputData: + _IO: ClassVar[dict] = { + "input_names": ["pixel_values"], + "input_shapes": [[None, 3, 8, 8]], + "input_types": ["float32"], + } + + def _write_npz(self, tmp_path, **arrays): + path = tmp_path / "inputs.npz" + np.savez(path, **arrays) + return path + + def test_loads_matching_npz(self, tmp_path) -> None: + path = self._write_npz(tmp_path, pixel_values=np.zeros((2, 3, 8, 8), dtype=np.float32)) + inputs = load_input_data(path, self._IO) + assert list(inputs) == ["pixel_values"] + assert inputs["pixel_values"].shape == (2, 3, 8, 8) + + def test_key_mismatch_errors(self, tmp_path) -> None: + path = self._write_npz(tmp_path, wrong=np.zeros((1, 3, 8, 8), dtype=np.float32)) + with pytest.raises(click.UsageError, match="do not match"): + load_input_data(path, self._IO) + + def test_dtype_cast_with_warning(self, tmp_path, caplog) -> None: + io = {"input_names": ["input_ids"], "input_shapes": [[None, 8]], "input_types": ["int32"]} + path = self._write_npz(tmp_path, input_ids=np.zeros((1, 8), dtype=np.int64)) + with caplog.at_level(logging.WARNING, logger="winml.modelkit.datasets.input_data"): + inputs = load_input_data(path, io) + assert inputs["input_ids"].dtype == np.int32 + assert "casting" in caplog.text.lower() + + def test_npy_rejected(self, tmp_path) -> None: + path = tmp_path / "inputs.npy" + np.save(path, np.zeros((1, 3, 8, 8), dtype=np.float32)) + with pytest.raises(click.UsageError, match=r"does not support \.npy"): + load_input_data(path, self._IO) + + +class TestInputDataDataset: + _IO: ClassVar[dict] = { + "input_names": ["x"], + "input_shapes": [[None, 4]], + "input_types": ["float32"], + } + + def _write_npz(self, tmp_path, **arrays): + path = tmp_path / "inputs.npz" + np.savez(path, **arrays) + return path + + def test_single_sample_as_torch_tensors(self, tmp_path) -> None: + path = self._write_npz(tmp_path, x=np.ones((2, 4), dtype=np.float32)) + ds = InputDataDataset(path, self._IO) + + assert len(ds) == 1 + sample = ds[0] + assert set(sample) == {"x"} + assert isinstance(sample["x"], torch.Tensor) + assert sample["x"].shape == (2, 4) + assert sample["x"].dtype == torch.float32 + + def test_index_out_of_range(self, tmp_path) -> None: + path = self._write_npz(tmp_path, x=np.ones((1, 4), dtype=np.float32)) + ds = InputDataDataset(path, self._IO) + with pytest.raises(IndexError): + _ = ds[1] + + def test_validates_keys_against_io_config(self, tmp_path) -> None: + path = self._write_npz(tmp_path, wrong=np.ones((1, 4), dtype=np.float32)) + with pytest.raises(click.UsageError, match="do not match"): + InputDataDataset(path, self._IO) diff --git a/tests/unit/eval/test_eval.py b/tests/unit/eval/test_eval.py index b40cfa313..c5338369d 100644 --- a/tests/unit/eval/test_eval.py +++ b/tests/unit/eval/test_eval.py @@ -72,6 +72,23 @@ def test_config_roundtrip_preserves_reference_path(self): assert restored.reference_path == "ref.onnx" assert restored.mode == "compare" + def test_input_data_default_is_none(self): + """input_data defaults to None and is omitted from to_dict.""" + config = WinMLEvaluationConfig(model_id="test/model") + assert config.input_data is None + assert "input_data" not in config.to_dict() + + def test_config_roundtrip_preserves_input_data(self): + """input_data survives to_dict/from_dict roundtrip.""" + config = WinMLEvaluationConfig( + model_path="cand.onnx", + reference_path="ref.onnx", + input_data="inputs.npz", + mode="compare", + ) + restored = WinMLEvaluationConfig.from_dict(config.to_dict()) + assert restored.input_data == "inputs.npz" + def test_eval_result_to_dict(self): config = WinMLEvaluationConfig( model_id="test/model", diff --git a/tests/unit/eval/test_tensor_similarity_evaluator.py b/tests/unit/eval/test_tensor_similarity_evaluator.py index 7903bc534..2018b0aa4 100644 --- a/tests/unit/eval/test_tensor_similarity_evaluator.py +++ b/tests/unit/eval/test_tensor_similarity_evaluator.py @@ -119,7 +119,7 @@ class _FakeSession: def __init__(self, onnx_path, device="auto", ep=None): _FakeSession.created.append((str(onnx_path), device, ep)) - self.io_config = {"input_names": ["input"]} + self.io_config = {"input_names": ["input"], "input_types": ["float32"]} def run(self, inputs): return {"logits": np.arange(3.0, dtype=np.float32).reshape(1, 3)} @@ -189,3 +189,37 @@ def test_io_config_delegates_to_session(self, monkeypatch): model = _ONNXSessionModel("x.onnx") assert model.io_config["input_names"] == ["input"] + + +# --------------------------------------------------------------------------- +# Real-input compare (input_data set) +# --------------------------------------------------------------------------- + + +class TestInputDataCompare: + def test_prepare_data_uses_input_data_npz(self, monkeypatch, tmp_path): + import winml.modelkit.session.session as session_mod + from winml.modelkit.datasets.input_data import InputDataDataset + + _FakeSession.created = [] + monkeypatch.setattr(session_mod, "WinMLSession", _FakeSession) + + npz = tmp_path / "inputs.npz" + np.savez(npz, input=np.ones((2, 3), dtype=np.float32)) + + config = WinMLEvaluationConfig( + model_path="cand.onnx", + reference_path="ref.onnx", + mode="compare", + input_data=str(npz), + ) + + # ``model`` is None in this path (evaluate._load_model returns None). + evaluator = TensorSimilarityEvaluator(config, None) # type: ignore[arg-type] + + assert isinstance(evaluator.data, InputDataDataset) + assert len(evaluator.data) == 1 + sample = evaluator.data[0] + assert set(sample) == {"input"} + assert isinstance(sample["input"], torch.Tensor) + assert sample["input"].shape == (2, 3) From fe807aa947e9cb0abdf870e513ea97794b2f9b49 Mon Sep 17 00:00:00 2001 From: Hualiang Xie Date: Mon, 20 Jul 2026 15:38:52 +0800 Subject: [PATCH 3/3] Support multiple samples in eval compare --input-data Treat the leading axis of each .npz array as the sample axis so a single archive can hold N samples. All inputs must share the same leading length N; scalars and empty arrays are rejected with a clear error. The compare loop runs each sample independently (sliced to a batch of 1), so the similarity metrics table reflects a real distribution (mean/std/min/max) across N inputs. The effective sample count is propagated to the report header and JSON output. --- docs/commands/eval.md | 4 +- src/winml/modelkit/commands/eval.py | 14 ++--- src/winml/modelkit/datasets/input_data.py | 60 +++++++++++++++---- src/winml/modelkit/eval/config.py | 4 +- .../eval/tensor_similarity_evaluator.py | 15 +++-- tests/unit/datasets/test_input_data.py | 56 +++++++++++++---- .../eval/test_tensor_similarity_evaluator.py | 7 ++- 7 files changed, 118 insertions(+), 42 deletions(-) diff --git a/docs/commands/eval.md b/docs/commands/eval.md index 17724bb46..3f899fe47 100644 --- a/docs/commands/eval.md +++ b/docs/commands/eval.md @@ -37,7 +37,7 @@ $ winml eval [options] | `--schema` | | flag | `false` | Print the expected dataset schema for the given `--task` and exit. Does not run evaluation. | | `--mode` | | `onnx\|compare` | `onnx` | Evaluation mode. `onnx` evaluates the ONNX candidate on a dataset. `compare` runs the ONNX candidate and a reference on identical inputs and reports per-tensor similarity metrics — no dataset required. The reference is the HuggingFace model from `--model-id` by default, or a second ONNX file when `--reference` is given. | | `--reference` | | `TEXT` | — | Reference `.onnx` file to compare the candidate against (used with `--mode compare`). Compares two ONNX models on identical random inputs; `--model-id` and `--task` are not required in this mode. Both models run on the same `--device` / `--ep`. | -| `--input-data` | | `PATH` | — | Path to a `.npz` file of real input tensors to compare with instead of randomly generated ones (used with `--mode compare`). Keys must match the candidate model's input names; the whole archive is treated as a single sample, so `--samples` / `--seed` are ignored. | +| `--input-data` | | `PATH` | — | Path to a `.npz` file of real input tensors to compare with instead of randomly generated ones (used with `--mode compare`). Keys must match the candidate model's input names. The **leading axis of each array is the sample axis**, so an archive whose arrays have shape `(N, ...)` yields `N` samples (mean/std/min/max are computed across them); all inputs must share the same `N`. | ## How it works @@ -79,7 +79,7 @@ Compare two ONNX files directly (e.g. an fp32 baseline vs a quantized build), re $ winml eval --mode compare -m quantized.onnx --reference baseline.onnx ``` -Compare on real input tensors instead of random ones by passing a `.npz` archive whose keys match the candidate's input names (the whole archive is one sample): +Compare on real input tensors instead of random ones by passing a `.npz` archive whose keys match the candidate's input names. The leading axis of each array is the sample axis, so an archive shaped `(N, ...)` runs `N` samples: ```bash $ winml eval --mode compare -m quantized.onnx --reference baseline.onnx --input-data inputs.npz diff --git a/src/winml/modelkit/commands/eval.py b/src/winml/modelkit/commands/eval.py index 73a8de1c6..5416da1c3 100644 --- a/src/winml/modelkit/commands/eval.py +++ b/src/winml/modelkit/commands/eval.py @@ -179,8 +179,8 @@ help=( "Path to a .npz file of real input tensors to compare with instead of " "randomly generated ones (use with --mode compare). Keys must match the " - "candidate model's input names; the whole archive is one sample, so " - "--samples / --seed are ignored." + "candidate model's input names; the leading axis of each array is the " + "sample axis (N samples), and all inputs must share the same N." ), ) @cli_utils.skip_build_option() @@ -662,14 +662,10 @@ def display_eval_report(result: EvalResult, console: Console) -> None: if cfg.reference_path: console.print(f"[dim]Reference:[/dim] {cfg.reference_path}") if cfg.input_data: - # A .npz archive is a single sample (one batch); the dataset sample - # count does not apply on this path. console.print(f"[dim]Input data:[/dim] {cfg.input_data}") - console.print("[dim]Samples:[/dim] 1") - else: - if ds.path: - console.print(f"[dim]Dataset:[/dim] {ds.path}") - console.print(f"[dim]Samples:[/dim] {ds.samples}") + elif ds.path: + console.print(f"[dim]Dataset:[/dim] {ds.path}") + console.print(f"[dim]Samples:[/dim] {ds.samples}") if cfg.model_path: console.print(f"[dim]ONNX:[/dim] {cfg.model_path}") diff --git a/src/winml/modelkit/datasets/input_data.py b/src/winml/modelkit/datasets/input_data.py index ccaf368e4..193ab7abb 100644 --- a/src/winml/modelkit/datasets/input_data.py +++ b/src/winml/modelkit/datasets/input_data.py @@ -9,7 +9,8 @@ and ``winml eval --mode compare`` (compare a candidate and reference on the same real inputs). :func:`load_input_data` validates and dtype-casts the archive against a model's I/O config; :class:`InputDataDataset` wraps the -loaded archive as a single-sample torch dataset the compare loop can iterate. +loaded archive as a torch dataset (leading axis = sample axis) the compare +loop can iterate. """ from __future__ import annotations @@ -117,13 +118,20 @@ def load_input_data( class InputDataDataset: - """Single-sample dataset backed by a validated ``.npz`` of real tensors. + """Multi-sample dataset backed by a validated ``.npz`` of real tensors. Loads the archive once via :func:`load_input_data` (keys and dtypes - validated/cast against ``io_config``) and exposes it as a one-sample - torch dataset, so ``--mode compare`` can run the candidate and reference - on identical real inputs. The whole archive is treated as a single batch - (one sample), mirroring how ``winml perf`` consumes ``--input-data``. + validated/cast against ``io_config``), then treats the **leading axis of + each array as the sample axis**: an archive whose arrays have shape + ``(N, ...)`` yields ``N`` samples, so ``--mode compare`` can run the + candidate and reference on many real inputs and report a real + distribution (mean/std/min/max) instead of a single point. + + Every input must share the same leading length ``N`` (a clear error is + raised otherwise). Each sample keeps a leading batch dim of 1 + (``arr[i:i+1]``), so any dynamic-batch model accepts it and no assumption + is made about output layout — each run is compared independently, exactly + like :class:`RandomDataset`'s per-sample flow. Args: path: Path to the ``.npz`` file of real input tensors. @@ -136,18 +144,44 @@ def __init__(self, path: str | Path, io_config: dict[str, Any]) -> None: import torch arrays = load_input_data(Path(path), io_config) + + # Leading axis = sample axis. Reject scalars (no sample axis) and any + # disagreement on N so a silent mis-pairing can't produce bogus metrics. + leading: dict[str, int] = {} + for name, arr in arrays.items(): + if arr.ndim == 0: + raise click.UsageError( + f"--input-data array '{name}' is a scalar (0-d); the leading " + "axis is the sample axis, so each input needs at least one dim." + ) + leading[name] = int(arr.shape[0]) + + distinct = set(leading.values()) + if len(distinct) != 1: + detail = ", ".join(f"{name}={leading[name]}" for name in arrays) + raise click.UsageError( + "--input-data arrays must share the same leading (sample) axis " + f"length; got {detail}." + ) + + self._num_samples = distinct.pop() + if self._num_samples == 0: + raise click.UsageError("--input-data arrays are empty (sample axis length 0).") + # np.load arrays are owned/writable; ascontiguousarray avoids the # non-contiguous from_numpy warning without an extra copy when possible. - self._sample: dict[str, torch.Tensor] = { + self._arrays: dict[str, torch.Tensor] = { name: torch.from_numpy(np.ascontiguousarray(arr)) for name, arr in arrays.items() } def __len__(self) -> int: - """A ``.npz`` archive is a single sample (one batch).""" - return 1 + """Number of samples (the shared leading-axis length of the inputs).""" + return self._num_samples def __getitem__(self, idx: int) -> dict[str, torch.Tensor]: - """Return the one sample; any index other than 0 is out of range.""" - if idx != 0: - raise IndexError(f"InputDataDataset has a single sample; got index {idx}.") - return self._sample + """Return sample ``idx`` with each input sliced to a batch of 1.""" + if not 0 <= idx < self._num_samples: + raise IndexError( + f"InputDataDataset index {idx} out of range for {self._num_samples} samples." + ) + return {name: tensor[idx : idx + 1] for name, tensor in self._arrays.items()} diff --git a/src/winml/modelkit/eval/config.py b/src/winml/modelkit/eval/config.py index e5b476989..62ee105ea 100644 --- a/src/winml/modelkit/eval/config.py +++ b/src/winml/modelkit/eval/config.py @@ -95,7 +95,9 @@ class WinMLEvaluationConfig: input_data: Path to a ``.npz`` archive of real input tensors for ``--mode compare``. When set, the candidate and reference are compared on these tensors (validated against the candidate's inputs) instead of - randomly generated ones. The whole archive is treated as a single sample. + randomly generated ones. The leading axis of each array is the sample + axis, so one archive can hold ``N`` samples; all inputs must share the + same leading length. task: HF pipeline task. Auto-detected from model_id if omitted. device: Target device for inference. ep: Explicit execution provider (e.g., "qnn", "dml"). Overrides diff --git a/src/winml/modelkit/eval/tensor_similarity_evaluator.py b/src/winml/modelkit/eval/tensor_similarity_evaluator.py index 52018f253..c8894efc2 100644 --- a/src/winml/modelkit/eval/tensor_similarity_evaluator.py +++ b/src/winml/modelkit/eval/tensor_similarity_evaluator.py @@ -131,15 +131,20 @@ def _load_reference_model(self) -> Any: def prepare_data(self) -> Any: """Build the compare dataset over the candidate ONNX's I/O spec. - Uses real tensors from ``config.input_data`` (wrapped as a - single-sample :class:`InputDataDataset`, validated against the - candidate's inputs) when provided, otherwise a :class:`RandomDataset` - of synthetic inputs sized by ``config.dataset``. + Uses real tensors from ``config.input_data`` (wrapped as a multi-sample + :class:`InputDataDataset` whose leading axis is the sample axis and + validated against the candidate's inputs) when provided, otherwise a + :class:`RandomDataset` of synthetic inputs sized by ``config.dataset``. """ if self.config.input_data is not None: from ..datasets.input_data import InputDataDataset - return InputDataDataset(self.config.input_data, self.model.io_config) + dataset = InputDataDataset(self.config.input_data, self.model.io_config) + # Reflect the real sample count (leading axis of the .npz) in the + # effective config so the report header / JSON show N, not the + # unused dataset default. + self.config.dataset.samples = len(dataset) + return dataset from ..datasets.random_dataset import RandomDataset diff --git a/tests/unit/datasets/test_input_data.py b/tests/unit/datasets/test_input_data.py index b944de07c..5f0b89918 100644 --- a/tests/unit/datasets/test_input_data.py +++ b/tests/unit/datasets/test_input_data.py @@ -6,7 +6,7 @@ """Unit tests for winml.modelkit.datasets.input_data. Covers the shared ``.npz`` loader (also exercised via ``winml perf``) and the -single-sample :class:`InputDataDataset` used by ``winml eval --mode compare``. +multi-sample :class:`InputDataDataset` used by ``winml eval --mode compare``. """ from __future__ import annotations @@ -67,27 +67,63 @@ class TestInputDataDataset: "input_types": ["float32"], } + _IO2: ClassVar[dict] = { + "input_names": ["x", "y"], + "input_shapes": [[None, 4], [None, 2]], + "input_types": ["float32", "float32"], + } + def _write_npz(self, tmp_path, **arrays): path = tmp_path / "inputs.npz" np.savez(path, **arrays) return path - def test_single_sample_as_torch_tensors(self, tmp_path) -> None: - path = self._write_npz(tmp_path, x=np.ones((2, 4), dtype=np.float32)) + def test_leading_axis_is_sample_axis(self, tmp_path) -> None: + # (3, 4) -> 3 samples, each sliced to a batch of 1: (1, 4). + path = self._write_npz(tmp_path, x=np.arange(12, dtype=np.float32).reshape(3, 4)) ds = InputDataDataset(path, self._IO) + assert len(ds) == 3 + for i in range(3): + sample = ds[i] + assert set(sample) == {"x"} + assert isinstance(sample["x"], torch.Tensor) + assert sample["x"].shape == (1, 4) + # Rows are the original rows, in order. + assert ds[0]["x"].tolist() == [[0.0, 1.0, 2.0, 3.0]] + assert ds[2]["x"].tolist() == [[8.0, 9.0, 10.0, 11.0]] + + def test_single_row_is_one_sample(self, tmp_path) -> None: + path = self._write_npz(tmp_path, x=np.ones((1, 4), dtype=np.float32)) + ds = InputDataDataset(path, self._IO) assert len(ds) == 1 - sample = ds[0] - assert set(sample) == {"x"} - assert isinstance(sample["x"], torch.Tensor) - assert sample["x"].shape == (2, 4) - assert sample["x"].dtype == torch.float32 + assert ds[0]["x"].shape == (1, 4) + + def test_multiple_inputs_share_leading_dim(self, tmp_path) -> None: + path = self._write_npz( + tmp_path, + x=np.zeros((2, 4), dtype=np.float32), + y=np.ones((2, 2), dtype=np.float32), + ) + ds = InputDataDataset(path, self._IO2) + assert len(ds) == 2 + assert ds[1]["x"].shape == (1, 4) + assert ds[1]["y"].shape == (1, 2) + + def test_mismatched_leading_dims_error(self, tmp_path) -> None: + path = self._write_npz( + tmp_path, + x=np.zeros((3, 4), dtype=np.float32), + y=np.ones((2, 2), dtype=np.float32), + ) + with pytest.raises(click.UsageError, match="same leading"): + InputDataDataset(path, self._IO2) def test_index_out_of_range(self, tmp_path) -> None: - path = self._write_npz(tmp_path, x=np.ones((1, 4), dtype=np.float32)) + path = self._write_npz(tmp_path, x=np.ones((2, 4), dtype=np.float32)) ds = InputDataDataset(path, self._IO) with pytest.raises(IndexError): - _ = ds[1] + _ = ds[2] def test_validates_keys_against_io_config(self, tmp_path) -> None: path = self._write_npz(tmp_path, wrong=np.ones((1, 4), dtype=np.float32)) diff --git a/tests/unit/eval/test_tensor_similarity_evaluator.py b/tests/unit/eval/test_tensor_similarity_evaluator.py index 2018b0aa4..134967725 100644 --- a/tests/unit/eval/test_tensor_similarity_evaluator.py +++ b/tests/unit/eval/test_tensor_similarity_evaluator.py @@ -218,8 +218,11 @@ def test_prepare_data_uses_input_data_npz(self, monkeypatch, tmp_path): evaluator = TensorSimilarityEvaluator(config, None) # type: ignore[arg-type] assert isinstance(evaluator.data, InputDataDataset) - assert len(evaluator.data) == 1 + # Leading axis is the sample axis: (2, 3) -> 2 samples of shape (1, 3). + assert len(evaluator.data) == 2 sample = evaluator.data[0] assert set(sample) == {"input"} assert isinstance(sample["input"], torch.Tensor) - assert sample["input"].shape == (2, 3) + assert sample["input"].shape == (1, 3) + # The effective config reflects the real sample count for the report/JSON. + assert evaluator.config.dataset.samples == 2