diff --git a/docs/commands/eval.md b/docs/commands/eval.md index f31a5df28..3f899fe47 100644 --- a/docs/commands/eval.md +++ b/docs/commands/eval.md @@ -35,7 +35,9 @@ $ winml eval [options] | `--label-mapping` | | `PATH` | — | Path to a JSON file mapping dataset label names to the integer class IDs the model emits: `{"label_name": id}`. | | `--output` | `-o` | `PATH` | — | Output JSON file path for the evaluation results. | | `--schema` | | flag | `false` | Print the expected dataset schema for the given `--task` and exit. Does not run evaluation. | -| `--mode` | | `onnx\|compare` | `onnx` | Evaluation mode. `onnx` evaluates the ONNX candidate on a dataset. `compare` runs the ONNX candidate and the HuggingFace reference on identical random inputs and reports per-tensor similarity metrics — no dataset required. | +| `--mode` | | `onnx\|compare` | `onnx` | Evaluation mode. `onnx` evaluates the ONNX candidate on a dataset. `compare` runs the ONNX candidate and a reference on identical inputs and reports per-tensor similarity metrics — no dataset required. The reference is the HuggingFace model from `--model-id` by default, or a second ONNX file when `--reference` is given. | +| `--reference` | | `TEXT` | — | Reference `.onnx` file to compare the candidate against (used with `--mode compare`). Compares two ONNX models on identical random inputs; `--model-id` and `--task` are not required in this mode. Both models run on the same `--device` / `--ep`. | +| `--input-data` | | `PATH` | — | Path to a `.npz` file of real input tensors to compare with instead of randomly generated ones (used with `--mode compare`). Keys must match the candidate model's input names. The **leading axis of each array is the sample axis**, so an archive whose arrays have shape `(N, ...)` yields `N` samples (mean/std/min/max are computed across them); all inputs must share the same `N`. | ## How it works @@ -71,6 +73,18 @@ Evaluate a BERT model on the MRPC paraphrase task with column remapping: $ winml eval -m Intel/bert-base-uncased-mrpc --dataset nyu-mll/glue --dataset-name mrpc --column input_column=sentence1 --column second_input_column=sentence2 --samples 500 ``` +Compare two ONNX files directly (e.g. an fp32 baseline vs a quantized build), reporting per-output tensor-similarity metrics on identical random inputs — no `--model-id` or dataset needed: + +```bash +$ winml eval --mode compare -m quantized.onnx --reference baseline.onnx +``` + +Compare on real input tensors instead of random ones by passing a `.npz` archive whose keys match the candidate's input names. The leading axis of each array is the sample axis, so an archive shaped `(N, ...)` runs `N` samples: + +```bash +$ winml eval --mode compare -m quantized.onnx --reference baseline.onnx --input-data inputs.npz +``` + Check what dataset columns are expected before running, then remap them to match your dataset: ```bash diff --git a/src/winml/modelkit/commands/eval.py b/src/winml/modelkit/commands/eval.py index f97f72294..5416da1c3 100644 --- a/src/winml/modelkit/commands/eval.py +++ b/src/winml/modelkit/commands/eval.py @@ -160,6 +160,29 @@ "random inputs and report tensor-similarity metrics per output tensor." ), ) +@click.option( + "--reference", + "reference", + type=str, + default=None, + help=( + "Reference ONNX file to compare the candidate against (use with " + "--mode compare). Compares two ONNX models on identical random inputs; " + "--model-id / --task are not required in this mode." + ), +) +@click.option( + "--input-data", + "input_data", + type=click.Path(exists=True, dir_okay=False), + default=None, + help=( + "Path to a .npz file of real input tensors to compare with instead of " + "randomly generated ones (use with --mode compare). Keys must match the " + "candidate model's input names; the leading axis of each array is the " + "sample axis (N samples), and all inputs must share the same N." + ), +) @cli_utils.skip_build_option() @cli_utils.format_option() @cli_utils.build_config_option() @@ -199,6 +222,8 @@ def eval( mode: EvalMode, config_file: Path | None, skip_build: bool, + reference: str | None, + input_data: str | None, ) -> None: r"""Evaluate a model for a task. @@ -207,6 +232,10 @@ def eval( winml eval -m model.onnx --model-id microsoft/resnet-50 + winml eval --mode compare -m candidate.onnx --reference baseline.onnx + + winml eval --mode compare -m cand.onnx --reference base.onnx --input-data inputs.npz + Run `winml eval --schema --task ` to see the dataset columns and options expected by each task. """ @@ -240,7 +269,12 @@ def eval( cfg = _build_eval_config(ctx, config_file, column, label_mapping_path) # ── 2. Resolve in place ── - _resolve_model(cfg, model, model_id) + if cfg.reference_path is not None and cfg.mode != "compare": + raise click.UsageError("--reference is only valid with --mode compare.") + if cfg.input_data is not None and cfg.mode != "compare": + raise click.UsageError("--input-data is only valid with --mode compare.") + _resolve_model(cfg, model, model_id, allow_missing_model_id=cfg.reference_path is not None) + _resolve_reference(cfg) _resolve_device(cfg) _resolve_label_mapping(cfg) _run_dataset_script(cfg, trust_remote_code) @@ -361,13 +395,54 @@ def _resolve_model( cfg: WinMLEvaluationConfig, model: tuple[str, ...], model_id: str | None, + *, + allow_missing_model_id: bool = False, ) -> None: """Resolve ``-m`` / ``--model-id`` into ``cfg.model_path`` / ``cfg.model_id``.""" - model_path, resolved_id = _resolve_model_path(model=model, model_id=model_id) + model_path, resolved_id = _resolve_model_path( + model=model, model_id=model_id, allow_missing_model_id=allow_missing_model_id + ) cfg.model_path = model_path cfg.model_id = resolved_id +def _resolve_reference(cfg: WinMLEvaluationConfig) -> None: + """Validate and normalize ``cfg.reference_path`` for two-ONNX compare. + + Requires the candidate (``-m``) to be a single ONNX file (composite + ``role=path`` candidates and build-from-id are not supported with + ``--reference`` yet). Resolves Hub-hosted ONNX refs to local paths. + """ + if cfg.reference_path is None: + return + + if not isinstance(cfg.model_path, str): + raise click.UsageError( + "--reference requires the candidate (-m) to be a single ONNX file. " + "Composite (role=path) candidates and build-from-id are not " + "supported with --reference." + ) + + ref = cfg.reference_path + if Path(ref).suffix.lower() != ".onnx": + raise click.BadParameter( + f"--reference must be an .onnx file, got: {ref}", + param_hint="--reference", + ) + try: + ref = cli_utils.normalize_model_arg(ref) or ref + except Exception as e: + raise click.ClickException( + f"Failed to resolve Hub-hosted reference ONNX path {ref!r}: {e}" + ) from e + if not Path(ref).exists(): + raise click.BadParameter( + f"Reference ONNX file not found: {ref}", + param_hint="--reference", + ) + cfg.reference_path = ref + + def _resolve_device(cfg: WinMLEvaluationConfig) -> None: """Resolve ``'auto'`` → concrete device string on *cfg* in place.""" if cfg.device and cfg.device.lower() != "auto": @@ -451,8 +526,14 @@ def _resolve_model_path( *, model: tuple[str, ...], model_id: str | None, + allow_missing_model_id: bool = False, ) -> tuple[str | dict[str, str] | None, str | None]: - """Turn repeated -m values + --model-id into (model_path, model_id).""" + """Turn repeated -m values + --model-id into (model_path, model_id). + + When ``allow_missing_model_id`` is set (two-ONNX ``--mode compare``), a + plain ``-m .onnx`` is accepted without ``--model-id`` because the + candidate runs as a raw ORT session with no HF config resolution. + """ if not model: if model_id is not None: return None, model_id @@ -528,6 +609,8 @@ def _resolve_model_path( param_hint="-m/--model", ) if model_id is None: + if allow_missing_model_id: + return value, None raise click.UsageError( "When using an ONNX file, --model-id is required " "for preprocessor and config resolution." @@ -576,7 +659,11 @@ def display_eval_report(result: EvalResult, console: Console) -> None: console.print() console.print(f"[dim]Task:[/dim] {cfg.task}") console.print(f"[dim]Device:[/dim] {cfg.device}") - if ds.path: + if cfg.reference_path: + console.print(f"[dim]Reference:[/dim] {cfg.reference_path}") + if cfg.input_data: + console.print(f"[dim]Input data:[/dim] {cfg.input_data}") + elif ds.path: console.print(f"[dim]Dataset:[/dim] {ds.path}") console.print(f"[dim]Samples:[/dim] {ds.samples}") if cfg.model_path: diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 30d533699..006115d0e 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -381,85 +381,15 @@ def load_input_data( ) -> dict[str, np.ndarray]: """Load benchmark inputs from a ``.npz`` file, validated against the model. - Lets ``winml perf`` profile with real input tensors instead of randomly - generated ones. Only ``.npz`` (a named-array archive) is supported today; - a single-array ``.npy`` carries no input names to bind against and is - rejected with guidance to repackage as ``.npz``. - - Validation: - - * the archive's keys must exactly match the model's input names -- any - missing or unexpected key is an error (an unexpected key is usually a - typo that would otherwise leave a required input silently unset); - * an array whose dtype differs from the model's expected input dtype is - cast to the expected dtype with a warning, matching the silent casting - ``WinMLSession._prepare_inputs`` does on a normal run (e.g. numpy's - default int64 literals binding to an int32 input). - - Shapes are taken from the arrays as-is; correctness beyond dtype (e.g. a - static dimension the data violates) surfaces as a runtime error from the - inference session. - - Args: - path: Path to the ``.npz`` file. - io_config: Model I/O configuration (``input_names``, ``input_types``). - - Returns: - Dictionary of ``input_name -> numpy array``. - - Raises: - click.UsageError: On a non-``.npz`` file or a key mismatch. + Thin wrapper over the shared + :func:`winml.modelkit.datasets.input_data.load_input_data`, which is also + used by ``winml eval --mode compare --input-data``. Imported lazily so + ``winml perf`` startup does not pull in the datasets package unless + ``--input-data`` is actually used. """ - if path.suffix.lower() == ".npy": - raise click.UsageError( - f"--input-data does not support .npy files ({path.name}). A single " - f"array carries no input names; save your inputs as a named .npz " - f"archive instead (e.g. np.savez('inputs.npz', input_ids=..., " - f"attention_mask=...))." - ) - if path.suffix.lower() != ".npz": - raise click.UsageError( - f"--input-data must be a .npz file, got '{path.suffix or path.name}'." - ) - - try: - with np.load(path, allow_pickle=False) as archive: - provided = {name: archive[name] for name in archive.files} - except Exception as exc: - raise click.UsageError(f"Could not read --input-data file {path}: {exc}") from exc - - expected_names = list(io_config["input_names"]) - expected_types = list(io_config["input_types"]) - - missing = [name for name in expected_names if name not in provided] - unexpected = [name for name in provided if name not in expected_names] - if missing or unexpected: - parts = [] - if missing: - parts.append(f"missing {missing}") - if unexpected: - parts.append(f"unexpected {unexpected}") - raise click.UsageError( - f"--input-data keys do not match the model inputs ({', '.join(parts)}). " - f"Expected exactly: {expected_names}." - ) - - # Cast dtype mismatches instead of failing, mirroring the session's - # _prepare_inputs, so inputs that would run fine on a normal invocation - # (e.g. int64 literals against an int32 input) don't hard-error here. - for name, expected_dtype in zip(expected_names, expected_types, strict=True): - want = np.dtype(expected_dtype) - got = provided[name].dtype - if got != want: - logger.warning( - "--input-data dtype for '%s' is %s; casting to the model's expected %s.", - name, - got, - want, - ) - provided[name] = provided[name].astype(want) + from ..datasets.input_data import load_input_data as _load_input_data - return provided + return _load_input_data(path, io_config) def effective_batch_size( diff --git a/src/winml/modelkit/datasets/input_data.py b/src/winml/modelkit/datasets/input_data.py new file mode 100644 index 000000000..193ab7abb --- /dev/null +++ b/src/winml/modelkit/datasets/input_data.py @@ -0,0 +1,187 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- + +"""Real input tensors loaded from a ``.npz`` archive. + +Shared by ``winml perf`` (benchmark on real tensors instead of random ones) +and ``winml eval --mode compare`` (compare a candidate and reference on the +same real inputs). :func:`load_input_data` validates and dtype-casts the +archive against a model's I/O config; :class:`InputDataDataset` wraps the +loaded archive as a torch dataset (leading axis = sample axis) the compare +loop can iterate. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import click +import numpy as np + + +if TYPE_CHECKING: + import torch + +logger = logging.getLogger(__name__) + + +def load_input_data( + path: Path, + io_config: dict[str, Any], +) -> dict[str, np.ndarray]: + """Load model inputs from a ``.npz`` file, validated against the model. + + Lets ``winml perf`` / ``winml eval`` run with real input tensors instead + of randomly generated ones. Only ``.npz`` (a named-array archive) is + supported today; a single-array ``.npy`` carries no input names to bind + against and is rejected with guidance to repackage as ``.npz``. + + Validation: + + * the archive's keys must exactly match the model's input names -- any + missing or unexpected key is an error (an unexpected key is usually a + typo that would otherwise leave a required input silently unset); + * an array whose dtype differs from the model's expected input dtype is + cast to the expected dtype with a warning, matching the silent casting + ``WinMLSession._prepare_inputs`` does on a normal run (e.g. numpy's + default int64 literals binding to an int32 input). + + Shapes are taken from the arrays as-is; correctness beyond dtype (e.g. a + static dimension the data violates) surfaces as a runtime error from the + inference session. + + Args: + path: Path to the ``.npz`` file. + io_config: Model I/O configuration (``input_names``, ``input_types``). + + Returns: + Dictionary of ``input_name -> numpy array``. + + Raises: + click.UsageError: On a non-``.npz`` file or a key mismatch. + """ + path = Path(path) + if path.suffix.lower() == ".npy": + raise click.UsageError( + f"--input-data does not support .npy files ({path.name}). A single " + f"array carries no input names; save your inputs as a named .npz " + f"archive instead (e.g. np.savez('inputs.npz', input_ids=..., " + f"attention_mask=...))." + ) + if path.suffix.lower() != ".npz": + raise click.UsageError( + f"--input-data must be a .npz file, got '{path.suffix or path.name}'." + ) + + try: + with np.load(path, allow_pickle=False) as archive: + provided = {name: archive[name] for name in archive.files} + except Exception as exc: + raise click.UsageError(f"Could not read --input-data file {path}: {exc}") from exc + + expected_names = list(io_config["input_names"]) + expected_types = list(io_config["input_types"]) + + missing = [name for name in expected_names if name not in provided] + unexpected = [name for name in provided if name not in expected_names] + if missing or unexpected: + parts = [] + if missing: + parts.append(f"missing {missing}") + if unexpected: + parts.append(f"unexpected {unexpected}") + raise click.UsageError( + f"--input-data keys do not match the model inputs ({', '.join(parts)}). " + f"Expected exactly: {expected_names}." + ) + + # Cast dtype mismatches instead of failing, mirroring the session's + # _prepare_inputs, so inputs that would run fine on a normal invocation + # (e.g. int64 literals against an int32 input) don't hard-error here. + for name, expected_dtype in zip(expected_names, expected_types, strict=True): + want = np.dtype(expected_dtype) + got = provided[name].dtype + if got != want: + logger.warning( + "--input-data dtype for '%s' is %s; casting to the model's expected %s.", + name, + got, + want, + ) + provided[name] = provided[name].astype(want) + + return provided + + +class InputDataDataset: + """Multi-sample dataset backed by a validated ``.npz`` of real tensors. + + Loads the archive once via :func:`load_input_data` (keys and dtypes + validated/cast against ``io_config``), then treats the **leading axis of + each array as the sample axis**: an archive whose arrays have shape + ``(N, ...)`` yields ``N`` samples, so ``--mode compare`` can run the + candidate and reference on many real inputs and report a real + distribution (mean/std/min/max) instead of a single point. + + Every input must share the same leading length ``N`` (a clear error is + raised otherwise). Each sample keeps a leading batch dim of 1 + (``arr[i:i+1]``), so any dynamic-batch model accepts it and no assumption + is made about output layout — each run is compared independently, exactly + like :class:`RandomDataset`'s per-sample flow. + + Args: + path: Path to the ``.npz`` file of real input tensors. + io_config: Candidate model I/O config (``input_names``, ``input_types``). + """ + + TASK_TYPE = "input_data" + + def __init__(self, path: str | Path, io_config: dict[str, Any]) -> None: + import torch + + arrays = load_input_data(Path(path), io_config) + + # Leading axis = sample axis. Reject scalars (no sample axis) and any + # disagreement on N so a silent mis-pairing can't produce bogus metrics. + leading: dict[str, int] = {} + for name, arr in arrays.items(): + if arr.ndim == 0: + raise click.UsageError( + f"--input-data array '{name}' is a scalar (0-d); the leading " + "axis is the sample axis, so each input needs at least one dim." + ) + leading[name] = int(arr.shape[0]) + + distinct = set(leading.values()) + if len(distinct) != 1: + detail = ", ".join(f"{name}={leading[name]}" for name in arrays) + raise click.UsageError( + "--input-data arrays must share the same leading (sample) axis " + f"length; got {detail}." + ) + + self._num_samples = distinct.pop() + if self._num_samples == 0: + raise click.UsageError("--input-data arrays are empty (sample axis length 0).") + + # np.load arrays are owned/writable; ascontiguousarray avoids the + # non-contiguous from_numpy warning without an extra copy when possible. + self._arrays: dict[str, torch.Tensor] = { + name: torch.from_numpy(np.ascontiguousarray(arr)) for name, arr in arrays.items() + } + + def __len__(self) -> int: + """Number of samples (the shared leading-axis length of the inputs).""" + return self._num_samples + + def __getitem__(self, idx: int) -> dict[str, torch.Tensor]: + """Return sample ``idx`` with each input sliced to a batch of 1.""" + if not 0 <= idx < self._num_samples: + raise IndexError( + f"InputDataDataset index {idx} out of range for {self._num_samples} samples." + ) + return {name: tensor[idx : idx + 1] for name, tensor in self._arrays.items()} diff --git a/src/winml/modelkit/eval/config.py b/src/winml/modelkit/eval/config.py index 923c0474d..62ee105ea 100644 --- a/src/winml/modelkit/eval/config.py +++ b/src/winml/modelkit/eval/config.py @@ -88,6 +88,16 @@ class WinMLEvaluationConfig: model_path: Path to .onnx model file, or a ``{role: path}`` dict for composite models (e.g. ``{"image-encoder": "...", "text-encoder": "..."}``). None = build from model_id. + reference_path: Path to a second ``.onnx`` file used as the reference in + ``--mode compare``. When set, both ``model_path`` and ``reference_path`` + run as raw ORT sessions and their output tensors are compared directly, + so no ``model_id`` / ``task`` / HF reference is needed. + input_data: Path to a ``.npz`` archive of real input tensors for + ``--mode compare``. When set, the candidate and reference are compared + on these tensors (validated against the candidate's inputs) instead of + randomly generated ones. The leading axis of each array is the sample + axis, so one archive can hold ``N`` samples; all inputs must share the + same leading length. task: HF pipeline task. Auto-detected from model_id if omitted. device: Target device for inference. ep: Explicit execution provider (e.g., "qnn", "dml"). Overrides @@ -100,7 +110,10 @@ class WinMLEvaluationConfig: labeled dataset. - ``"compare"``: compare ONNX vs HF reference output tensors on identical random inputs and report tensor-similarity - metrics per output tensor. + metrics per output tensor. When ``reference_path`` is set, + the reference is a second ONNX file instead of the HF model. + When ``input_data`` is set, real tensors from a ``.npz`` are + used instead of random inputs. Usage: config = WinMLEvaluationConfig( @@ -111,6 +124,8 @@ class WinMLEvaluationConfig: model_id: str | None = None model_path: str | dict[str, str] | None = None + reference_path: str | None = field(default=None, metadata={"cli_name": "reference"}) + input_data: str | None = None task: str | None = None device: str = "auto" precision: str = "auto" @@ -134,6 +149,10 @@ def to_dict(self) -> dict: result["model_id"] = self.model_id if self.model_path is not None: result["model_path"] = self.model_path + if self.reference_path is not None: + result["reference_path"] = self.reference_path + if self.input_data is not None: + result["input_data"] = self.input_data if self.task is not None: result["task"] = self.task result["device"] = self.device @@ -181,6 +200,8 @@ def from_dict(cls, data: dict) -> WinMLEvaluationConfig: return cls( model_id=data.get("model_id"), model_path=data.get("model_path"), + reference_path=data.get("reference_path"), + input_data=data.get("input_data"), task=data.get("task"), device=data.get("device", "auto"), precision=data.get("precision", "auto"), diff --git a/src/winml/modelkit/eval/evaluate.py b/src/winml/modelkit/eval/evaluate.py index 74863970f..c5f2827e7 100644 --- a/src/winml/modelkit/eval/evaluate.py +++ b/src/winml/modelkit/eval/evaluate.py @@ -231,6 +231,11 @@ def _load_model(config: WinMLEvaluationConfig) -> WinMLPreTrainedModel | WinMLCo from ..models import WinMLAutoModel from ..utils import cli as cli_utils + # Two-ONNX compare: the evaluator builds both raw ORT sessions directly from + # config.model_path / config.reference_path — no WinMLAutoModel / HF config. + if config.mode == "compare" and config.reference_path is not None: + return None + quant_override: Any = None if not config.quant: from ..config import WinMLBuildConfig @@ -327,8 +332,14 @@ def evaluate(config: WinMLEvaluationConfig) -> EvalResult: mode = config.mode if config.mode is not None else "onnx" if mode not in EVAL_MODES: raise ValueError(f"Invalid mode {mode!r}; expected one of {EVAL_MODES} or None.") + # Two-ONNX compare: both candidate and reference run as raw ORT sessions, so + # HF task resolution / model_id are not required — keep task as-is. + onnx_compare = mode == "compare" and config.reference_path is not None config = replace( - config, mode=mode, task=_resolve_task(config), dataset=deepcopy(config.dataset) + config, + mode=mode, + task=config.task if onnx_compare else _resolve_task(config), + dataset=deepcopy(config.dataset), ) if config.mode != "compare" and config.dataset.path is None: default = _DEFAULT_DATASETS.get(config.task) if config.task is not None else None @@ -390,10 +401,16 @@ def print_config(config: WinMLEvaluationConfig) -> None: """Print effective evaluation config to the console (quantize.py style).""" ds = config.dataset output_console = Console() - output_console.print(f"[bold blue]Model:[/bold blue] {config.model_id}") + if config.model_id is not None: + output_console.print(f"[bold blue]Model:[/bold blue] {config.model_id}") if config.model_path is not None: output_console.print(f"[bold blue]Model path:[/bold blue] {config.model_path}") - output_console.print(f"[bold blue]Task:[/bold blue] {config.task}") + if config.reference_path is not None: + output_console.print(f"[bold blue]Reference:[/bold blue] {config.reference_path}") + if config.input_data is not None: + output_console.print(f"[bold blue]Input data:[/bold blue] {config.input_data}") + if config.task is not None: + output_console.print(f"[bold blue]Task:[/bold blue] {config.task}") output_console.print(f"[bold blue]Device:[/bold blue] {config.device}") if config.ep is not None: output_console.print(f"[bold blue]EP:[/bold blue] {config.ep}") diff --git a/src/winml/modelkit/eval/tensor_similarity_evaluator.py b/src/winml/modelkit/eval/tensor_similarity_evaluator.py index 1b3ee7d84..c8894efc2 100644 --- a/src/winml/modelkit/eval/tensor_similarity_evaluator.py +++ b/src/winml/modelkit/eval/tensor_similarity_evaluator.py @@ -5,10 +5,16 @@ """Tensor-similarity evaluator. -Runs an ONNX candidate and an HF PyTorch reference on identical random -inputs (drawn from :class:`RandomDataset` over the candidate's ONNX I/O) -and reports per-output tensor-parity metrics (SQNR, PSNR, cosine, MSE, -max absolute diff) via :class:`TensorSimilarityMetric`. +Runs an ONNX candidate and a reference on identical inputs (random by +default, drawn from :class:`RandomDataset` over the candidate's ONNX I/O) +and reports per-output tensor-parity metrics (SQNR, PSNR, cosine, MSE, max +absolute diff) via :class:`TensorSimilarityMetric`. + +The reference is an HF PyTorch model resolved from ``model_id`` by default. +When ``config.reference_path`` is set, the reference is instead a second +ONNX file and both sides run as raw ORT sessions (no HF config / task). +When ``config.input_data`` is set, both sides run on real tensors from a +``.npz`` archive instead of random inputs. No labeled dataset, no HF pipeline, no preprocessor — any divergence reflects the build pipeline (optimize / quantize / compile) only. @@ -29,6 +35,37 @@ logger = logging.getLogger(__name__) +class _ONNXSessionModel: + """Minimal raw-ORT wrapper for two-ONNX ``--mode compare``. + + Exposes just the slice of the :class:`WinMLPreTrainedModel` surface that + the tensor-similarity loop needs — ``onnx_path``, ``io_config`` and a + callable returning named ``torch`` tensors — without any HF config or + task-specific output renaming, so both sides compare on their raw ONNX + output names. + """ + + def __init__(self, onnx_path: str, device: str = "auto", ep: Any | None = None) -> None: + from pathlib import Path + + from ..session.session import WinMLSession + + self.onnx_path = Path(onnx_path) + self._session = WinMLSession(onnx_path=self.onnx_path, device=device, ep=ep) + + @property + def io_config(self) -> dict: + """ONNX I/O metadata (delegated to the session).""" + return self._session.io_config + + def __call__(self, **inputs: Any) -> dict[str, Any]: + """Run one sample and return raw outputs as named ``torch`` tensors.""" + import torch + + outputs = self._session.run(inputs) + return {name: torch.from_numpy(arr) for name, arr in outputs.items()} + + class TensorSimilarityEvaluator: """Per-output tensor parity between an ONNX candidate and an HF reference.""" @@ -39,6 +76,20 @@ def __init__( ) -> None: from ..models.winml.composite_model import WinMLCompositeModel + self.config = config + + # Two-ONNX compare: build both raw ORT sessions directly, bypassing the + # HF PyTorch reference. ``model`` is None here (see evaluate._load_model). + if config.reference_path is not None: + self.model = _ONNXSessionModel( + str(config.model_path), device=config.device, ep=config.ep + ) + self.reference_model = _ONNXSessionModel( + str(config.reference_path), device=config.device, ep=config.ep + ) + self.data = self.prepare_data() + return + # Composite models must be split into their sub-components before # tensor-similarity comparison — the union param keeps this runtime # guard live for type checkers. @@ -50,7 +101,6 @@ def __init__( "Example: winml eval --mode compare --task " f"--model --model-id {config.model_id}" ) - self.config = config self.model = model self.reference_model = self._load_reference_model() self.data = self.prepare_data() @@ -79,7 +129,23 @@ def _load_reference_model(self) -> Any: ).eval() def prepare_data(self) -> Any: - """Build a RandomDataset over the candidate ONNX's I/O spec.""" + """Build the compare dataset over the candidate ONNX's I/O spec. + + Uses real tensors from ``config.input_data`` (wrapped as a multi-sample + :class:`InputDataDataset` whose leading axis is the sample axis and + validated against the candidate's inputs) when provided, otherwise a + :class:`RandomDataset` of synthetic inputs sized by ``config.dataset``. + """ + if self.config.input_data is not None: + from ..datasets.input_data import InputDataDataset + + dataset = InputDataDataset(self.config.input_data, self.model.io_config) + # Reflect the real sample count (leading axis of the .npz) in the + # effective config so the report header / JSON show N, not the + # unused dataset default. + self.config.dataset.samples = len(dataset) + return dataset + from ..datasets.random_dataset import RandomDataset ds = self.config.dataset diff --git a/tests/unit/commands/test_eval.py b/tests/unit/commands/test_eval.py index a73b9f520..dbfb6e585 100644 --- a/tests/unit/commands/test_eval.py +++ b/tests/unit/commands/test_eval.py @@ -14,7 +14,8 @@ import pytest from click.testing import CliRunner -from winml.modelkit.commands.eval import _resolve_model_path +from winml.modelkit.commands.eval import _resolve_model_path, _resolve_reference +from winml.modelkit.eval import WinMLEvaluationConfig # --------------------------------------------------------------------------- @@ -101,6 +102,14 @@ def test_plain_onnx_without_model_id_raises(self, onnx_file): with pytest.raises(click.UsageError, match="--model-id is required"): _resolve_model_path(model=(str(onnx_file),), model_id=None) + def test_plain_onnx_without_model_id_allowed_for_compare(self, onnx_file): + """allow_missing_model_id (two-ONNX compare) accepts a bare ONNX path.""" + path, mid = _resolve_model_path( + model=(str(onnx_file),), model_id=None, allow_missing_model_id=True + ) + assert path == str(onnx_file) + assert mid is None + def test_plain_onnx_missing_file_raises(self, tmp_path): missing = tmp_path / "does-not-exist.onnx" with pytest.raises(click.BadParameter, match="ONNX file not found"): @@ -234,13 +243,8 @@ def test_composite_hub_refs_resolved(self, tmp_path): enc_local.write_bytes(b"") dec_local = tmp_path / "prompt_encoder_mask_decoder_int8.onnx" dec_local.write_bytes(b"") - enc_ref = ( - "onnx-community/sam3-tracker-ONNX/onnx/vision_encoder_int8.onnx" - ) - dec_ref = ( - "onnx-community/sam3-tracker-ONNX/onnx/" - "prompt_encoder_mask_decoder_int8.onnx" - ) + enc_ref = "onnx-community/sam3-tracker-ONNX/onnx/vision_encoder_int8.onnx" + dec_ref = "onnx-community/sam3-tracker-ONNX/onnx/prompt_encoder_mask_decoder_int8.onnx" # Map each Hub ref to its (different) local cache location. def fake_resolve(ref, **kwargs): @@ -271,9 +275,7 @@ def test_composite_mixed_hub_and_local(self, onnx_vision, tmp_path): """One role is a Hub ref, the other is a local path -- both work.""" dec_local = tmp_path / "decoder.onnx" dec_local.write_bytes(b"") - enc_ref = ( - "onnx-community/sam3-tracker-ONNX/onnx/vision_encoder_int8.onnx" - ) + enc_ref = "onnx-community/sam3-tracker-ONNX/onnx/vision_encoder_int8.onnx" # ``resolve_hf_onnx_path`` is the underlying downloader; the # unified classifier+resolver only calls it for hub_onnx inputs. @@ -334,6 +336,108 @@ def test_model_help_mentions_onnx_model_id_and_role_path(self, runner: CliRunner assert "requires --model-id" in result.output assert "role=path" in result.output + def test_help_mentions_reference(self, runner: CliRunner): + from winml.modelkit.commands.eval import eval as eval_cmd + + result = runner.invoke(eval_cmd, ["--help"]) + + assert result.exit_code == 0, result.output + assert "--reference" in result.output + + def test_help_mentions_input_data(self, runner: CliRunner): + from winml.modelkit.commands.eval import eval as eval_cmd + + result = runner.invoke(eval_cmd, ["--help"]) + + assert result.exit_code == 0, result.output + assert "--input-data" in result.output + + +class TestResolveReference: + def test_none_is_noop(self): + cfg = WinMLEvaluationConfig(model_path="m.onnx", mode="compare") + _resolve_reference(cfg) + assert cfg.reference_path is None + + def test_happy_path(self, onnx_file, onnx_vision): + cfg = WinMLEvaluationConfig( + model_path=str(onnx_file), + reference_path=str(onnx_vision), + mode="compare", + ) + _resolve_reference(cfg) + assert cfg.reference_path == str(onnx_vision) + + def test_requires_onnx_candidate(self): + cfg = WinMLEvaluationConfig( + model_path=None, + reference_path="ref.onnx", + mode="compare", + ) + with pytest.raises(click.UsageError, match="single ONNX file"): + _resolve_reference(cfg) + + def test_composite_candidate_rejected(self, onnx_vision): + cfg = WinMLEvaluationConfig( + model_path={"encoder": "a.onnx"}, + reference_path=str(onnx_vision), + mode="compare", + ) + with pytest.raises(click.UsageError, match="single ONNX file"): + _resolve_reference(cfg) + + def test_non_onnx_suffix_raises(self, onnx_file, tmp_path): + bad = tmp_path / "ref.txt" + bad.write_bytes(b"") + cfg = WinMLEvaluationConfig( + model_path=str(onnx_file), + reference_path=str(bad), + mode="compare", + ) + with pytest.raises(click.BadParameter, match=r"must be an \.onnx file"): + _resolve_reference(cfg) + + def test_missing_file_raises(self, onnx_file, tmp_path): + missing = tmp_path / "missing.onnx" + cfg = WinMLEvaluationConfig( + model_path=str(onnx_file), + reference_path=str(missing), + mode="compare", + ) + with pytest.raises(click.BadParameter, match="not found"): + _resolve_reference(cfg) + + +class TestReferenceModeGuard: + def test_reference_requires_compare_mode(self, runner: CliRunner, onnx_file): + from winml.modelkit.commands.eval import eval as eval_cmd + + result = runner.invoke( + eval_cmd, + ["-m", str(onnx_file), "--reference", str(onnx_file)], + obj={"debug": False}, + ) + assert result.exit_code != 0 + assert "--reference is only valid with --mode compare" in result.output + + +class TestInputDataModeGuard: + def test_input_data_requires_compare_mode(self, runner: CliRunner, onnx_file, tmp_path): + import numpy as np + + from winml.modelkit.commands.eval import eval as eval_cmd + + npz = tmp_path / "inputs.npz" + np.savez(npz, x=np.zeros((1, 4), dtype=np.float32)) + + result = runner.invoke( + eval_cmd, + ["-m", str(onnx_file), "--input-data", str(npz)], + obj={"debug": False}, + ) + assert result.exit_code != 0 + assert "--input-data is only valid with --mode compare" in result.output + @pytest.fixture def eval_config_file(tmp_path): diff --git a/tests/unit/commands/test_perf_cli.py b/tests/unit/commands/test_perf_cli.py index be4984977..65826ef6e 100644 --- a/tests/unit/commands/test_perf_cli.py +++ b/tests/unit/commands/test_perf_cli.py @@ -1052,7 +1052,7 @@ def test_dtype_cast_with_warning(self, tmp_path, caplog) -> None: } path = self._write_npz(tmp_path, input_ids=np.zeros((1, 8), dtype=np.int64)) - with caplog.at_level(logging.WARNING, logger="winml.modelkit.commands.perf"): + with caplog.at_level(logging.WARNING, logger="winml.modelkit.datasets.input_data"): inputs = load_input_data(path, io) assert inputs["input_ids"].dtype == np.int32 diff --git a/tests/unit/datasets/test_input_data.py b/tests/unit/datasets/test_input_data.py new file mode 100644 index 000000000..5f0b89918 --- /dev/null +++ b/tests/unit/datasets/test_input_data.py @@ -0,0 +1,131 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- + +"""Unit tests for winml.modelkit.datasets.input_data. + +Covers the shared ``.npz`` loader (also exercised via ``winml perf``) and the +multi-sample :class:`InputDataDataset` used by ``winml eval --mode compare``. +""" + +from __future__ import annotations + +import logging +from typing import ClassVar + +import click +import numpy as np +import pytest +import torch + +from winml.modelkit.datasets.input_data import InputDataDataset, load_input_data + + +class TestLoadInputData: + _IO: ClassVar[dict] = { + "input_names": ["pixel_values"], + "input_shapes": [[None, 3, 8, 8]], + "input_types": ["float32"], + } + + def _write_npz(self, tmp_path, **arrays): + path = tmp_path / "inputs.npz" + np.savez(path, **arrays) + return path + + def test_loads_matching_npz(self, tmp_path) -> None: + path = self._write_npz(tmp_path, pixel_values=np.zeros((2, 3, 8, 8), dtype=np.float32)) + inputs = load_input_data(path, self._IO) + assert list(inputs) == ["pixel_values"] + assert inputs["pixel_values"].shape == (2, 3, 8, 8) + + def test_key_mismatch_errors(self, tmp_path) -> None: + path = self._write_npz(tmp_path, wrong=np.zeros((1, 3, 8, 8), dtype=np.float32)) + with pytest.raises(click.UsageError, match="do not match"): + load_input_data(path, self._IO) + + def test_dtype_cast_with_warning(self, tmp_path, caplog) -> None: + io = {"input_names": ["input_ids"], "input_shapes": [[None, 8]], "input_types": ["int32"]} + path = self._write_npz(tmp_path, input_ids=np.zeros((1, 8), dtype=np.int64)) + with caplog.at_level(logging.WARNING, logger="winml.modelkit.datasets.input_data"): + inputs = load_input_data(path, io) + assert inputs["input_ids"].dtype == np.int32 + assert "casting" in caplog.text.lower() + + def test_npy_rejected(self, tmp_path) -> None: + path = tmp_path / "inputs.npy" + np.save(path, np.zeros((1, 3, 8, 8), dtype=np.float32)) + with pytest.raises(click.UsageError, match=r"does not support \.npy"): + load_input_data(path, self._IO) + + +class TestInputDataDataset: + _IO: ClassVar[dict] = { + "input_names": ["x"], + "input_shapes": [[None, 4]], + "input_types": ["float32"], + } + + _IO2: ClassVar[dict] = { + "input_names": ["x", "y"], + "input_shapes": [[None, 4], [None, 2]], + "input_types": ["float32", "float32"], + } + + def _write_npz(self, tmp_path, **arrays): + path = tmp_path / "inputs.npz" + np.savez(path, **arrays) + return path + + def test_leading_axis_is_sample_axis(self, tmp_path) -> None: + # (3, 4) -> 3 samples, each sliced to a batch of 1: (1, 4). + path = self._write_npz(tmp_path, x=np.arange(12, dtype=np.float32).reshape(3, 4)) + ds = InputDataDataset(path, self._IO) + + assert len(ds) == 3 + for i in range(3): + sample = ds[i] + assert set(sample) == {"x"} + assert isinstance(sample["x"], torch.Tensor) + assert sample["x"].shape == (1, 4) + # Rows are the original rows, in order. + assert ds[0]["x"].tolist() == [[0.0, 1.0, 2.0, 3.0]] + assert ds[2]["x"].tolist() == [[8.0, 9.0, 10.0, 11.0]] + + def test_single_row_is_one_sample(self, tmp_path) -> None: + path = self._write_npz(tmp_path, x=np.ones((1, 4), dtype=np.float32)) + ds = InputDataDataset(path, self._IO) + assert len(ds) == 1 + assert ds[0]["x"].shape == (1, 4) + + def test_multiple_inputs_share_leading_dim(self, tmp_path) -> None: + path = self._write_npz( + tmp_path, + x=np.zeros((2, 4), dtype=np.float32), + y=np.ones((2, 2), dtype=np.float32), + ) + ds = InputDataDataset(path, self._IO2) + assert len(ds) == 2 + assert ds[1]["x"].shape == (1, 4) + assert ds[1]["y"].shape == (1, 2) + + def test_mismatched_leading_dims_error(self, tmp_path) -> None: + path = self._write_npz( + tmp_path, + x=np.zeros((3, 4), dtype=np.float32), + y=np.ones((2, 2), dtype=np.float32), + ) + with pytest.raises(click.UsageError, match="same leading"): + InputDataDataset(path, self._IO2) + + def test_index_out_of_range(self, tmp_path) -> None: + path = self._write_npz(tmp_path, x=np.ones((2, 4), dtype=np.float32)) + ds = InputDataDataset(path, self._IO) + with pytest.raises(IndexError): + _ = ds[2] + + def test_validates_keys_against_io_config(self, tmp_path) -> None: + path = self._write_npz(tmp_path, wrong=np.ones((1, 4), dtype=np.float32)) + with pytest.raises(click.UsageError, match="do not match"): + InputDataDataset(path, self._IO) diff --git a/tests/unit/eval/test_eval.py b/tests/unit/eval/test_eval.py index 1fdbf024e..c5338369d 100644 --- a/tests/unit/eval/test_eval.py +++ b/tests/unit/eval/test_eval.py @@ -55,6 +55,40 @@ def test_dataset_config_revision_default_is_none(self): assert ds.revision is None assert "revision" not in ds.to_dict() + def test_reference_path_default_is_none(self): + """reference_path defaults to None and is omitted from to_dict.""" + config = WinMLEvaluationConfig(model_id="test/model") + assert config.reference_path is None + assert "reference_path" not in config.to_dict() + + def test_config_roundtrip_preserves_reference_path(self): + """reference_path survives to_dict/from_dict roundtrip.""" + config = WinMLEvaluationConfig( + model_path="cand.onnx", + reference_path="ref.onnx", + mode="compare", + ) + restored = WinMLEvaluationConfig.from_dict(config.to_dict()) + assert restored.reference_path == "ref.onnx" + assert restored.mode == "compare" + + def test_input_data_default_is_none(self): + """input_data defaults to None and is omitted from to_dict.""" + config = WinMLEvaluationConfig(model_id="test/model") + assert config.input_data is None + assert "input_data" not in config.to_dict() + + def test_config_roundtrip_preserves_input_data(self): + """input_data survives to_dict/from_dict roundtrip.""" + config = WinMLEvaluationConfig( + model_path="cand.onnx", + reference_path="ref.onnx", + input_data="inputs.npz", + mode="compare", + ) + restored = WinMLEvaluationConfig.from_dict(config.to_dict()) + assert restored.input_data == "inputs.npz" + def test_eval_result_to_dict(self): config = WinMLEvaluationConfig( model_id="test/model", @@ -221,6 +255,50 @@ def test_none_mode_normalizes_to_onnx(self): result = eval_mod.evaluate(config) assert result.config.mode == "onnx" + def test_onnx_compare_skips_task_resolution_and_dataset(self): + """Two-ONNX compare skips HF task resolution and default-dataset lookup.""" + import importlib + import sys + + eval_mod = sys.modules.get( + "winml.modelkit.eval.evaluate", + ) or importlib.import_module("winml.modelkit.eval.evaluate") + + config = WinMLEvaluationConfig( + model_path="cand.onnx", + reference_path="ref.onnx", + mode="compare", + ) + + evaluator = MagicMock() + evaluator.compute.return_value = {"cosine_mean": {"logits": 1.0}} + with ( + patch.object( + eval_mod, + "_resolve_task", + side_effect=AssertionError("task resolution must be skipped"), + ), + patch.object(eval_mod, "_load_model", return_value=None) as load_model, + patch.object(eval_mod, "get_evaluator_class", return_value=lambda *_a, **_k: evaluator), + ): + result = eval_mod.evaluate(config) + + assert result.config.mode == "compare" + assert result.config.task is None + assert result.metrics == {"cosine_mean": {"logits": 1.0}} + load_model.assert_called_once() + + def test_load_model_returns_none_for_onnx_compare(self): + """_load_model short-circuits (no model_id needed) for two-ONNX compare.""" + from winml.modelkit.eval.evaluate import _load_model + + config = WinMLEvaluationConfig( + model_path="cand.onnx", + reference_path="ref.onnx", + mode="compare", + ) + assert _load_model(config) is None + def test_no_dataset_no_default_raises(self): """Tasks without a default dataset raise ValueError.""" import importlib diff --git a/tests/unit/eval/test_tensor_similarity_evaluator.py b/tests/unit/eval/test_tensor_similarity_evaluator.py index 61bffb962..134967725 100644 --- a/tests/unit/eval/test_tensor_similarity_evaluator.py +++ b/tests/unit/eval/test_tensor_similarity_evaluator.py @@ -22,7 +22,10 @@ from transformers.modeling_outputs import BaseModelOutput from winml.modelkit.eval import DatasetConfig, WinMLEvaluationConfig -from winml.modelkit.eval.tensor_similarity_evaluator import TensorSimilarityEvaluator +from winml.modelkit.eval.tensor_similarity_evaluator import ( + TensorSimilarityEvaluator, + _ONNXSessionModel, +) from winml.modelkit.models.winml.composite_model import WinMLCompositeModel @@ -30,6 +33,7 @@ # _inference_model # --------------------------------------------------------------------------- + class _EchoModel: """Minimal stand-in that returns a BaseModelOutput from the inputs.""" @@ -77,6 +81,7 @@ def test_returns_numpy_dict_only_for_tensor_fields(self): # composite-model guard in __init__ # --------------------------------------------------------------------------- + class _FakeCompositeModel(WinMLCompositeModel): _SUB_MODEL_CONFIG: ClassVar[dict[str, str]] = { "encoder": "image-feature-extraction", @@ -86,9 +91,7 @@ class _FakeCompositeModel(WinMLCompositeModel): class TestCompositeGuard: def test_rejects_composite_with_helpful_message(self): - composite = _FakeCompositeModel( - sub_models={}, config=PretrainedConfig() - ) + composite = _FakeCompositeModel(sub_models={}, config=PretrainedConfig()) config = WinMLEvaluationConfig( model_id="Salesforce/blip-image-captioning-base", task="image-to-text", @@ -102,3 +105,124 @@ def test_rejects_composite_with_helpful_message(self): assert "image-feature-extraction" in msg assert "text-generation" in msg assert "Salesforce/blip-image-captioning-base" in msg + + +# --------------------------------------------------------------------------- +# Two-ONNX compare (reference_path set) +# --------------------------------------------------------------------------- + + +class _FakeSession: + """Stand-in for WinMLSession that records construction and echoes outputs.""" + + created: ClassVar[list[tuple[str, str, object]]] = [] + + def __init__(self, onnx_path, device="auto", ep=None): + _FakeSession.created.append((str(onnx_path), device, ep)) + self.io_config = {"input_names": ["input"], "input_types": ["float32"]} + + def run(self, inputs): + return {"logits": np.arange(3.0, dtype=np.float32).reshape(1, 3)} + + +class _FakeRandomDataset: + def __init__(self, **kwargs): + self.kwargs = kwargs + + def __len__(self): + return 0 + + +class TestONNXReferenceInit: + def test_builds_two_raw_sessions_honoring_device(self, monkeypatch): + import winml.modelkit.datasets.random_dataset as rd_mod + import winml.modelkit.session.session as session_mod + + _FakeSession.created = [] + monkeypatch.setattr(session_mod, "WinMLSession", _FakeSession) + monkeypatch.setattr(rd_mod, "RandomDataset", _FakeRandomDataset) + + config = WinMLEvaluationConfig( + model_path="cand.onnx", + reference_path="ref.onnx", + mode="compare", + device="cpu", + ep="dml", + dataset=DatasetConfig(samples=5, seed=1), + ) + + # ``model`` is None in this path (evaluate._load_model returns None). + evaluator = TensorSimilarityEvaluator(config, None) # type: ignore[arg-type] + + assert isinstance(evaluator.model, _ONNXSessionModel) + assert isinstance(evaluator.reference_model, _ONNXSessionModel) + # Candidate first, reference second; both honor --device / --ep. + assert _FakeSession.created[0][0].endswith("cand.onnx") + assert _FakeSession.created[1][0].endswith("ref.onnx") + assert [c[1] for c in _FakeSession.created] == ["cpu", "cpu"] + assert [c[2] for c in _FakeSession.created] == ["dml", "dml"] + # RandomDataset is built over the candidate ONNX I/O. + assert evaluator.data.kwargs["model_path"].endswith("cand.onnx") + assert evaluator.data.kwargs["max_samples"] == 5 + assert evaluator.data.kwargs["seed"] == 1 + + +class TestONNXSessionModel: + def test_call_returns_named_torch_tensors(self, monkeypatch): + import winml.modelkit.session.session as session_mod + + _FakeSession.created = [] + monkeypatch.setattr(session_mod, "WinMLSession", _FakeSession) + + model = _ONNXSessionModel("x.onnx", device="cpu") + out = model(input=torch.zeros(1, 3)) + + assert set(out) == {"logits"} + assert isinstance(out["logits"], torch.Tensor) + assert out["logits"].shape == (1, 3) + + def test_io_config_delegates_to_session(self, monkeypatch): + import winml.modelkit.session.session as session_mod + + _FakeSession.created = [] + monkeypatch.setattr(session_mod, "WinMLSession", _FakeSession) + + model = _ONNXSessionModel("x.onnx") + assert model.io_config["input_names"] == ["input"] + + +# --------------------------------------------------------------------------- +# Real-input compare (input_data set) +# --------------------------------------------------------------------------- + + +class TestInputDataCompare: + def test_prepare_data_uses_input_data_npz(self, monkeypatch, tmp_path): + import winml.modelkit.session.session as session_mod + from winml.modelkit.datasets.input_data import InputDataDataset + + _FakeSession.created = [] + monkeypatch.setattr(session_mod, "WinMLSession", _FakeSession) + + npz = tmp_path / "inputs.npz" + np.savez(npz, input=np.ones((2, 3), dtype=np.float32)) + + config = WinMLEvaluationConfig( + model_path="cand.onnx", + reference_path="ref.onnx", + mode="compare", + input_data=str(npz), + ) + + # ``model`` is None in this path (evaluate._load_model returns None). + evaluator = TensorSimilarityEvaluator(config, None) # type: ignore[arg-type] + + assert isinstance(evaluator.data, InputDataDataset) + # Leading axis is the sample axis: (2, 3) -> 2 samples of shape (1, 3). + assert len(evaluator.data) == 2 + sample = evaluator.data[0] + assert set(sample) == {"input"} + assert isinstance(sample["input"], torch.Tensor) + assert sample["input"].shape == (1, 3) + # The effective config reflects the real sample count for the report/JSON. + assert evaluator.config.dataset.samples == 2