Add two-ONNX comparison to winml eval --mode compare

docs/commands/eval.md

@@ -35,7 +35,9 @@ $ winml eval [options]
 | `--label-mapping` | | `PATH` | — | Path to a JSON file mapping dataset label names to the integer class IDs the model emits: `{"label_name": id}`. |
 | `--output` | `-o` | `PATH` | — | Output JSON file path for the evaluation results. |
 | `--schema` | | flag | `false` | Print the expected dataset schema for the given `--task` and exit. Does not run evaluation. |
-| `--mode` | | `onnx\|compare` | `onnx` | Evaluation mode. `onnx` evaluates the ONNX candidate on a dataset. `compare` runs the ONNX candidate and the HuggingFace reference on identical random inputs and reports per-tensor similarity metrics — no dataset required. |
+| `--mode` | | `onnx\|compare` | `onnx` | Evaluation mode. `onnx` evaluates the ONNX candidate on a dataset. `compare` runs the ONNX candidate and a reference on identical inputs and reports per-tensor similarity metrics — no dataset required. The reference is the HuggingFace model from `--model-id` by default, or a second ONNX file when `--reference` is given. |
+| `--reference` | | `TEXT` | — | Reference `.onnx` file to compare the candidate against (used with `--mode compare`). Compares two ONNX models on identical random inputs; `--model-id` and `--task` are not required in this mode. Both models run on the same `--device` / `--ep`. |
+| `--input-data` | | `PATH` | — | Path to a `.npz` file of real input tensors to compare with instead of randomly generated ones (used with `--mode compare`). Keys must match the candidate model's input names. The **leading axis of each array is the sample axis**, so an archive whose arrays have shape `(N, ...)` yields `N` samples (mean/std/min/max are computed across them); all inputs must share the same `N`. |
 
 ## How it works
 
@@ -71,6 +73,18 @@ Evaluate a BERT model on the MRPC paraphrase task with column remapping:
 $ winml eval -m Intel/bert-base-uncased-mrpc --dataset nyu-mll/glue --dataset-name mrpc --column input_column=sentence1 --column second_input_column=sentence2 --samples 500
 ```
 
+Compare two ONNX files directly (e.g. an fp32 baseline vs a quantized build), reporting per-output tensor-similarity metrics on identical random inputs — no `--model-id` or dataset needed:
+
+```bash
+$ winml eval --mode compare -m quantized.onnx --reference baseline.onnx
+```
+
+Compare on real input tensors instead of random ones by passing a `.npz` archive whose keys match the candidate's input names. The leading axis of each array is the sample axis, so an archive shaped `(N, ...)` runs `N` samples:
+
+```bash
+$ winml eval --mode compare -m quantized.onnx --reference baseline.onnx --input-data inputs.npz
+```
+
 Check what dataset columns are expected before running, then remap them to match your dataset:
 
 ```bash

src/winml/modelkit/commands/eval.py

@@ -160,6 +160,29 @@
         "random inputs and report tensor-similarity metrics per output tensor."
     ),
 )
+@click.option(
+    "--reference",
+    "reference",
+    type=str,
+    default=None,
+    help=(
+        "Reference ONNX file to compare the candidate against (use with "
+        "--mode compare). Compares two ONNX models on identical random inputs; "
+        "--model-id / --task are not required in this mode."
+    ),
+)
+@click.option(
+    "--input-data",
+    "input_data",
+    type=click.Path(exists=True, dir_okay=False),
+    default=None,
+    help=(
+        "Path to a .npz file of real input tensors to compare with instead of "
+        "randomly generated ones (use with --mode compare). Keys must match the "
+        "candidate model's input names; the leading axis of each array is the "
+        "sample axis (N samples), and all inputs must share the same N."
+    ),
+)
 @cli_utils.skip_build_option()
 @cli_utils.format_option()
 @cli_utils.build_config_option()
@@ -199,6 +222,8 @@ def eval(
     mode: EvalMode,
     config_file: Path | None,
     skip_build: bool,
+    reference: str | None,
+    input_data: str | None,
 ) -> None:
     r"""Evaluate a model for a task.
 
@@ -207,6 +232,10 @@ def eval(
 
         winml eval -m model.onnx --model-id microsoft/resnet-50
 
+        winml eval --mode compare -m candidate.onnx --reference baseline.onnx
+
+        winml eval --mode compare -m cand.onnx --reference base.onnx --input-data inputs.npz
+
     Run `winml eval --schema --task <task>` to see the dataset columns
     and options expected by each task.
     """
@@ -240,7 +269,12 @@ def eval(
     cfg = _build_eval_config(ctx, config_file, column, label_mapping_path)
 
     # ── 2. Resolve in place ──
-    _resolve_model(cfg, model, model_id)
+    if cfg.reference_path is not None and cfg.mode != "compare":
+        raise click.UsageError("--reference is only valid with --mode compare.")
+    if cfg.input_data is not None and cfg.mode != "compare":
+        raise click.UsageError("--input-data is only valid with --mode compare.")
+    _resolve_model(cfg, model, model_id, allow_missing_model_id=cfg.reference_path is not None)
+    _resolve_reference(cfg)
     _resolve_device(cfg)
     _resolve_label_mapping(cfg)
     _run_dataset_script(cfg, trust_remote_code)
@@ -361,13 +395,54 @@ def _resolve_model(
     cfg: WinMLEvaluationConfig,
     model: tuple[str, ...],
     model_id: str | None,
+    *,
+    allow_missing_model_id: bool = False,
 ) -> None:
     """Resolve ``-m`` / ``--model-id`` into ``cfg.model_path`` / ``cfg.model_id``."""
-    model_path, resolved_id = _resolve_model_path(model=model, model_id=model_id)
+    model_path, resolved_id = _resolve_model_path(
+        model=model, model_id=model_id, allow_missing_model_id=allow_missing_model_id
+    )
     cfg.model_path = model_path
     cfg.model_id = resolved_id
 
 
+def _resolve_reference(cfg: WinMLEvaluationConfig) -> None:
+    """Validate and normalize ``cfg.reference_path`` for two-ONNX compare.
+
+    Requires the candidate (``-m``) to be a single ONNX file (composite
+    ``role=path`` candidates and build-from-id are not supported with
+    ``--reference`` yet). Resolves Hub-hosted ONNX refs to local paths.
+    """
+    if cfg.reference_path is None:
+        return
+
+    if not isinstance(cfg.model_path, str):
+        raise click.UsageError(
+            "--reference requires the candidate (-m) to be a single ONNX file. "
+            "Composite (role=path) candidates and build-from-id are not "
+            "supported with --reference."
+        )
+
+    ref = cfg.reference_path
+    if Path(ref).suffix.lower() != ".onnx":
+        raise click.BadParameter(
+            f"--reference must be an .onnx file, got: {ref}",
+            param_hint="--reference",
+        )
+    try:
+        ref = cli_utils.normalize_model_arg(ref) or ref
+    except Exception as e:
+        raise click.ClickException(
+            f"Failed to resolve Hub-hosted reference ONNX path {ref!r}: {e}"
+        ) from e
+    if not Path(ref).exists():
+        raise click.BadParameter(
+            f"Reference ONNX file not found: {ref}",
+            param_hint="--reference",
+        )
+    cfg.reference_path = ref
+
+
 def _resolve_device(cfg: WinMLEvaluationConfig) -> None:
     """Resolve ``'auto'`` → concrete device string on *cfg* in place."""
     if cfg.device and cfg.device.lower() != "auto":
@@ -451,8 +526,14 @@ def _resolve_model_path(
     *,
     model: tuple[str, ...],
     model_id: str | None,
+    allow_missing_model_id: bool = False,
 ) -> tuple[str | dict[str, str] | None, str | None]:
-    """Turn repeated -m values + --model-id into (model_path, model_id)."""
+    """Turn repeated -m values + --model-id into (model_path, model_id).
+
+    When ``allow_missing_model_id`` is set (two-ONNX ``--mode compare``), a
+    plain ``-m <file>.onnx`` is accepted without ``--model-id`` because the
+    candidate runs as a raw ORT session with no HF config resolution.
+    """
     if not model:
         if model_id is not None:
             return None, model_id
@@ -528,6 +609,8 @@ def _resolve_model_path(
                 param_hint="-m/--model",
             )
         if model_id is None:
+            if allow_missing_model_id:
+                return value, None
             raise click.UsageError(
                 "When using an ONNX file, --model-id is required "
                 "for preprocessor and config resolution."
@@ -576,7 +659,11 @@ def display_eval_report(result: EvalResult, console: Console) -> None:
     console.print()
     console.print(f"[dim]Task:[/dim]       {cfg.task}")
     console.print(f"[dim]Device:[/dim]     {cfg.device}")
-    if ds.path:
+    if cfg.reference_path:
+        console.print(f"[dim]Reference:[/dim]  {cfg.reference_path}")
+    if cfg.input_data:
+        console.print(f"[dim]Input data:[/dim] {cfg.input_data}")
+    elif ds.path:
         console.print(f"[dim]Dataset:[/dim]    {ds.path}")
     console.print(f"[dim]Samples:[/dim]    {ds.samples}")
     if cfg.model_path:

src/winml/modelkit/commands/perf.py

@@ -381,85 +381,15 @@ def load_input_data(
 ) -> dict[str, np.ndarray]:
     """Load benchmark inputs from a ``.npz`` file, validated against the model.
 
-    Lets ``winml perf`` profile with real input tensors instead of randomly
-    generated ones. Only ``.npz`` (a named-array archive) is supported today;
-    a single-array ``.npy`` carries no input names to bind against and is
-    rejected with guidance to repackage as ``.npz``.
-
-    Validation:
-
-    * the archive's keys must exactly match the model's input names -- any
-      missing or unexpected key is an error (an unexpected key is usually a
-      typo that would otherwise leave a required input silently unset);
-    * an array whose dtype differs from the model's expected input dtype is
-      cast to the expected dtype with a warning, matching the silent casting
-      ``WinMLSession._prepare_inputs`` does on a normal run (e.g. numpy's
-      default int64 literals binding to an int32 input).
-
-    Shapes are taken from the arrays as-is; correctness beyond dtype (e.g. a
-    static dimension the data violates) surfaces as a runtime error from the
-    inference session.
-
-    Args:
-        path: Path to the ``.npz`` file.
-        io_config: Model I/O configuration (``input_names``, ``input_types``).
-
-    Returns:
-        Dictionary of ``input_name -> numpy array``.
-
-    Raises:
-        click.UsageError: On a non-``.npz`` file or a key mismatch.
+    Thin wrapper over the shared
+    :func:`winml.modelkit.datasets.input_data.load_input_data`, which is also
+    used by ``winml eval --mode compare --input-data``. Imported lazily so
+    ``winml perf`` startup does not pull in the datasets package unless
+    ``--input-data`` is actually used.
     """
-    if path.suffix.lower() == ".npy":
-        raise click.UsageError(
-            f"--input-data does not support .npy files ({path.name}). A single "
-            f"array carries no input names; save your inputs as a named .npz "
-            f"archive instead (e.g. np.savez('inputs.npz', input_ids=..., "
-            f"attention_mask=...))."
-        )
-    if path.suffix.lower() != ".npz":
-        raise click.UsageError(
-            f"--input-data must be a .npz file, got '{path.suffix or path.name}'."
-        )
-
-    try:
-        with np.load(path, allow_pickle=False) as archive:
-            provided = {name: archive[name] for name in archive.files}
-    except Exception as exc:
-        raise click.UsageError(f"Could not read --input-data file {path}: {exc}") from exc
-
-    expected_names = list(io_config["input_names"])
-    expected_types = list(io_config["input_types"])
-
-    missing = [name for name in expected_names if name not in provided]
-    unexpected = [name for name in provided if name not in expected_names]
-    if missing or unexpected:
-        parts = []
-        if missing:
-            parts.append(f"missing {missing}")
-        if unexpected:
-            parts.append(f"unexpected {unexpected}")
-        raise click.UsageError(
-            f"--input-data keys do not match the model inputs ({', '.join(parts)}). "
-            f"Expected exactly: {expected_names}."
-        )
-
-    # Cast dtype mismatches instead of failing, mirroring the session's
-    # _prepare_inputs, so inputs that would run fine on a normal invocation
-    # (e.g. int64 literals against an int32 input) don't hard-error here.
-    for name, expected_dtype in zip(expected_names, expected_types, strict=True):
-        want = np.dtype(expected_dtype)
-        got = provided[name].dtype
-        if got != want:
-            logger.warning(
-                "--input-data dtype for '%s' is %s; casting to the model's expected %s.",
-                name,
-                got,
-                want,
-            )
-            provided[name] = provided[name].astype(want)
+    from ..datasets.input_data import load_input_data as _load_input_data
 
-    return provided
+    return _load_input_data(path, io_config)
 
 
 def effective_batch_size(