Prefer metadata over store reads in _decode_cf_datetime_dtype

doc/whats-new.rst

@@ -163,6 +163,19 @@ Deprecations
 ~~~~~~~~~~~~
 
 
+Performance
+~~~~~~~~~~~
+
+- Add ``infer_dtype`` parameter to :py:class:`xarray.coders.CFDatetimeCoder`.
+  With ``infer_dtype="metadata"``, dtype inference uses only ``units``,
+  ``calendar``, ``use_cftime`` and ``time_unit`` — no data reads. This
+  significantly speeds up :py:func:`open_dataset` / :py:func:`open_datatree`
+  on remote-backed stores (zarr on S3, icechunk) with many time-encoded
+  variables. The default (``infer_dtype="data"``) preserves the current
+  behavior of reading the first and last values to verify decoding
+  (:issue:`11303`).
+  By `Alfonso Ladino <https://github.com/aladinor>`_.
+
 Bug Fixes
 ~~~~~~~~~
 

xarray/coding/times.py

@@ -6,7 +6,7 @@
 from collections.abc import Callable, Hashable
 from datetime import datetime, timedelta
 from functools import partial
-from typing import TYPE_CHECKING, Union, cast
+from typing import TYPE_CHECKING, Literal, Union, cast
 
 import numpy as np
 import pandas as pd
@@ -336,21 +336,31 @@ def _unpack_time_units_and_ref_date_cftime(units: str, calendar: str):
     return time_units, ref_date
 
 
+_DECODE_TIME_ERROR_MSG = (
+    "unable to decode time units {units!r} with {calendar_msg}. Try opening "
+    "your dataset with decode_times=False or installing cftime if it is not "
+    "installed."
+)
+
+
 def _decode_cf_datetime_dtype(
     data,
     units: str,
     calendar: str | None,
     use_cftime: bool | None,
     time_unit: PDDatetimeUnitOptions = "ns",
 ) -> np.dtype:
-    # Verify that at least the first and last date can be decoded
-    # successfully. Otherwise, tracebacks end up swallowed by
-    # Dataset.__repr__ when users try to view their lazily decoded array.
+    """Infer the decoded dtype by reading the first and last data values.
+
+    This is the default ``infer_dtype="data"`` path: it verifies that the
+    first and last dates can actually be decoded, so any overflow
+    surfaces at open time (rather than producing wrong-dtype lazy
+    wrappers).
+    """
     values = indexing.ImplicitToExplicitIndexingAdapter(indexing.as_indexable(data))
     example_value = np.concatenate(
         [to_numpy(first_n_items(values, 1)), to_numpy(last_item(values))]
     )
-
     try:
         result = decode_cf_datetime(
             example_value, units, calendar, use_cftime, time_unit
@@ -359,16 +369,55 @@ def _decode_cf_datetime_dtype(
         calendar_msg = (
             "the default calendar" if calendar is None else f"calendar {calendar!r}"
         )
-        msg = (
-            f"unable to decode time units {units!r} with {calendar_msg!r}. Try "
-            "opening your dataset with decode_times=False or installing cftime "
-            "if it is not installed."
-        )
-        raise ValueError(msg) from err
-    else:
-        dtype = getattr(result, "dtype", np.dtype("object"))
+        raise ValueError(
+            _DECODE_TIME_ERROR_MSG.format(units=units, calendar_msg=calendar_msg)
+        ) from err
+    return getattr(result, "dtype", np.dtype("object"))
 
-    return dtype
+
+def _decode_cf_datetime_dtype_from_metadata(
+    units: str,
+    calendar: str | None,
+    use_cftime: bool | None,
+    time_unit: PDDatetimeUnitOptions = "ns",
+) -> np.dtype:
+    """Infer the decoded dtype from metadata alone, without reading data.
+
+    This is the opt-in ``infer_dtype="metadata"`` path. Short-circuits to
+    a deterministic dtype when ``use_cftime`` is set or the calendar is
+    non-standard (so ``cftime`` does not need to be importable);
+    otherwise runs ``decode_cf_datetime`` on a synthetic ``[0, 1]`` array
+    so the resolution-selection logic matches real-data output. With
+    this path, the dtype no longer depends on the actual values in the
+    store — out-of-range values may produce a dask cast mismatch.
+    """
+    calendar_msg = (
+        "the default calendar" if calendar is None else f"calendar {calendar!r}"
+    )
+    # Validate units up front so every metadata path fails fast on
+    # malformed strings (parity with the data-reading path).
+    try:
+        _unpack_time_unit_and_ref_date(units)
+    except Exception as err:
+        raise ValueError(
+            _DECODE_TIME_ERROR_MSG.format(units=units, calendar_msg=calendar_msg)
+        ) from err
+
+    if use_cftime is True:
+        return np.dtype("object")
+    if use_cftime is False:
+        return np.dtype(f"datetime64[{time_unit}]")
+    if calendar is not None and not _is_standard_calendar(calendar):
+        return np.dtype("object")
+    try:
+        result = decode_cf_datetime(
+            np.array([0, 1]), units, calendar, use_cftime, time_unit
+        )
+    except Exception as err:
+        raise ValueError(
+            _DECODE_TIME_ERROR_MSG.format(units=units, calendar_msg=calendar_msg)
+        ) from err
+    return getattr(result, "dtype", np.dtype("object"))
 
 
 def _decode_datetime_with_cftime(
@@ -1370,15 +1419,37 @@ class CFDatetimeCoder(VariableCoder):
         May not be supported by all the backends.
     time_unit : PDDatetimeUnitOptions
           Target resolution when decoding dates. Defaults to "ns".
+    infer_dtype : {"data", "metadata"}, default: "data"
+        How to infer the dtype of the decoded variable:
+
+        - ``"data"`` (default): read the first and last values from the
+          store to verify they decode successfully. Preserves
+          backwards-compatible behavior, including the auto-fallback to
+          ``cftime.datetime`` for values outside the
+          ``np.datetime64[ns]`` range.
+        - ``"metadata"``: infer the dtype from ``units``, ``calendar``,
+          ``use_cftime`` and ``time_unit`` alone, with no reads from the
+          store. Significantly speeds up ``open_dataset`` /
+          ``open_datatree`` on remote stores (zarr on S3, icechunk) with
+          many time-encoded variables, but does not verify that all
+          values are in range — if you have dates outside
+          ``np.datetime64[ns]`` (before 1677 or after 2262), pass
+          ``use_cftime=True`` as well.
     """
 
     def __init__(
         self,
         use_cftime: bool | None = None,
         time_unit: PDDatetimeUnitOptions = "ns",
+        infer_dtype: Literal["data", "metadata"] = "data",
     ) -> None:
+        if infer_dtype not in ("data", "metadata"):
+            raise ValueError(
+                f"infer_dtype must be 'data' or 'metadata', got {infer_dtype!r}"
+            )
         self.use_cftime = use_cftime
         self.time_unit = time_unit
+        self.infer_dtype = infer_dtype
 
     def encode(self, variable: Variable, name: T_Name = None) -> Variable:
         if np.issubdtype(variable.dtype, np.datetime64) or contains_cftime_datetimes(
@@ -1411,9 +1482,14 @@ def decode(self, variable: Variable, name: T_Name = None) -> Variable:
 
             units = pop_to(attrs, encoding, "units")
             calendar = pop_to(attrs, encoding, "calendar")
-            dtype = _decode_cf_datetime_dtype(
-                data, units, calendar, self.use_cftime, self.time_unit
-            )
+            if self.infer_dtype == "metadata":
+                dtype = _decode_cf_datetime_dtype_from_metadata(
+                    units, calendar, self.use_cftime, self.time_unit
+                )
+            else:
+                dtype = _decode_cf_datetime_dtype(
+                    data, units, calendar, self.use_cftime, self.time_unit
+                )
             transform = partial(
                 decode_cf_datetime,
                 units=units,

xarray/tests/test_coding_times.py

@@ -46,6 +46,8 @@
     _STANDARD_CALENDARS,
     DuckArrayWrapper,
     FirstElementAccessibleArray,
+    InaccessibleArray,
+    UnexpectedDataAccess,
     _all_cftime_date_types,
     arm_xfail,
     assert_array_equal,
@@ -2217,3 +2219,74 @@ def test_roundtrip_empty_datetime64_array(time_unit: PDDatetimeUnitOptions) -> N
     )
     assert_identical(variable, roundtripped)
     assert roundtripped.dtype == variable.dtype
+
+
+class TestInferDtypeOptIn:
+    """Coverage for the ``infer_dtype="metadata"`` opt-in on CFDatetimeCoder.
+
+    With ``infer_dtype="data"`` (default), the coder reads the first and
+    last values of the encoded array to verify they decode successfully —
+    this is the legacy behavior covered elsewhere in this module.
+
+    With ``infer_dtype="metadata"``, dtype is inferred from
+    ``(units, calendar, use_cftime, time_unit)`` alone, with no data
+    reads. The tests below exercise the metadata path and prove no data
+    access happens by wrapping the variable in ``InaccessibleArray``.
+    """
+
+    @staticmethod
+    def _inaccessible_variable(
+        units: str,
+        calendar: str | None = None,
+        dtype: str = "int64",
+        shape: tuple[int, ...] = (3,),
+    ) -> Variable:
+        attrs: dict[str, str] = {"units": units}
+        if calendar is not None:
+            attrs["calendar"] = calendar
+        return Variable(
+            ("t",), InaccessibleArray(np.zeros(shape, dtype=dtype)), attrs=attrs
+        )
+
+    def test_metadata_use_cftime_true_no_reads(self):
+        # ``use_cftime=True`` short-circuits to object dtype without
+        # calling decode_cf_datetime, so it works even when cftime is
+        # not installed.
+        variable = self._inaccessible_variable("days since 2000-01-01", "standard")
+        coder = CFDatetimeCoder(use_cftime=True, infer_dtype="metadata")
+        assert coder.decode(variable).dtype == np.dtype("object")
+
+    @requires_cftime
+    def test_metadata_nonstandard_calendar_no_reads(self):
+        variable = self._inaccessible_variable("days since 2000-01-01", "noleap")
+        coder = CFDatetimeCoder(infer_dtype="metadata")
+        assert coder.decode(variable).dtype == np.dtype("object")
+
+    @pytest.mark.parametrize("time_unit", ["s", "ms", "us", "ns"])
+    def test_metadata_standard_calendar_no_reads(self, time_unit):
+        variable = self._inaccessible_variable("days since 2000-01-01", "standard")
+        coder = CFDatetimeCoder(time_unit=time_unit, infer_dtype="metadata")
+        assert coder.decode(variable).dtype.kind == "M"
+
+    def test_metadata_invalid_units_raises_value_error(self):
+        # "nonsense" is not a valid unit but contains "since", so the
+        # coder enters the CF-datetime branch and the metadata path
+        # surfaces the parse error.
+        variable = self._inaccessible_variable("nonsense since whenever", "standard")
+        coder = CFDatetimeCoder(infer_dtype="metadata")
+        with pytest.raises(ValueError, match="unable to decode time units"):
+            coder.decode(variable)
+
+    def test_invalid_infer_dtype_raises(self):
+        with pytest.raises(
+            ValueError, match="infer_dtype must be 'data' or 'metadata'"
+        ):
+            CFDatetimeCoder(infer_dtype="bogus")  # type: ignore[arg-type]
+
+    def test_data_default_still_reads_first_and_last(self):
+        # Backwards-compat guarantee: the default path inspects the
+        # data, so wrapping in InaccessibleArray raises.
+        variable = self._inaccessible_variable("days since 2000-01-01", "standard")
+        coder = CFDatetimeCoder()
+        with pytest.raises(UnexpectedDataAccess):
+            coder.decode(variable)