fix(metadata): read Zarr V3.1 short-hand extension names

changes/4506.bugfix.md

@@ -0,0 +1,9 @@
+Array metadata that uses Zarr V3.1 short-hand extension names, such as
+`"codecs": ["bytes", "crc32c"]` or `"chunk_key_encoding": "default"`, can now be
+read. A bare name is read as an object with only a `name` key. Previously, a
+short-hand codec or chunk key encoding raised
+`TypeError: Expected dict, got <class 'str'>`, and a short-hand storage
+transformer was kept as a bare string. Metadata is still written in the object
+form, which Zarr V3.0 readers require. The `chunk_key_encoding` parameter of
+`create_array` and the other array creation functions also accepts a bare name,
+such as `chunk_key_encoding="v2"`.

src/zarr/core/chunk_key_encodings.py

@@ -11,6 +11,7 @@
 from zarr.core.common import (
     JSON,
     NamedConfig,
+    expand_short_hand_name,
     parse_named_configuration,
 )
 from zarr.core.json_parse import parse_field
@@ -62,7 +63,7 @@ def encode_chunk_key(self, chunk_coords: tuple[int, ...]) -> str:
 
 
 type ChunkKeyEncodingLike = (
-    dict[str, JSON] | ChunkKeyEncodingParams | ChunkKeyEncoding | NamedConfig[str, Any]
+    str | dict[str, JSON] | ChunkKeyEncodingParams | ChunkKeyEncoding | NamedConfig[str, Any]
 )
 
 
@@ -112,6 +113,8 @@ def parse_chunk_key_encoding(data: ChunkKeyEncodingLike) -> ChunkKeyEncoding:
     if isinstance(data, ChunkKeyEncoding):
         return data
 
+    data = expand_short_hand_name(data)
+
     # handle ChunkKeyEncodingParams
     if "name" in data and "separator" in data:
         data = {"name": data["name"], "configuration": {"separator": data["separator"]}}  # type: ignore[typeddict-item]

src/zarr/core/common.py

@@ -220,6 +220,27 @@ def parse_name(data: JSON, expected: str | None = None) -> str:
     raise ValueError(f"Expected '{expected}'. Got {data} instead.")
 
 
+@overload
+def expand_short_hand_name(data: str) -> dict[str, JSON]: ...
+
+
+@overload
+def expand_short_hand_name[T](data: T) -> T: ...
+
+
+def expand_short_hand_name(data: object) -> object:
+    """
+    Expand the short-hand name form of a Zarr V3 extension definition: a bare string `name`
+    is equivalent to the object `{"name": name}`. Any other value is returned unchanged.
+
+    Zarr V3.1 permits the short-hand form at every extension point of array metadata. See
+    https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#short-hand-names
+    """
+    if isinstance(data, str):
+        return {"name": data}
+    return data
+
+
 def parse_configuration(data: JSON) -> JSON:
     if not isinstance(data, dict):
         raise TypeError(f"Expected dict, got {type(data)}")

src/zarr/core/metadata/v3.py

@@ -28,6 +28,7 @@
     NodeType,
     compress_rle,
     expand_rle,
+    expand_short_hand_name,
     parse_chunk_edge,
     parse_chunk_shape,
     parse_named_configuration,
@@ -82,6 +83,7 @@ def parse_codecs(data: object) -> tuple[Codec, ...]:
         ):  # Can't use Codec here because of mypy limitation
             out += (c,)
         else:
+            c = expand_short_hand_name(c)
             name_parsed, _ = parse_named_configuration(c, require_configuration=False)
 
             codec_cls = get_codec_class(name_parsed)
@@ -156,7 +158,7 @@ def parse_storage_transformers(data: object) -> tuple[dict[str, JSON], ...]:
         # and then returned ``data`` itself, which exhausted (and discarded) a
         # one-shot iterable and could return a value typed as a tuple that was not
         # actually a tuple.
-        return tuple(data)
+        return tuple(expand_short_hand_name(t) for t in data)
     raise TypeError(
         f"Invalid storage_transformers. Expected an iterable of dicts. Got {type(data)} instead."
     )
@@ -458,6 +460,7 @@ def parse_chunk_grid(
     if isinstance(data, (RegularChunkGridMetadata, RectilinearChunkGridMetadata)):
         return data
 
+    data = expand_short_hand_name(data)
     name, _ = parse_named_configuration(data)
     if name == "regular":
         return RegularChunkGridMetadata.from_dict(data)  # type: ignore[arg-type]
@@ -712,7 +715,7 @@ def from_dict(cls, data: dict[str, JSON], *, path: str | None = None) -> Self:
         metadata = cls(
             shape=_data_typed["shape"],
             chunk_grid=_data_typed["chunk_grid"],  # type: ignore[arg-type]
-            chunk_key_encoding=_data_typed["chunk_key_encoding"],  # type: ignore[arg-type]
+            chunk_key_encoding=_data_typed["chunk_key_encoding"],
             codecs=_data_typed["codecs"],
             # Attribute values are arbitrary JSON, so they have no field-specific
             # schema to validate. `__init__` checks the outer dict via `parse_attributes`.

tests/test_chunk_key_encodings.py

@@ -1,8 +1,15 @@
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
 import pytest
 
+import zarr
 from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding, V2ChunkKeyEncoding
+from zarr.storage import MemoryStore
+
+if TYPE_CHECKING:
+    from zarr.core.common import ZarrFormat
 
 
 @pytest.mark.parametrize("separator", ["/", "."])
@@ -63,3 +70,32 @@ def test_default_encoding_rejects_key_using_the_other_separator() -> None:
 
     with pytest.raises(ValueError, match="Invalid chunk key"):
         encoding.decode_chunk_key("c.0.1")
+
+
+@pytest.mark.parametrize(
+    ("name", "zarr_format", "chunk_keys"),
+    [
+        ("default", 3, ["c/0/0", "c/0/1"]),
+        ("v2", 3, ["0.0", "0.1"]),
+        ("v2", 2, ["0.0", "0.1"]),
+    ],
+)
+async def test_create_array_short_hand_chunk_key_encoding(
+    name: str, zarr_format: ZarrFormat, chunk_keys: list[str]
+) -> None:
+    """A bare chunk key encoding name given at array creation selects that encoding with
+    its default separator, as it does when read from Zarr V3.1 short-hand metadata."""
+    store = MemoryStore()
+    array = zarr.create_array(
+        store,
+        shape=(1, 2),
+        chunks=(1, 1),
+        dtype="uint8",
+        fill_value=0,
+        chunk_key_encoding=name,
+        zarr_format=zarr_format,
+    )
+    array[:] = 1
+    metadata_keys = {"zarr.json", ".zarray", ".zattrs"}
+    keys = sorted([key async for key in store.list() if key not in metadata_keys])
+    assert keys == chunk_keys

tests/test_metadata/test_v3.py

@@ -260,6 +260,61 @@ def test_array_metadata_keys_matches_typeddict() -> None:
             ),
             id="extra_fields",
         ),
+        # Zarr V3.1 short-hand names: a bare string is read as `{"name": <string>}`, and
+        # written back in the object form that Zarr V3.0 readers require.
+        Expect(
+            input={"codecs": ["bytes"]},
+            output=minimal_metadata_dict_v3(codecs=_UINT8_CODECS),
+            id="short_hand_codec",
+        ),
+        Expect(
+            input={"codecs": [{"name": "bytes"}, "crc32c"]},
+            output=minimal_metadata_dict_v3(codecs=(*_UINT8_CODECS, {"name": "crc32c"})),
+            id="short_hand_codec_mixed_with_object",
+        ),
+        Expect(
+            input={
+                "codecs": [
+                    {
+                        "name": "sharding_indexed",
+                        "configuration": {
+                            "chunk_shape": [2, 2],
+                            "codecs": ["bytes"],
+                            "index_codecs": [*_FLOAT64_CODECS, "crc32c"],
+                        },
+                    }
+                ]
+            },
+            output=minimal_metadata_dict_v3(
+                codecs=(
+                    {
+                        "name": "sharding_indexed",
+                        "configuration": {
+                            "chunk_shape": (2, 2),
+                            "codecs": _UINT8_CODECS,
+                            "index_codecs": (*_FLOAT64_CODECS, {"name": "crc32c"}),
+                            "index_location": "end",
+                        },
+                    },
+                )
+            ),
+            id="short_hand_codecs_nested_in_sharding",
+        ),
+        Expect(
+            input={"chunk_key_encoding": "v2"},
+            output=minimal_metadata_dict_v3(
+                chunk_key_encoding={"name": "v2", "configuration": {"separator": "."}},
+                codecs=_UINT8_CODECS,
+            ),
+            id="short_hand_chunk_key_encoding",
+        ),
+        Expect(
+            input={"storage_transformers": ["foo"]},
+            output=minimal_metadata_dict_v3(
+                storage_transformers=({"name": "foo"},), codecs=_UINT8_CODECS
+            ),
+            id="short_hand_storage_transformer",
+        ),
     ],
     ids=lambda case: case.id,
 )
@@ -299,6 +354,33 @@ def test_array_metadata_from_dict_fails(case: ExpectFail[dict[str, Any]]) -> Non
         ArrayV3Metadata.from_dict(d)  # type: ignore[arg-type]
 
 
+@pytest.mark.parametrize(
+    "overrides",
+    [
+        {"chunk_grid": "regular"},
+        {"codecs": ["bytes", "gzip"]},
+        {"codecs": ["sharding_indexed"]},
+    ],
+    ids=["chunk_grid", "codec", "sharding_codec"],
+)
+def test_short_hand_name_requiring_configuration_rejected(overrides: dict[str, Any]) -> None:
+    """A short-hand name is only valid for an extension that needs no configuration."""
+    with pytest.raises(ValueError, match="does not have a 'configuration' key"):
+        ArrayV3Metadata.from_dict(minimal_metadata_dict_v3(**overrides))  # type: ignore[arg-type]
+
+
+def test_short_hand_codec_name_unknown_rejected() -> None:
+    """A short-hand codec name with no registered implementation is rejected by name."""
+    with pytest.raises(UnknownCodecError, match="'not-a-codec'"):
+        ArrayV3Metadata.from_dict(minimal_metadata_dict_v3(codecs=["bytes", "not-a-codec"]))  # type: ignore[arg-type]
+
+
+def test_short_hand_chunk_key_encoding_name_unknown_rejected() -> None:
+    """A short-hand chunk key encoding name with no registered implementation is rejected."""
+    with pytest.raises(ValueError, match="'not-an-encoding' not found"):
+        ArrayV3Metadata.from_dict(minimal_metadata_dict_v3(chunk_key_encoding="not-an-encoding"))  # type: ignore[arg-type]
+
+
 @pytest.mark.parametrize(
     "case",
     [