diff --git a/changes/4506.bugfix.md b/changes/4506.bugfix.md new file mode 100644 index 0000000000..3a308b2049 --- /dev/null +++ b/changes/4506.bugfix.md @@ -0,0 +1,9 @@ +Array metadata that uses Zarr V3.1 short-hand extension names, such as +`"codecs": ["bytes", "crc32c"]` or `"chunk_key_encoding": "default"`, can now be +read. A bare name is read as an object with only a `name` key. Previously, a +short-hand codec or chunk key encoding raised +`TypeError: Expected dict, got `, and a short-hand storage +transformer was kept as a bare string. Metadata is still written in the object +form, which Zarr V3.0 readers require. The `chunk_key_encoding` parameter of +`create_array` and the other array creation functions also accepts a bare name, +such as `chunk_key_encoding="v2"`. diff --git a/src/zarr/core/chunk_key_encodings.py b/src/zarr/core/chunk_key_encodings.py index d871e279d2..9ca12f9a21 100644 --- a/src/zarr/core/chunk_key_encodings.py +++ b/src/zarr/core/chunk_key_encodings.py @@ -11,6 +11,7 @@ from zarr.core.common import ( JSON, NamedConfig, + expand_short_hand_name, parse_named_configuration, ) from zarr.core.json_parse import parse_field @@ -62,7 +63,7 @@ def encode_chunk_key(self, chunk_coords: tuple[int, ...]) -> str: type ChunkKeyEncodingLike = ( - dict[str, JSON] | ChunkKeyEncodingParams | ChunkKeyEncoding | NamedConfig[str, Any] + str | dict[str, JSON] | ChunkKeyEncodingParams | ChunkKeyEncoding | NamedConfig[str, Any] ) @@ -112,6 +113,8 @@ def parse_chunk_key_encoding(data: ChunkKeyEncodingLike) -> ChunkKeyEncoding: if isinstance(data, ChunkKeyEncoding): return data + data = expand_short_hand_name(data) + # handle ChunkKeyEncodingParams if "name" in data and "separator" in data: data = {"name": data["name"], "configuration": {"separator": data["separator"]}} # type: ignore[typeddict-item] diff --git a/src/zarr/core/common.py b/src/zarr/core/common.py index 40046124ec..47f79b2e72 100644 --- a/src/zarr/core/common.py +++ b/src/zarr/core/common.py @@ -220,6 +220,27 @@ def parse_name(data: JSON, expected: str | None = None) -> str: raise ValueError(f"Expected '{expected}'. Got {data} instead.") +@overload +def expand_short_hand_name(data: str) -> dict[str, JSON]: ... + + +@overload +def expand_short_hand_name[T](data: T) -> T: ... + + +def expand_short_hand_name(data: object) -> object: + """ + Expand the short-hand name form of a Zarr V3 extension definition: a bare string `name` + is equivalent to the object `{"name": name}`. Any other value is returned unchanged. + + Zarr V3.1 permits the short-hand form at every extension point of array metadata. See + https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#short-hand-names + """ + if isinstance(data, str): + return {"name": data} + return data + + def parse_configuration(data: JSON) -> JSON: if not isinstance(data, dict): raise TypeError(f"Expected dict, got {type(data)}") diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py index 2d5d169e6b..63d525ba2d 100644 --- a/src/zarr/core/metadata/v3.py +++ b/src/zarr/core/metadata/v3.py @@ -28,6 +28,7 @@ NodeType, compress_rle, expand_rle, + expand_short_hand_name, parse_chunk_edge, parse_chunk_shape, parse_named_configuration, @@ -82,6 +83,7 @@ def parse_codecs(data: object) -> tuple[Codec, ...]: ): # Can't use Codec here because of mypy limitation out += (c,) else: + c = expand_short_hand_name(c) name_parsed, _ = parse_named_configuration(c, require_configuration=False) codec_cls = get_codec_class(name_parsed) @@ -156,7 +158,7 @@ def parse_storage_transformers(data: object) -> tuple[dict[str, JSON], ...]: # and then returned ``data`` itself, which exhausted (and discarded) a # one-shot iterable and could return a value typed as a tuple that was not # actually a tuple. - return tuple(data) + return tuple(expand_short_hand_name(t) for t in data) raise TypeError( f"Invalid storage_transformers. Expected an iterable of dicts. Got {type(data)} instead." ) @@ -458,6 +460,7 @@ def parse_chunk_grid( if isinstance(data, (RegularChunkGridMetadata, RectilinearChunkGridMetadata)): return data + data = expand_short_hand_name(data) name, _ = parse_named_configuration(data) if name == "regular": return RegularChunkGridMetadata.from_dict(data) # type: ignore[arg-type] @@ -712,7 +715,7 @@ def from_dict(cls, data: dict[str, JSON], *, path: str | None = None) -> Self: metadata = cls( shape=_data_typed["shape"], chunk_grid=_data_typed["chunk_grid"], # type: ignore[arg-type] - chunk_key_encoding=_data_typed["chunk_key_encoding"], # type: ignore[arg-type] + chunk_key_encoding=_data_typed["chunk_key_encoding"], codecs=_data_typed["codecs"], # Attribute values are arbitrary JSON, so they have no field-specific # schema to validate. `__init__` checks the outer dict via `parse_attributes`. diff --git a/tests/test_chunk_key_encodings.py b/tests/test_chunk_key_encodings.py index dcc93b9249..70b4530f24 100644 --- a/tests/test_chunk_key_encodings.py +++ b/tests/test_chunk_key_encodings.py @@ -1,8 +1,15 @@ from __future__ import annotations +from typing import TYPE_CHECKING + import pytest +import zarr from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding, V2ChunkKeyEncoding +from zarr.storage import MemoryStore + +if TYPE_CHECKING: + from zarr.core.common import ZarrFormat @pytest.mark.parametrize("separator", ["/", "."]) @@ -63,3 +70,32 @@ def test_default_encoding_rejects_key_using_the_other_separator() -> None: with pytest.raises(ValueError, match="Invalid chunk key"): encoding.decode_chunk_key("c.0.1") + + +@pytest.mark.parametrize( + ("name", "zarr_format", "chunk_keys"), + [ + ("default", 3, ["c/0/0", "c/0/1"]), + ("v2", 3, ["0.0", "0.1"]), + ("v2", 2, ["0.0", "0.1"]), + ], +) +async def test_create_array_short_hand_chunk_key_encoding( + name: str, zarr_format: ZarrFormat, chunk_keys: list[str] +) -> None: + """A bare chunk key encoding name given at array creation selects that encoding with + its default separator, as it does when read from Zarr V3.1 short-hand metadata.""" + store = MemoryStore() + array = zarr.create_array( + store, + shape=(1, 2), + chunks=(1, 1), + dtype="uint8", + fill_value=0, + chunk_key_encoding=name, + zarr_format=zarr_format, + ) + array[:] = 1 + metadata_keys = {"zarr.json", ".zarray", ".zattrs"} + keys = sorted([key async for key in store.list() if key not in metadata_keys]) + assert keys == chunk_keys diff --git a/tests/test_metadata/test_v3.py b/tests/test_metadata/test_v3.py index 94240c36b9..5872a638fc 100644 --- a/tests/test_metadata/test_v3.py +++ b/tests/test_metadata/test_v3.py @@ -260,6 +260,61 @@ def test_array_metadata_keys_matches_typeddict() -> None: ), id="extra_fields", ), + # Zarr V3.1 short-hand names: a bare string is read as `{"name": }`, and + # written back in the object form that Zarr V3.0 readers require. + Expect( + input={"codecs": ["bytes"]}, + output=minimal_metadata_dict_v3(codecs=_UINT8_CODECS), + id="short_hand_codec", + ), + Expect( + input={"codecs": [{"name": "bytes"}, "crc32c"]}, + output=minimal_metadata_dict_v3(codecs=(*_UINT8_CODECS, {"name": "crc32c"})), + id="short_hand_codec_mixed_with_object", + ), + Expect( + input={ + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [2, 2], + "codecs": ["bytes"], + "index_codecs": [*_FLOAT64_CODECS, "crc32c"], + }, + } + ] + }, + output=minimal_metadata_dict_v3( + codecs=( + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": (2, 2), + "codecs": _UINT8_CODECS, + "index_codecs": (*_FLOAT64_CODECS, {"name": "crc32c"}), + "index_location": "end", + }, + }, + ) + ), + id="short_hand_codecs_nested_in_sharding", + ), + Expect( + input={"chunk_key_encoding": "v2"}, + output=minimal_metadata_dict_v3( + chunk_key_encoding={"name": "v2", "configuration": {"separator": "."}}, + codecs=_UINT8_CODECS, + ), + id="short_hand_chunk_key_encoding", + ), + Expect( + input={"storage_transformers": ["foo"]}, + output=minimal_metadata_dict_v3( + storage_transformers=({"name": "foo"},), codecs=_UINT8_CODECS + ), + id="short_hand_storage_transformer", + ), ], ids=lambda case: case.id, ) @@ -299,6 +354,33 @@ def test_array_metadata_from_dict_fails(case: ExpectFail[dict[str, Any]]) -> Non ArrayV3Metadata.from_dict(d) # type: ignore[arg-type] +@pytest.mark.parametrize( + "overrides", + [ + {"chunk_grid": "regular"}, + {"codecs": ["bytes", "gzip"]}, + {"codecs": ["sharding_indexed"]}, + ], + ids=["chunk_grid", "codec", "sharding_codec"], +) +def test_short_hand_name_requiring_configuration_rejected(overrides: dict[str, Any]) -> None: + """A short-hand name is only valid for an extension that needs no configuration.""" + with pytest.raises(ValueError, match="does not have a 'configuration' key"): + ArrayV3Metadata.from_dict(minimal_metadata_dict_v3(**overrides)) # type: ignore[arg-type] + + +def test_short_hand_codec_name_unknown_rejected() -> None: + """A short-hand codec name with no registered implementation is rejected by name.""" + with pytest.raises(UnknownCodecError, match="'not-a-codec'"): + ArrayV3Metadata.from_dict(minimal_metadata_dict_v3(codecs=["bytes", "not-a-codec"])) # type: ignore[arg-type] + + +def test_short_hand_chunk_key_encoding_name_unknown_rejected() -> None: + """A short-hand chunk key encoding name with no registered implementation is rejected.""" + with pytest.raises(ValueError, match="'not-an-encoding' not found"): + ArrayV3Metadata.from_dict(minimal_metadata_dict_v3(chunk_key_encoding="not-an-encoding")) # type: ignore[arg-type] + + @pytest.mark.parametrize( "case", [