From b46615fd61053900eab7c6ab881d29502b1dbc30 Mon Sep 17 00:00:00 2001 From: inchang-ing <197932532+inchang-ing@users.noreply.github.com> Date: Sun, 18 Oct 2026 18:23:41 +0800 Subject: [PATCH 1/4] fix: tolerate unencodable default_scalar in ShardingCodec.validate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ShardingCodec._validate_inner_codecs (and ArrayV3Metadata codec threading) probe the codec chain with dtype.default_scalar() as the fill value. When the chain maps only a subrange of the dtype — e.g. ScaleOffset + CastValue scaling float32 values in [100, 200] to uint16 — the probe value itself is unencodable, and array creation failed even though no real value ever needs that encoding. Make the fill-value threading in ScaleOffset.resolve_metadata and CastValue.resolve_metadata best-effort: if encoding the fill value raises a value-domain error (ValueError/OverflowError), carry the fill value forward unchanged instead of failing. Shape/dtype threading and per-codec validate() are untouched, so structurally invalid configurations (wrong dtype for a codec, scale=0, out_of_range='wrap' with a float target, ...) still raise at creation time. Genuine value-domain errors for actual chunk data still surface in _encode_sync. Fixes #4511 --- src/zarr/codecs/cast_value.py | 18 ++++++--- src/zarr/codecs/scale_offset.py | 13 ++++++- tests/test_codecs/test_cast_value.py | 54 +++++++++++++++++++++++++- tests/test_codecs/test_scale_offset.py | 42 ++++++++++++++++++++ tests/test_codecs/test_sharding.py | 29 ++++++++++++++ 5 files changed, 148 insertions(+), 8 deletions(-) diff --git a/src/zarr/codecs/cast_value.py b/src/zarr/codecs/cast_value.py index 629b4e7a27..e69e362f62 100644 --- a/src/zarr/codecs/cast_value.py +++ b/src/zarr/codecs/cast_value.py @@ -371,11 +371,19 @@ def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: fill = chunk_spec.fill_value fill_arr = np.array([fill], dtype=source_native) - - new_fill_arr = self._do_cast( - fill_arr, target_dtype=target_native, scalar_map=self._get_scalar_map("encode") - ) - new_fill = target_native.type(new_fill_arr[0]) + try: + new_fill_arr = self._do_cast( + fill_arr, target_dtype=target_native, scalar_map=self._get_scalar_map("encode") + ) + new_fill = target_native.type(new_fill_arr[0]) + except (ValueError, OverflowError): + # The fill value may be outside the encodable domain of this codec (e.g. a + # source value that cannot be cast to the target dtype without an explicit + # out_of_range policy). That is a property of the fill value, not of the + # codec configuration, so metadata threading (shape/dtype) must not fail + # because of it: carry the fill value forward unchanged. Genuine value + # errors for actual chunk data still surface in `_encode_sync`. + new_fill = fill return replace(chunk_spec, dtype=target_zdtype, fill_value=new_fill) diff --git a/src/zarr/codecs/scale_offset.py b/src/zarr/codecs/scale_offset.py index 2ec6015b6a..e1d00b0105 100644 --- a/src/zarr/codecs/scale_offset.py +++ b/src/zarr/codecs/scale_offset.py @@ -408,8 +408,17 @@ def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: fill = np.asarray(zdtype.cast_scalar(chunk_spec.fill_value)) offset = cast("np.generic", zdtype.from_json_scalar(self.offset, zarr_format=3)) scale = cast("np.generic", zdtype.from_json_scalar(self.scale, zarr_format=3)) - new_fill = _encode(fill, offset, scale) - return replace(chunk_spec, fill_value=new_fill.reshape(()).item()) + try: + new_fill = _encode(fill, offset, scale).reshape(()).item() + except (ValueError, OverflowError): + # The fill value may lie outside the encodable domain of this codec (e.g. a + # scale/offset pair mapping only a subrange of the dtype, with the fill value + # outside that subrange). That is a property of the fill value, not of the + # codec configuration, so metadata threading (shape/dtype) must not fail + # because of it: carry the fill value forward unchanged. Genuine value + # errors for actual chunk data still surface in `_encode_sync`. + new_fill = chunk_spec.fill_value + return replace(chunk_spec, fill_value=new_fill) def _decode_sync( self, diff --git a/tests/test_codecs/test_cast_value.py b/tests/test_codecs/test_cast_value.py index c2e78770d9..2c19fc8334 100644 --- a/tests/test_codecs/test_cast_value.py +++ b/tests/test_codecs/test_cast_value.py @@ -8,7 +8,7 @@ import zarr from tests.conftest import Expect, ExpectFail -from zarr.codecs import BytesCodec, TransposeCodec +from zarr.codecs import BytesCodec, ScaleOffset, TransposeCodec from zarr.codecs.cast_value import CastValue from zarr.storage import MemoryStore @@ -611,3 +611,55 @@ def make_array(filters: list[Any]) -> Any: array[:] = data assert_array_equal(array[:], data) + + +# --------------------------------------------------------------------------- +# Fill-value tolerance in resolve_metadata (issue #4511) +# --------------------------------------------------------------------------- + + +@requires_cast_value_rs +def test_creation_with_unencodable_default_fill_value() -> None: + """A ScaleOffset+CastValue chain whose valid input range excludes the default + fill value (e.g. float32 values known to lie in [100, 200], scaled to uint16) + must not fail array creation: the fill value is metadata, not chunk data, and + is never required to pass through the codec chain + (https://github.com/zarr-developers/zarr-python/issues/4511). + """ + minimum, maximum = 100.0, 200.0 + arr = zarr.create_array( + store=MemoryStore(), + shape=(4,), + dtype="float32", + chunks=(4,), + filters=[ + ScaleOffset(scale=65535 / (maximum - minimum), offset=minimum), + CastValue(data_type="uint16", rounding="nearest-even"), + ], + compressors=None, + ) + # Full-chunk writes contain no fill positions, so they encode cleanly. + data = np.array([120.0, 180.0, 100.0, 200.0], dtype="float32") + arr[:] = data + # Quantization to uint16 introduces a small rounding error. + np.testing.assert_allclose(arr[:], data, atol=1e-2) + + +@requires_cast_value_rs +def test_unencodable_fill_value_does_not_mask_invalid_config() -> None: + """Tolerating an unencodable fill value must not weaken validation: a codec + configuration that is structurally incompatible with the dtype still raises + at array creation time. + """ + with pytest.raises(ValueError, match="only supports integer and floating-point"): + zarr.create_array( + store=MemoryStore(), + shape=(4,), + dtype="str", + chunks=(4,), + filters=[ + ScaleOffset(scale=1, offset=100), + CastValue(data_type="uint16", rounding="nearest-even"), + ], + compressors=None, + ) diff --git a/tests/test_codecs/test_scale_offset.py b/tests/test_codecs/test_scale_offset.py index 0100d9fe80..b81ae004cc 100644 --- a/tests/test_codecs/test_scale_offset.py +++ b/tests/test_codecs/test_scale_offset.py @@ -488,3 +488,45 @@ def test_compute_encoded_size() -> None: # The chunk_spec argument is unused; pass any sentinel assert codec.compute_encoded_size(input_byte_length=100, _chunk_spec=None) == 100 # type: ignore[arg-type] assert codec.compute_encoded_size(input_byte_length=0, _chunk_spec=None) == 0 # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- +# Fill-value tolerance in resolve_metadata (issue #4511) +# --------------------------------------------------------------------------- + + +def test_creation_with_unencodable_default_fill_value() -> None: + """A scale/offset pair mapping only a subrange of the dtype makes the default + fill value (0) unencodable. Array creation must still succeed: the fill value + is metadata, not chunk data, and is never required to pass through the codec + chain (https://github.com/zarr-developers/zarr-python/issues/4511). + """ + arr = zarr.create_array( + store=MemoryStore(), + shape=(4,), + dtype="uint16", + chunks=(4,), + filters=[ScaleOffset(offset=100, scale=1)], + compressors=None, + ) + # Full-chunk writes contain no fill positions, so they encode cleanly. + # Encoded values must stay within uint16 after decode adds the offset back. + data = np.array([100, 150, 200, 65435], dtype="uint16") + arr[:] = data + np.testing.assert_array_equal(arr[:], data) + + +def test_unencodable_fill_value_does_not_mask_invalid_config() -> None: + """Tolerating an unencodable fill value must not weaken validation: a codec + configuration that is structurally incompatible with the dtype still raises + at array creation time. + """ + with pytest.raises(ValueError, match="only supports integer and floating-point"): + zarr.create_array( + store=MemoryStore(), + shape=(4,), + dtype="str", + chunks=(4,), + filters=[ScaleOffset(offset=100, scale=1)], + compressors=None, + ) diff --git a/tests/test_codecs/test_sharding.py b/tests/test_codecs/test_sharding.py index 5712c9576d..976a942406 100644 --- a/tests/test_codecs/test_sharding.py +++ b/tests/test_codecs/test_sharding.py @@ -19,6 +19,7 @@ BytesCodec, Crc32cCodec, GzipCodec, + ScaleOffset, ShardingCodec, TransposeCodec, ) @@ -1522,3 +1523,31 @@ def test_open_rejects_stored_nested_sharding_with_indivisible_inner_chunk_shape( doc["codecs"][0]["configuration"]["codecs"][0]["configuration"]["chunk_shape"] = (4, 7) with pytest.raises(ValueError, match="not divisible"): ArrayV3Metadata.from_dict(doc) + + +# --------------------------------------------------------------------------- +# Inner-codec validation tolerance for unencodable fill values (issue #4511) +# --------------------------------------------------------------------------- + + +def test_create_with_unencodable_default_fill_value_inner_codecs() -> None: + """`ShardingCodec._validate_inner_codecs` probes the inner codec chain with + `dtype.default_scalar()` as the fill value. When the chain maps only a + subrange of the dtype (e.g. a scale/offset pair for values in [100, 200]), + that probe value is unencodable — but the configuration is still valid, so + array creation must succeed + (https://github.com/zarr-developers/zarr-python/issues/4511). + """ + arr = zarr.create_array( + store=MemoryStore(), + shape=(16,), + chunks=(4,), + shards=(16,), + dtype="uint16", + filters=[ScaleOffset(offset=100, scale=1)], + ) + # Full-shard writes contain no fill positions, so they encode cleanly. + # Encoded values must stay within uint16 after decode adds the offset back. + data = np.array([100, 150, 200, 65435] * 4, dtype="uint16") + arr[:] = data + np.testing.assert_array_equal(arr[:], data) From eedcc47b3f43cc8aa6f8f5c7c0a9c8684e3ee872 Mon Sep 17 00:00:00 2001 From: inchang-ing <197932532+inchang-ing@users.noreply.github.com> Date: Sun, 18 Oct 2026 18:24:57 +0800 Subject: [PATCH 2/4] docs: add changelog fragment for #4526 --- changes/4526.bugfix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/4526.bugfix.md diff --git a/changes/4526.bugfix.md b/changes/4526.bugfix.md new file mode 100644 index 0000000000..6f2b513d54 --- /dev/null +++ b/changes/4526.bugfix.md @@ -0,0 +1 @@ +Array creation no longer fails when the codec chain cannot encode the array's fill value (which defaults to `dtype.default_scalar()`), e.g. a `ScaleOffset` + `CastValue` chain mapping only a subrange of the dtype. The fill value is now carried through the chain's metadata resolution unchanged if it cannot be encoded; genuine value errors for actual chunk data still surface at write time, and structurally invalid codec configurations are still rejected at creation. From d6a2e71c1ecf182548a28a9a807014510554279f Mon Sep 17 00:00:00 2001 From: inchang-ing <197932532+inchang-ing@users.noreply.github.com> Date: Sun, 18 Oct 2026 18:42:34 +0800 Subject: [PATCH 3/4] fix: wrap array slice in np.asarray to satisfy mypy in cast_value test --- tests/test_codecs/test_cast_value.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_codecs/test_cast_value.py b/tests/test_codecs/test_cast_value.py index 2c19fc8334..22fb38172b 100644 --- a/tests/test_codecs/test_cast_value.py +++ b/tests/test_codecs/test_cast_value.py @@ -642,7 +642,7 @@ def test_creation_with_unencodable_default_fill_value() -> None: data = np.array([120.0, 180.0, 100.0, 200.0], dtype="float32") arr[:] = data # Quantization to uint16 introduces a small rounding error. - np.testing.assert_allclose(arr[:], data, atol=1e-2) + np.testing.assert_allclose(np.asarray(arr[:]), data, atol=1e-2) @requires_cast_value_rs From a88305369139ec21bd82266aa6e2ce66959fd96a Mon Sep 17 00:00:00 2001 From: inchang-ing <197932532+inchang-ing@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:09:22 +0800 Subject: [PATCH 4/4] fix: validate the codec chain against the array's fill value ShardingCodec._validate_inner_codecs probed the inner codec chain with dtype.default_scalar() as the fill value. When the chain maps only a subrange of the dtype (e.g. ScaleOffset + CastValue for float32 values in [100, 200] stored as uint16), the probe value itself was unencodable and array creation failed, even though the array's actual fill value round-trips fine. Thread the array's fill value into Codec.validate (new optional keyword) and use it for the probe. The fill value is what an unwritten chunk returns, so it is the value that must round-trip: if the array's fill value cannot be encoded, creation still fails. Codec subclasses that override validate() accept and ignore the new keyword; callers that have no fill value keep the old behavior. Fixes #4511 Co-Authored-By: Claude --- changes/4526.bugfix.md | 2 +- src/zarr/abc/codec.py | 14 ++++++++++ src/zarr/codecs/cast_value.py | 19 +++++--------- src/zarr/codecs/scale_offset.py | 14 +++------- src/zarr/codecs/sharding.py | 15 ++++++++--- src/zarr/codecs/transpose.py | 1 + src/zarr/core/array.py | 9 +++++++ src/zarr/core/codec_pipeline.py | 6 +++-- src/zarr/core/metadata/v3.py | 14 ++++++++-- tests/test_codecs/test_cast_value.py | 36 +++++++++++++++++++++----- tests/test_codecs/test_scale_offset.py | 30 +++++++++++++++++---- tests/test_codecs/test_sharding.py | 29 +++++++++++++++++---- 12 files changed, 141 insertions(+), 48 deletions(-) diff --git a/changes/4526.bugfix.md b/changes/4526.bugfix.md index 6f2b513d54..98a3281c98 100644 --- a/changes/4526.bugfix.md +++ b/changes/4526.bugfix.md @@ -1 +1 @@ -Array creation no longer fails when the codec chain cannot encode the array's fill value (which defaults to `dtype.default_scalar()`), e.g. a `ScaleOffset` + `CastValue` chain mapping only a subrange of the dtype. The fill value is now carried through the chain's metadata resolution unchanged if it cannot be encoded; genuine value errors for actual chunk data still surface at write time, and structurally invalid codec configurations are still rejected at creation. +Array creation no longer fails when the codec chain cannot encode the data type's default scalar as the fill value, e.g. a `ScaleOffset` + `CastValue` chain mapping only a subrange of the dtype (`ShardingCodec` now probes its inner codec chain with the array's fill value instead of `dtype.default_scalar()`). If the array's fill value itself cannot be round-tripped through the codec chain, creation still fails: an unwritten chunk returns the fill value, so the array has to be readable before any chunks have been written. diff --git a/src/zarr/abc/codec.py b/src/zarr/abc/codec.py index c5e753d120..e565cc478c 100644 --- a/src/zarr/abc/codec.py +++ b/src/zarr/abc/codec.py @@ -168,6 +168,7 @@ def validate( shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGridMetadata, + fill_value: TBaseScalar | None = None, ) -> None: """Validates that the codec configuration is compatible with the array metadata. Raises errors when the codec configuration is not compatible. @@ -180,6 +181,12 @@ def validate( The array data type chunk_grid : ChunkGridMetadata The array chunk grid metadata + fill_value : TBaseScalar, optional + The array fill value. Codecs that probe the codec chain with a fill + value (e.g. ``ShardingCodec``) use this instead of the data type's + default scalar, so that the value an unwritten chunk would return is + the one validated. If ``None``, such codecs fall back to the data + type's default scalar. """ async def _decode_single(self, chunk_data: CO, chunk_spec: ArraySpec) -> CI: @@ -383,6 +390,7 @@ def validate( shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGridMetadata, + fill_value: TBaseScalar | None = None, ) -> None: """Validates that all codec configurations are compatible with the array metadata. Raises errors when a codec configuration is not compatible. @@ -395,6 +403,12 @@ def validate( The array data type chunk_grid : ChunkGridMetadata The array chunk grid metadata + fill_value : TBaseScalar, optional + The array fill value. Codecs that probe the codec chain with a fill + value (e.g. ``ShardingCodec``) use this instead of the data type's + default scalar, so that the value an unwritten chunk would return is + the one validated. If ``None``, such codecs fall back to the data + type's default scalar. """ ... diff --git a/src/zarr/codecs/cast_value.py b/src/zarr/codecs/cast_value.py index e69e362f62..bb3ed6d9b1 100644 --- a/src/zarr/codecs/cast_value.py +++ b/src/zarr/codecs/cast_value.py @@ -284,6 +284,7 @@ def validate( shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGridMetadata, + fill_value: TBaseScalar | None = None, ) -> None: # `dtype` is the source (the array's dtype); `self.dtype` is the # cast target. The spec requires both to be permitted, and rules @@ -371,19 +372,11 @@ def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: fill = chunk_spec.fill_value fill_arr = np.array([fill], dtype=source_native) - try: - new_fill_arr = self._do_cast( - fill_arr, target_dtype=target_native, scalar_map=self._get_scalar_map("encode") - ) - new_fill = target_native.type(new_fill_arr[0]) - except (ValueError, OverflowError): - # The fill value may be outside the encodable domain of this codec (e.g. a - # source value that cannot be cast to the target dtype without an explicit - # out_of_range policy). That is a property of the fill value, not of the - # codec configuration, so metadata threading (shape/dtype) must not fail - # because of it: carry the fill value forward unchanged. Genuine value - # errors for actual chunk data still surface in `_encode_sync`. - new_fill = fill + + new_fill_arr = self._do_cast( + fill_arr, target_dtype=target_native, scalar_map=self._get_scalar_map("encode") + ) + new_fill = target_native.type(new_fill_arr[0]) return replace(chunk_spec, dtype=target_zdtype, fill_value=new_fill) diff --git a/src/zarr/codecs/scale_offset.py b/src/zarr/codecs/scale_offset.py index e1d00b0105..b1ba702ad1 100644 --- a/src/zarr/codecs/scale_offset.py +++ b/src/zarr/codecs/scale_offset.py @@ -383,6 +383,7 @@ def validate( shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGridMetadata, + fill_value: TBaseScalar | None = None, ) -> None: native = dtype.to_native_dtype() if not np.issubdtype(native, np.integer) and not np.issubdtype(native, np.floating): @@ -408,17 +409,8 @@ def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: fill = np.asarray(zdtype.cast_scalar(chunk_spec.fill_value)) offset = cast("np.generic", zdtype.from_json_scalar(self.offset, zarr_format=3)) scale = cast("np.generic", zdtype.from_json_scalar(self.scale, zarr_format=3)) - try: - new_fill = _encode(fill, offset, scale).reshape(()).item() - except (ValueError, OverflowError): - # The fill value may lie outside the encodable domain of this codec (e.g. a - # scale/offset pair mapping only a subrange of the dtype, with the fill value - # outside that subrange). That is a property of the fill value, not of the - # codec configuration, so metadata threading (shape/dtype) must not fail - # because of it: carry the fill value forward unchanged. Genuine value - # errors for actual chunk data still surface in `_encode_sync`. - new_fill = chunk_spec.fill_value - return replace(chunk_spec, fill_value=new_fill) + new_fill = _encode(fill, offset, scale) + return replace(chunk_spec, fill_value=new_fill.reshape(()).item()) def _decode_sync( self, diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index d4ccf71cb9..82e671d258 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -619,6 +619,7 @@ def validate( shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGridMetadata, + fill_value: TBaseScalar | None = None, ) -> None: if len(self.chunk_shape) != len(shape): raise ValueError( @@ -642,18 +643,26 @@ def validate( f"Chunk edge length {edge} in dimension {i} is not " f"divisible by the shard's inner chunk size {inner}." ) - self._validate_inner_codecs(dtype) + self._validate_inner_codecs(dtype, fill_value) - def _validate_inner_codecs(self, dtype: ZDType[TBaseDType, TBaseScalar]) -> None: + def _validate_inner_codecs( + self, dtype: ZDType[TBaseDType, TBaseScalar], fill_value: TBaseScalar | None = None + ) -> None: """Validate the codecs that encode each inner chunk, as array metadata validates its codecs: each against the chunk the codecs before it leave it, a regular grid of one inner chunk. A sharding codec among them is thereby checked against the chunk it splits, which `validate` of the outer codec alone does not see. + + The codec chain is probed with the array's fill value, because that is the + value an unwritten chunk must return: the array has to be readable before any + chunks have been written. A fill value that cannot be round-tripped through + the inner codecs is therefore a genuine error and is raised here. If no fill + value is provided, the data type's default scalar is used. """ spec = ArraySpec( shape=self.chunk_shape, dtype=dtype, - fill_value=dtype.default_scalar(), + fill_value=dtype.default_scalar() if fill_value is None else fill_value, config=ArrayConfig.from_dict({}), prototype=default_buffer_prototype(), ) diff --git a/src/zarr/codecs/transpose.py b/src/zarr/codecs/transpose.py index 5756fba2b4..d885cfdfc1 100644 --- a/src/zarr/codecs/transpose.py +++ b/src/zarr/codecs/transpose.py @@ -52,6 +52,7 @@ def validate( shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGridMetadata, + fill_value: TBaseScalar | None = None, ) -> None: if len(self.order) != len(shape): raise ValueError( diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 06f7338755..f59207ca80 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -4701,10 +4701,19 @@ async def init_array( sharding_codec = ShardingCodec( chunk_shape=inner_chunks_flat, codecs=sub_codecs, index_location=index_location ) + # Probe the inner codec chain with the array's fill value, since that + # is the value an unwritten chunk must return. Resolve the sentinel + # the same way `_create_metadata_v3` does, so validation and metadata + # agree on the fill value. + if fill_value is DEFAULT_FILL_VALUE or fill_value is None: + fill_value_probe: Any = zdtype.default_scalar() + else: + fill_value_probe = fill_value sharding_codec.validate( shape=inner_chunks_flat, dtype=zdtype, chunk_grid=grid, + fill_value=fill_value_probe, ) codecs_out = (sharding_codec,) else: diff --git a/src/zarr/core/codec_pipeline.py b/src/zarr/core/codec_pipeline.py index dd893ea8d8..ee56249cc0 100644 --- a/src/zarr/core/codec_pipeline.py +++ b/src/zarr/core/codec_pipeline.py @@ -657,9 +657,10 @@ def validate( shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGridMetadata, + fill_value: TBaseScalar | None = None, ) -> None: for codec in self: - codec.validate(shape=shape, dtype=dtype, chunk_grid=chunk_grid) + codec.validate(shape=shape, dtype=dtype, chunk_grid=chunk_grid, fill_value=fill_value) def compute_encoded_size(self, byte_length: int, array_spec: ArraySpec) -> int: for codec in self: @@ -1068,9 +1069,10 @@ def validate( shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGridMetadata, + fill_value: TBaseScalar | None = None, ) -> None: for codec in self.codecs: - codec.validate(shape=shape, dtype=dtype, chunk_grid=chunk_grid) + codec.validate(shape=shape, dtype=dtype, chunk_grid=chunk_grid, fill_value=fill_value) def compute_encoded_size(self, byte_length: int, array_spec: ArraySpec) -> int: for codec in self: diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py index 2d5d169e6b..b7fb7de547 100644 --- a/src/zarr/core/metadata/v3.py +++ b/src/zarr/core/metadata/v3.py @@ -571,7 +571,12 @@ def __init__( spec = array_spec for c in codecs_parsed_partial: evolved_codec = c.evolve_from_array_spec(spec) - evolved_codec.validate(shape=spec.shape, dtype=spec.dtype, chunk_grid=chunk_grid_parsed) + evolved_codec.validate( + shape=spec.shape, + dtype=spec.dtype, + chunk_grid=chunk_grid_parsed, + fill_value=spec.fill_value, + ) evolved.append(evolved_codec) spec = evolved_codec.resolve_metadata(spec) codecs_parsed = tuple(evolved) @@ -602,7 +607,12 @@ def _validate_metadata(self) -> None: if self.fill_value is None: raise ValueError("`fill_value` is required.") for codec in self.codecs: - codec.validate(shape=self.shape, dtype=self.data_type, chunk_grid=self.chunk_grid) + codec.validate( + shape=self.shape, + dtype=self.data_type, + chunk_grid=self.chunk_grid, + fill_value=self.fill_value, + ) @property def ndim(self) -> int: diff --git a/tests/test_codecs/test_cast_value.py b/tests/test_codecs/test_cast_value.py index 22fb38172b..c20f22c593 100644 --- a/tests/test_codecs/test_cast_value.py +++ b/tests/test_codecs/test_cast_value.py @@ -619,18 +619,19 @@ def make_array(filters: list[Any]) -> Any: @requires_cast_value_rs -def test_creation_with_unencodable_default_fill_value() -> None: - """A ScaleOffset+CastValue chain whose valid input range excludes the default - fill value (e.g. float32 values known to lie in [100, 200], scaled to uint16) - must not fail array creation: the fill value is metadata, not chunk data, and - is never required to pass through the codec chain - (https://github.com/zarr-developers/zarr-python/issues/4511). +def test_creation_with_encodable_fill_value() -> None: + """A ScaleOffset+CastValue chain whose valid input range excludes the dtype's + default scalar (e.g. float32 values known to lie in [100, 200], scaled to + uint16) must not fail array creation when the array's fill value lies inside + that range: the fill value is the one probed against the codec chain, and it + round-trips (https://github.com/zarr-developers/zarr-python/issues/4511). """ minimum, maximum = 100.0, 200.0 arr = zarr.create_array( store=MemoryStore(), shape=(4,), dtype="float32", + fill_value=minimum, chunks=(4,), filters=[ ScaleOffset(scale=65535 / (maximum - minimum), offset=minimum), @@ -645,6 +646,29 @@ def test_creation_with_unencodable_default_fill_value() -> None: np.testing.assert_allclose(np.asarray(arr[:]), data, atol=1e-2) +@requires_cast_value_rs +def test_creation_with_unencodable_fill_value_raises() -> None: + """An array whose fill value cannot be round-tripped through the codec chain + must fail creation: the array has to be readable before any chunks have been + written, and an unwritten chunk returns the fill value + (https://github.com/zarr-developers/zarr-python/issues/4511). + """ + minimum, maximum = 100.0, 200.0 + with pytest.raises(ValueError, match="out of range"): + zarr.create_array( + store=MemoryStore(), + shape=(4,), + dtype="float32", + # The default fill value (0.0) lies outside [100, 200]. + chunks=(4,), + filters=[ + ScaleOffset(scale=65535 / (maximum - minimum), offset=minimum), + CastValue(data_type="uint16", rounding="nearest-even"), + ], + compressors=None, + ) + + @requires_cast_value_rs def test_unencodable_fill_value_does_not_mask_invalid_config() -> None: """Tolerating an unencodable fill value must not weaken validation: a codec diff --git a/tests/test_codecs/test_scale_offset.py b/tests/test_codecs/test_scale_offset.py index b81ae004cc..a29518d844 100644 --- a/tests/test_codecs/test_scale_offset.py +++ b/tests/test_codecs/test_scale_offset.py @@ -495,16 +495,18 @@ def test_compute_encoded_size() -> None: # --------------------------------------------------------------------------- -def test_creation_with_unencodable_default_fill_value() -> None: - """A scale/offset pair mapping only a subrange of the dtype makes the default - fill value (0) unencodable. Array creation must still succeed: the fill value - is metadata, not chunk data, and is never required to pass through the codec - chain (https://github.com/zarr-developers/zarr-python/issues/4511). +def test_creation_with_encodable_fill_value() -> None: + """A scale/offset pair mapping only a subrange of the dtype makes the dtype's + default scalar (0) unencodable, but array creation succeeds when the array's + fill value lies inside that subrange: the fill value is the one probed + against the codec chain, and it round-trips + (https://github.com/zarr-developers/zarr-python/issues/4511). """ arr = zarr.create_array( store=MemoryStore(), shape=(4,), dtype="uint16", + fill_value=100, chunks=(4,), filters=[ScaleOffset(offset=100, scale=1)], compressors=None, @@ -516,6 +518,24 @@ def test_creation_with_unencodable_default_fill_value() -> None: np.testing.assert_array_equal(arr[:], data) +def test_creation_with_unencodable_fill_value_raises() -> None: + """An array whose fill value cannot be round-tripped through the codec chain + must fail creation: the array has to be readable before any chunks have been + written, and an unwritten chunk returns the fill value + (https://github.com/zarr-developers/zarr-python/issues/4511). + """ + with pytest.raises(ValueError, match="outside the range"): + zarr.create_array( + store=MemoryStore(), + shape=(4,), + dtype="uint16", + # The default fill value (0) lies outside [100, 65535]. + chunks=(4,), + filters=[ScaleOffset(offset=100, scale=1)], + compressors=None, + ) + + def test_unencodable_fill_value_does_not_mask_invalid_config() -> None: """Tolerating an unencodable fill value must not weaken validation: a codec configuration that is structurally incompatible with the dtype still raises diff --git a/tests/test_codecs/test_sharding.py b/tests/test_codecs/test_sharding.py index 976a942406..bee94405fd 100644 --- a/tests/test_codecs/test_sharding.py +++ b/tests/test_codecs/test_sharding.py @@ -1530,12 +1530,11 @@ def test_open_rejects_stored_nested_sharding_with_indivisible_inner_chunk_shape( # --------------------------------------------------------------------------- -def test_create_with_unencodable_default_fill_value_inner_codecs() -> None: +def test_create_with_encodable_fill_value_inner_codecs() -> None: """`ShardingCodec._validate_inner_codecs` probes the inner codec chain with - `dtype.default_scalar()` as the fill value. When the chain maps only a - subrange of the dtype (e.g. a scale/offset pair for values in [100, 200]), - that probe value is unencodable — but the configuration is still valid, so - array creation must succeed + the array's fill value. When the chain maps only a subrange of the dtype + (e.g. a scale/offset pair for values in [100, 200]) but the fill value lies + inside that subrange, the configuration is valid and array creation succeeds (https://github.com/zarr-developers/zarr-python/issues/4511). """ arr = zarr.create_array( @@ -1544,6 +1543,7 @@ def test_create_with_unencodable_default_fill_value_inner_codecs() -> None: chunks=(4,), shards=(16,), dtype="uint16", + fill_value=100, filters=[ScaleOffset(offset=100, scale=1)], ) # Full-shard writes contain no fill positions, so they encode cleanly. @@ -1551,3 +1551,22 @@ def test_create_with_unencodable_default_fill_value_inner_codecs() -> None: data = np.array([100, 150, 200, 65435] * 4, dtype="uint16") arr[:] = data np.testing.assert_array_equal(arr[:], data) + + +def test_create_with_unencodable_fill_value_inner_codecs_raises() -> None: + """`ShardingCodec._validate_inner_codecs` must raise when the array's fill + value cannot be round-tripped through the inner codec chain: an unwritten + shard returns the fill value, so the array has to be readable before any + chunks have been written + (https://github.com/zarr-developers/zarr-python/issues/4511). + """ + with pytest.raises(ValueError, match="outside the range"): + zarr.create_array( + store=MemoryStore(), + shape=(16,), + chunks=(4,), + shards=(16,), + dtype="uint16", + # The default fill value (0) lies outside [100, 65535]. + filters=[ScaleOffset(offset=100, scale=1)], + )