diff --git a/changes/4560.bugfix.md b/changes/4560.bugfix.md new file mode 100644 index 0000000000..3dcf01a342 --- /dev/null +++ b/changes/4560.bugfix.md @@ -0,0 +1,8 @@ +Encoding a chunk through a numcodecs filter no longer passes the +filter's flattened output to the next codec: `_encode_sync` reshapes the +flattened array to the filter's output shape (from +`resolve_metadata`), so downstream codecs (e.g. `TransposeCodec` or an `imagecodecs` serializer chained after a +`Delta` or `FixedScaleOffset` filter) receive a chunk of the expected +shape instead of a flat 1-D array that could make writing fail or +silently corrupt data. Shape-changing codecs such as `PackBits` get +their own resolved output shape. diff --git a/src/zarr/codecs/numcodecs/_codecs.py b/src/zarr/codecs/numcodecs/_codecs.py index c980743166..5cd1611447 100644 --- a/src/zarr/codecs/numcodecs/_codecs.py +++ b/src/zarr/codecs/numcodecs/_codecs.py @@ -164,7 +164,12 @@ def _encode_sync(self, chunk_data: NDBuffer, chunk_spec: ArraySpec) -> NDBuffer: # would be encoded in transposed element order (gh-3558) chunk_ndarray = np.ascontiguousarray(chunk_data.as_ndarray_like()) out = self._codec.encode(chunk_ndarray) - return chunk_spec.prototype.nd_buffer.from_ndarray_like(out) + # numcodecs filters return a flattened array; reshape it to the + # output shape so downstream codecs receive the chunk shape that + # resolve_metadata promised them (gh-4553) + return chunk_spec.prototype.nd_buffer.from_ndarray_like( + out.reshape(self.resolve_metadata(chunk_spec).shape) + ) async def _encode_single(self, chunk_data: NDBuffer, chunk_spec: ArraySpec) -> NDBuffer: return await asyncio.to_thread(self._encode_sync, chunk_data, chunk_spec) diff --git a/tests/test_codecs/test_numcodecs.py b/tests/test_codecs/test_numcodecs.py index d78b73f34c..bf45017757 100644 --- a/tests/test_codecs/test_numcodecs.py +++ b/tests/test_codecs/test_numcodecs.py @@ -11,6 +11,7 @@ from zarr import config, create_array, open_array from zarr.abc.numcodec import _is_numcodec, _is_numcodec_cls from zarr.codecs import numcodecs as _numcodecs +from zarr.codecs.transpose import TransposeCodec from zarr.errors import UnknownCodecError from zarr.registry import get_codec_class, get_numcodec @@ -192,6 +193,45 @@ def test_generic_filter_f_contiguous( np.testing.assert_array_equal(data, a[:, :]) +def test_generic_filter_flattened_output_reshaped() -> None: + # gh-4553: numcodecs filters return a flattened array; `_encode_sync` must + # reshape it back to the chunk shape, so downstream codecs receive chunks + # with the shape the pipeline promised them + data = np.arange(256, dtype="int64").reshape(16, 16) + + a = create_array( + {}, + shape=data.shape, + chunks=(16, 16), + dtype=data.dtype, + fill_value=0, + filters=[_numcodecs.Delta(dtype="int64"), TransposeCodec(order=(1, 0))], + ) + + a[:, :] = data + with codec_conf(): + b = open_array(a.store, mode="r") + np.testing.assert_array_equal(data, b[:, :]) + + +def test_generic_filter_packedbits_output_shape() -> None: + # PackBits changes the size of the chunk, so `_encode_sync` must reshape its + # output to the shape from `resolve_metadata`, not the input chunk shape + data = np.tril(np.ones((16, 16), dtype=bool)) + + a = create_array( + {}, + shape=data.shape, + chunks=(16, 16), + dtype=data.dtype, + fill_value=0, + filters=[_numcodecs.PackBits()], + ) + + a[:, :] = data + np.testing.assert_array_equal(data, a[:, :]) + + def test_generic_filter_bitround() -> None: data = np.linspace(0, 1, 256, dtype="float32").reshape((16, 16))