diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 0000000..3a6249d --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,3 @@ +reviews: + path_filters: + - "!test/data/**/*.zarr/**" diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 824071a..e25a525 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -21,6 +21,7 @@ jobs: duckdb_version: v1.5.0 ci_tools_version: v1.5.0 extension_name: duckdb_zarr + opt_in_archs: linux_amd64;linux_arm64;osx_amd64;osx_arm64;windows_amd64;windows_amd64_mingw;wasm_mvp;wasm_eh;wasm_threads code-quality-check: name: Code Quality Check diff --git a/.github/workflows/PublishExtensionRepository.yml b/.github/workflows/PublishExtensionRepository.yml index 7577273..39a0d20 100644 --- a/.github/workflows/PublishExtensionRepository.yml +++ b/.github/workflows/PublishExtensionRepository.yml @@ -24,6 +24,7 @@ jobs: duckdb_version: v1.5.0 ci_tools_version: v1.5.0 extension_name: duckdb_zarr + opt_in_archs: linux_amd64;linux_arm64;osx_amd64;osx_arm64;windows_amd64 package-repository: name: Package release assets diff --git a/.gitignore b/.gitignore index 586a81c..d68563a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ testext test/python/__pycache__/ scripts/__pycache__/ .Rhistory +__pycache__/ +duckdb_zarr-v1.5.0-*.duckdb_extension.gz +duckdb_zarr.duckdb_extension diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 92245a9..8ceb4e2 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -28,9 +28,9 @@ ## Current MVP -- Implemented now: local Zarr v2 metadata discovery, consolidated `.zmetadata` support, gzip/uncompressed numeric chunk decode, and DuckDB table functions (`zarr_groups`, `zarr_arrays`, `zarr_chunks`, `zarr_cells`) with projection-aware scans and chunk pruning. -- Partially implemented: the Store Adapter includes practical remote support for consolidated HTTP/S3-style stores via DuckDB `httpfs`, and the Planner path performs predicate pushdown for the current `zarr_cells()` scan. -- Planned: the Arrow Materializer, broader codec chains such as Blosc, richer convention adapters, and the fuller target flow described below. +- Implemented now: local Zarr v2 and v3 metadata discovery, Zarr v2 consolidated `.zmetadata` support, gzip/uncompressed numeric chunk decode, and DuckDB table functions (`zarr_groups`, `zarr_arrays`, `zarr_chunks`, `zarr_cells`, `zarr`) with projection-aware scans and chunk pruning. +- Partially implemented: the Store Adapter includes practical remote support for consolidated HTTP/S3-style Zarr v2 stores via DuckDB `httpfs`, and the Codec Pipeline now handles the current v3 subset of `bytes`, optional `gzip`, constrained `transpose`, and `sharding_indexed` with inner Blosc (`zstd`) for common OME-Zarr image and label data. +- Planned: the Arrow Materializer, broader codec chains beyond the current Blosc path, richer convention adapters, and the fuller target flow described below. ## Flow diff --git a/Makefile b/Makefile index 8f76d0e..ca87fa4 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) EXT_NAME=duckdb_zarr EXT_CONFIG=${PROJ_DIR}extension_config.cmake -.PHONY: fixture test_metadata +.PHONY: fixture test_metadata test_python_example fixture: python3 scripts/create_sample_zarr.py @@ -13,5 +13,8 @@ fixture: test_metadata: fixture ${MAKE} test +test_python_example: fixture + uvx --with duckdb==1.5.0 --with numpy --with pandas python test/python_smoke.py + # Include the Makefile from extension-ci-tools include extension-ci-tools/makefiles/duckdb_extension.Makefile diff --git a/README.md b/README.md index 491e7be..a35540f 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,11 @@ The current implementation follows the project documents conservatively: Today’s extension provides metadata and relational scan table functions for local stores and a constrained remote read path: - `zarr(path)` for a simple array overview -- `zarr(path, array_path)` as a convenience alias for `zarr_cells(path, array_path)` -- `zarr_groups(path)` -- `zarr_arrays(path)` -- `zarr_chunks(path)` -- `zarr_cells(path, array_path)` for Zarr dtypes `|b1`, `i1`, `i2`, `i4`, `i8`, `u1`, `u2`, `u4`, `u8`, `f2`, `f4`, and `f8` +- `zarr(path, array_path)` and `zarr(path, array_path, version_override)` as convenience aliases for `zarr_cells(...)` +- `zarr_groups(path)` and `zarr_groups(path, version_override)` +- `zarr_arrays(path)` and `zarr_arrays(path, version_override)` +- `zarr_chunks(path)` and `zarr_chunks(path, version_override)` +- `zarr_cells(path, array_path)` and `zarr_cells(path, array_path, version_override)` for Zarr dtypes `|b1`, `i1`, `i2`, `i4`, `i8`, `u1`, `u2`, `u4`, `u8`, `f2`, `f4`, and `f8` This gives a usable SQL entrypoint for understanding a Zarr store and projecting dense numeric arrays into relational rows. Other dtypes such as complex values and other unsupported or non-standard Zarr dtypes are currently rejected by `zarr_cells(path, array_path)`. @@ -24,14 +24,21 @@ This gives a usable SQL entrypoint for understanding a Zarr store and projecting The MVP currently supports: - Local filesystem Zarr v2 discovery +- Local filesystem Zarr v3 discovery from `zarr.json` - Zarr v2 consolidated metadata discovery from `.zmetadata` - Remote `http://`, `https://`, and `s3://` stores when DuckDB `httpfs` is available and the store is consolidated - OME-Zarr-style multiscales groups built on Zarr v2, including level arrays such as `0` +- Real-world OME-Zarr v3 metadata discovery for stores such as image-label hierarchies and multiscales groups - Group enumeration from `.zgroup` -- Array enumeration from `.zarray` -- Chunk enumeration for both `.` and `/` dimension separators +- Array enumeration from `.zarray` and `zarr.json` +- Automatic v2/v3 detection for local stores, with explicit `version_override` support (`auto`, `v2`, `v3`) on the lower-level SQL functions +- Chunk enumeration for v2 `.` and `/` separators plus Zarr v3 `default` and `v2` chunk-key encodings - `zarr_cells(path, array_path)` for dense arrays with dtypes `|b1`, `i1`, `i2`, `i4`, `i8`, `u1`, `u2`, `u4`, `u8`, `f2`, `f4`, and `f8` - Uncompressed and gzip-compressed chunk payloads +- Zarr v3 regular chunk grids with `bytes` and optional `gzip` codecs +- Zarr v3 transpose handling for identity and full-reverse permutations +- Zarr v3 `sharding_indexed` arrays with inner Blosc (`zstd`) decode for common OME-Zarr image and label data +- Real OME-Zarr v3 cell scans against `test/data/idr0062A/6001240_labels.zarr` - Missing-chunk fill-value materialization for supported `zarr_cells()` dtypes when `fill_value` is present - Dynamic `(dim_0, ..., value)` projection based on array rank and dtype - Projection-aware `zarr_cells()` scans @@ -43,9 +50,9 @@ The MVP currently supports: The MVP does not yet support: - `zarr_cells()` for dtypes outside `|b1`, `i1`, `i2`, `i4`, `i8`, `u1`, `u2`, `u4`, `u8`, `f2`, `f4`, and `f8` -- Blosc chunk decode +- Blosc codecs outside the currently supported `zstd`-backed path - Non-consolidated remote store discovery -- Zarr v3 metadata +- Remote hierarchical discovery for non-consolidated Zarr v3 group stores - Arrow materialization ## Quick Start @@ -60,10 +67,10 @@ Install developer tooling for formatting and hooks: ```sh python3 -m pip install pre-commit clang-format==11.0.1 -pre-commit install +pre-commit install --hook-type pre-commit --hook-type pre-push ``` -That single `pre-commit install` command now installs both: +Re-run that command after pulling hook configuration changes. It installs both: - a `pre-commit` hook that auto-fixes DuckDB formatting - a `pre-push` hook that runs the same formatter in `--check` mode before CI sees the push @@ -79,6 +86,18 @@ Run the extension tests: make test_metadata ``` +Run the Python smoke example with the same environment shape many Python users will use: + +```sh +uvx --with duckdb==1.5.0 --with numpy --with pandas python test/python_smoke.py +``` + +There is also a matching make target: + +```sh +make test_python_example +``` + Package a static extension repository layout from downloaded CI artifacts: ```sh @@ -115,7 +134,14 @@ SELECT * FROM zarr_arrays('test/data/simple_v2.zarr'); SELECT * FROM zarr_chunks('test/data/simple_v2.zarr'); SELECT * FROM zarr_cells('test/data/simple_v2.zarr', 'temperature'); SELECT * FROM zarr('test/data/simple_v2.zarr'); +SELECT * FROM zarr_arrays('test/data/simple_v3.zarr', 'v3'); +SELECT * FROM zarr('test/data/simple_v3.zarr', 'temperature_v3', 'v3'); SELECT * FROM zarr('test/data/ome_example.ome.zarr', '0'); +SELECT * FROM zarr('test/data/idr0062A/6001240_labels.zarr'); +SELECT * FROM zarr('test/data/idr0062A/6001240_labels.zarr', '0') LIMIT 5; +SELECT SUM(value) +FROM zarr('test/data/idr0062A/6001240_labels.zarr', 'labels/0/0') +WHERE dim_0 = 0 AND dim_1 = 0 AND dim_2 < 4 AND dim_3 < 4; ``` For remote stores in this phase, the practical contract is: @@ -124,6 +150,13 @@ For remote stores in this phase, the practical contract is: - DuckDB must have `httpfs` available for the URI scheme you use - chunk data are still read directly from remote chunk object paths +Version detection notes: + +- `zarr(path)` auto-detects between local v2 and v3 stores +- lower-level functions accept an optional `version_override` argument: `auto`, `v2`, or `v3` +- the convenience cell entrypoint also accepts an override as `zarr(path, array_path, version_override)` +- explicit override is useful when a path is ambiguous or when you want failures to be version-specific + ## Install Like An Extension This repo now includes a release-asset publishing workflow in @@ -209,7 +242,7 @@ The current code maps directly to the architecture document: - Store Adapter: local filesystem traversal through DuckDB’s `FileSystem` - Store Adapter: consolidated remote discovery for `http://`, `https://`, and `s3://` paths through DuckDB filesystem routing -- Metadata Parser: `.zgroup` and `.zarray` parsing via `yyjson` +- Metadata Parser: Zarr v2 `.zgroup`/`.zarray` plus Zarr v3 `zarr.json` parsing via `yyjson` - DuckDB Bridge: table functions registered from the extension entrypoint - Relational Cell Scan: `zarr_cells()` with pushed filters, projection-aware output, and dimension-based chunk pruning diff --git a/ROADMAP.md b/ROADMAP.md index f093eb6..a997c86 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -4,12 +4,17 @@ - [x] zarr_groups - [x] zarr_arrays - [x] local store discovery +- [x] local Zarr v3 metadata discovery ## [x] Phase 2: Cell Scan - [x] zarr_cells - [x] numeric types -- [~] gzip -- [ ] blosc +- [x] gzip +- [x] automatic v2/v3 detection +- [x] explicit version override support +- [x] constrained Zarr v3 codec support (`bytes`, optional `gzip`, constrained `transpose`) +- [x] Zarr v3 `sharding_indexed` with Blosc (`zstd`) for local OME-Zarr-style arrays +- [~] broader blosc coverage ## [x] Phase 3: Optimization - [x] chunk pruning @@ -18,6 +23,7 @@ ## [~] Phase 4: Remote - [~] S3 / HTTP for consolidated stores +- [ ] non-consolidated Zarr v3 remote discovery - [ ] caching ## [ ] Phase 5: Advanced diff --git a/scripts/create_sample_zarr.py b/scripts/create_sample_zarr.py index 0f8594b..fa0e90e 100644 --- a/scripts/create_sample_zarr.py +++ b/scripts/create_sample_zarr.py @@ -11,6 +11,7 @@ ROOT = Path(__file__).resolve().parents[1] STORE = ROOT / "test" / "data" / "simple_v2.zarr" +STORE_V3 = ROOT / "test" / "data" / "simple_v3.zarr" OME_STORE = ROOT / "test" / "data" / "ome_example.ome.zarr" @@ -198,8 +199,63 @@ def create_ome_store() -> None: ) +def create_simple_v3_store() -> None: + shutil.rmtree(STORE_V3, ignore_errors=True) + root_group = {"zarr_format": 3, "node_type": "group", "attributes": {"name": "simple_v3"}} + write_json(STORE_V3 / "zarr.json", root_group) + + temperature_array = { + "zarr_format": 3, + "node_type": "array", + "shape": [3, 2], + "data_type": "float64", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [2, 2]}}, + "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, + "fill_value": None, + "codecs": [ + {"name": "bytes", "configuration": {"endian": "little"}}, + {"name": "gzip", "configuration": {"level": 1}}, + ], + "attributes": {"units": "celsius"}, + } + write_json(STORE_V3 / "temperature_v3" / "zarr.json", temperature_array) + write_chunk(STORE_V3 / "temperature_v3" / "c" / "0" / "0", gzip_chunk(struct.pack("<4d", 1.0, 2.0, 3.0, 4.0))) + write_chunk(STORE_V3 / "temperature_v3" / "c" / "1" / "0", gzip_chunk(struct.pack("<4d", 5.0, 6.0, 0.0, 0.0))) + + mask_array = { + "zarr_format": 3, + "node_type": "array", + "shape": [2, 3], + "data_type": "bool", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [2, 2]}}, + "chunk_key_encoding": {"name": "v2", "configuration": {"separator": "."}}, + "fill_value": False, + "codecs": [{"name": "bytes"}], + } + write_json(STORE_V3 / "mask_v3" / "zarr.json", mask_array) + write_chunk(STORE_V3 / "mask_v3" / "0.0", bytes([1, 0, 1, 1])) + write_chunk(STORE_V3 / "mask_v3" / "0.1", bytes([0, 0, 1, 0])) + + fortran_array = { + "zarr_format": 3, + "node_type": "array", + "shape": [2, 2], + "data_type": "int32", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [2, 2]}}, + "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, + "fill_value": 0, + "codecs": [ + {"name": "transpose", "configuration": {"order": [1, 0]}}, + {"name": "bytes", "configuration": {"endian": "little"}}, + ], + } + write_json(STORE_V3 / "fortran_v3" / "zarr.json", fortran_array) + write_chunk(STORE_V3 / "fortran_v3" / "c" / "0" / "0", struct.pack("<4i", 1, 3, 2, 4)) + + def main() -> None: create_simple_store() + create_simple_v3_store() create_ome_store() diff --git a/src/include/zarr_metadata.hpp b/src/include/zarr_metadata.hpp index d65911d..9f0e8e0 100644 --- a/src/include/zarr_metadata.hpp +++ b/src/include/zarr_metadata.hpp @@ -1,17 +1,18 @@ #pragma once +#include "duckdb/function/function_set.hpp" #include "duckdb/function/table_function.hpp" namespace duckdb { class ZarrMetadata { public: - static TableFunction GetGroupsFunction(); - static TableFunction GetArraysFunction(); - static TableFunction GetChunksFunction(); - static TableFunction GetCellsFunction(); - static TableFunction GetZarrFunction(); - static TableFunction GetZarrCellsAliasFunction(); + static TableFunctionSet GetGroupsFunction(); + static TableFunctionSet GetArraysFunction(); + static TableFunctionSet GetChunksFunction(); + static TableFunctionSet GetCellsFunction(); + static TableFunctionSet GetZarrFunction(); + static TableFunctionSet GetZarrCellsAliasFunction(); }; } // namespace duckdb diff --git a/src/zarr_metadata.cpp b/src/zarr_metadata.cpp index 45e488e..b8ca2c9 100644 --- a/src/zarr_metadata.cpp +++ b/src/zarr_metadata.cpp @@ -12,6 +12,7 @@ #include "duckdb/planner/filter/null_filter.hpp" #include "miniz_wrapper.hpp" #include "yyjson.hpp" +#include "zstd.h" #include #include @@ -30,6 +31,24 @@ struct ZarrGroupEntry { }; struct ZarrArrayEntry { + ZarrArrayEntry() { + } + ZarrArrayEntry(string store_path_p, string array_path_p, int64_t zarr_format_p, int64_t rank_p, + vector shape_p, vector chunks_p, string dtype_p, string order_p, + string compressor_p, string fill_value_p, string chunk_key_encoding_p, string dimension_separator_p, + string metadata_path_p, bool supports_cells_p, string cells_error_p, bool is_sharding_indexed_p, + vector storage_chunks_p, vector inner_chunks_p, string inner_compressor_p, + string index_codecs_p, string index_location_p) + : store_path(std::move(store_path_p)), array_path(std::move(array_path_p)), zarr_format(zarr_format_p), + rank(rank_p), shape(std::move(shape_p)), chunks(std::move(chunks_p)), dtype(std::move(dtype_p)), + order(std::move(order_p)), compressor(std::move(compressor_p)), fill_value(std::move(fill_value_p)), + chunk_key_encoding(std::move(chunk_key_encoding_p)), dimension_separator(std::move(dimension_separator_p)), + metadata_path(std::move(metadata_path_p)), supports_cells(supports_cells_p), + cells_error(std::move(cells_error_p)), is_sharding_indexed(is_sharding_indexed_p), + storage_chunks(std::move(storage_chunks_p)), inner_chunks(std::move(inner_chunks_p)), + inner_compressor(std::move(inner_compressor_p)), index_codecs(std::move(index_codecs_p)), + index_location(std::move(index_location_p)) { + } string store_path; string array_path; int64_t zarr_format; @@ -40,11 +59,30 @@ struct ZarrArrayEntry { string order; string compressor; string fill_value; + string chunk_key_encoding; string dimension_separator; string metadata_path; + bool supports_cells; + string cells_error; + bool is_sharding_indexed = false; + vector storage_chunks; + vector inner_chunks; + string inner_compressor; + string index_codecs; + string index_location; }; struct ZarrChunkEntry { + ZarrChunkEntry() { + } + ZarrChunkEntry(string store_path_p, string array_path_p, string chunk_key_p, vector chunk_coords_p, + string file_path_p, int64_t file_size_bytes_p, bool present_p, bool is_virtual_inner_chunk_p, + vector storage_chunk_coords_p, vector inner_chunk_coords_p) + : store_path(std::move(store_path_p)), array_path(std::move(array_path_p)), chunk_key(std::move(chunk_key_p)), + chunk_coords(std::move(chunk_coords_p)), file_path(std::move(file_path_p)), + file_size_bytes(file_size_bytes_p), present(present_p), is_virtual_inner_chunk(is_virtual_inner_chunk_p), + storage_chunk_coords(std::move(storage_chunk_coords_p)), inner_chunk_coords(std::move(inner_chunk_coords_p)) { + } string store_path; string array_path; string chunk_key; @@ -52,6 +90,9 @@ struct ZarrChunkEntry { string file_path; int64_t file_size_bytes; bool present; + bool is_virtual_inner_chunk = false; + vector storage_chunk_coords; + vector inner_chunk_coords; }; struct ZarrNumericType { @@ -100,10 +141,54 @@ struct ZarrCellsGlobalState : public GlobalTableFunctionState { vector decoded_chunk; }; +enum class ZarrVersionOverride : uint8_t { AUTO, V2, V3 }; + static ZarrGroupEntry ParseGroupMetadata(yyjson_val *root, const string &store_path, const string &relative_path, const string &metadata_path); +static ZarrGroupEntry ParseGroupMetadataV3(yyjson_val *root, const string &store_path, const string &relative_path, + const string &metadata_path); static ZarrArrayEntry ParseArrayMetadataObject(yyjson_val *root, const string &store_path, const string &relative_path, const string &metadata_path); +static ZarrArrayEntry ParseArrayMetadataObjectV3(yyjson_val *root, const string &store_path, + const string &relative_path, const string &metadata_path); +static vector GenerateChunkEntries(FileSystem &fs, const string &store_path, + const ZarrArrayEntry &array, bool include_missing); +static idx_t FlattenCoordsCOrder(const vector &coords, const vector &shape); +static vector ShardInnerChunksPerShard(const ZarrArrayEntry &array); +static string RequireNodeType(yyjson_val *root, const string &metadata_path); + +static ZarrArrayEntry MakeArrayEntry(string store_path, string array_path, int64_t zarr_format, int64_t rank, + vector shape, vector chunks, string dtype, string order, + string compressor, string fill_value, string chunk_key_encoding, + string dimension_separator, string metadata_path, bool supports_cells, + string cells_error, bool is_sharding_indexed, vector storage_chunks, + vector inner_chunks, string inner_compressor, string index_codecs, + string index_location) { + ZarrArrayEntry entry; + entry.store_path = std::move(store_path); + entry.array_path = std::move(array_path); + entry.zarr_format = zarr_format; + entry.shape = std::move(shape); + entry.chunks = std::move(chunks); + entry.rank = NumericCast(entry.shape.size()); + D_ASSERT(rank == entry.rank); + entry.dtype = std::move(dtype); + entry.order = std::move(order); + entry.compressor = std::move(compressor); + entry.fill_value = std::move(fill_value); + entry.chunk_key_encoding = std::move(chunk_key_encoding); + entry.dimension_separator = std::move(dimension_separator); + entry.metadata_path = std::move(metadata_path); + entry.supports_cells = supports_cells; + entry.cells_error = std::move(cells_error); + entry.is_sharding_indexed = is_sharding_indexed; + entry.storage_chunks = std::move(storage_chunks); + entry.inner_chunks = std::move(inner_chunks); + entry.inner_compressor = std::move(inner_compressor); + entry.index_codecs = std::move(index_codecs); + entry.index_location = std::move(index_location); + return entry; +} static string JoinNodePath(const string &base, const string &name) { if (base.empty()) { @@ -188,6 +273,22 @@ static vector RequireInt64Array(yyjson_val *obj, const char *key) { return result; } +static yyjson_val *RequireObject(yyjson_val *obj, const char *key) { + auto value = RequireObjectKey(obj, key); + if (!yyjson_is_obj(value)) { + throw InvalidInputException("Zarr metadata key \"%s\" is not an object", key); + } + return value; +} + +static yyjson_val *RequireArray(yyjson_val *obj, const char *key) { + auto value = RequireObjectKey(obj, key); + if (!yyjson_is_arr(value)) { + throw InvalidInputException("Zarr metadata key \"%s\" is not an array", key); + } + return value; +} + static string JsonToString(yyjson_val *value) { if (!value || yyjson_is_null(value)) { return ""; @@ -221,6 +322,23 @@ static string NormalizeStorePath(FileSystem &fs, const string &path) { return fs.ExpandPath(path); } +static ZarrVersionOverride ParseZarrVersionOverride(const Value &value, const string &function_name) { + if (value.IsNull()) { + return ZarrVersionOverride::AUTO; + } + auto text = StringUtil::Lower(StringValue::Get(value)); + if (text.empty() || text == "auto") { + return ZarrVersionOverride::AUTO; + } + if (text == "2" || text == "v2") { + return ZarrVersionOverride::V2; + } + if (text == "3" || text == "v3") { + return ZarrVersionOverride::V3; + } + throw BinderException("%s version override must be one of auto, v2, or v3", function_name); +} + static idx_t Product(const vector &values) { idx_t result = 1; for (auto value : values) { @@ -249,7 +367,7 @@ static string ParseCompressorId(const string &compressor) { if (!yyjson_is_obj(root)) { throw InvalidInputException("Zarr compressor metadata is not an object"); } - return OptionalString(root, "id"); + return OptionalString(root, "id", OptionalString(root, "name")); } static bool HostIsLittleEndian() { @@ -366,6 +484,121 @@ static ZarrNumericType ParseNumericDType(const string &dtype) { throw InvalidInputException("Unsupported Zarr dtype: %s", dtype); } +static string GetExtensionName(yyjson_val *extension, const string &metadata_path, const string &field_name) { + if (yyjson_is_str(extension)) { + return string(unsafe_yyjson_get_str(extension), unsafe_yyjson_get_len(extension)); + } + if (yyjson_is_obj(extension)) { + auto name = OptionalString(extension, "name"); + if (name.empty()) { + throw InvalidInputException("%s contains an extension without a name in %s", field_name, metadata_path); + } + return name; + } + throw InvalidInputException("%s contains an invalid extension value in %s", field_name, metadata_path); +} + +static yyjson_val *GetExtensionConfiguration(yyjson_val *extension, const string &metadata_path, + const string &field_name) { + if (!yyjson_is_obj(extension)) { + return nullptr; + } + auto configuration = yyjson_obj_get(extension, "configuration"); + if (!configuration) { + return nullptr; + } + if (!yyjson_is_obj(configuration)) { + throw InvalidInputException("%s configuration is not an object in %s", field_name, metadata_path); + } + return configuration; +} + +static string ParseV3DataType(yyjson_val *root, const string &metadata_path, bool little_endian) { + auto data_type = RequireObjectKey(root, "data_type"); + if (!yyjson_is_str(data_type)) { + throw InvalidInputException("Only string Zarr v3 data_type values are currently supported: %s", metadata_path); + } + string type_name(unsafe_yyjson_get_str(data_type), unsafe_yyjson_get_len(data_type)); + if (type_name == "bool") { + return "|b1"; + } + if (type_name == "int8") { + return "|i1"; + } + if (type_name == "uint8") { + return "|u1"; + } + string endian_prefix = little_endian ? "<" : ">"; + if (type_name == "int16") { + return endian_prefix + "i2"; + } + if (type_name == "int32") { + return endian_prefix + "i4"; + } + if (type_name == "int64") { + return endian_prefix + "i8"; + } + if (type_name == "uint16") { + return endian_prefix + "u2"; + } + if (type_name == "uint32") { + return endian_prefix + "u4"; + } + if (type_name == "uint64") { + return endian_prefix + "u8"; + } + if (type_name == "float16") { + return endian_prefix + "f2"; + } + if (type_name == "float32") { + return endian_prefix + "f4"; + } + if (type_name == "float64") { + return endian_prefix + "f8"; + } + throw InvalidInputException("Unsupported Zarr v3 data_type in %s: %s", metadata_path, type_name); +} + +static bool IsIdentityPermutation(const vector &order) { + for (idx_t i = 0; i < order.size(); i++) { + if (order[i] != NumericCast(i)) { + return false; + } + } + return true; +} + +static bool IsReversePermutation(const vector &order) { + for (idx_t i = 0; i < order.size(); i++) { + auto expected = NumericCast(order.size() - i - 1); + if (order[i] != expected) { + return false; + } + } + return true; +} + +static string JsonObjectWithId(yyjson_val *codec_value, const string &codec_name) { + auto object = JsonToString(codec_value); + if (!object.empty()) { + return object; + } + return "{\"id\":\"" + codec_name + "\"}"; +} + +static bool IsSupportedBloscShuffle(const string &shuffle, idx_t typesize) { + if (shuffle.empty() || shuffle == "noshuffle") { + return true; + } + if (shuffle == "shuffle") { + return true; + } + if (shuffle == "bitshuffle") { + return typesize == 1; + } + return false; +} + static float DecodeFloat16(uint16_t half_bits) { auto sign = (half_bits >> 15) & 0x1; auto exponent = (half_bits >> 10) & 0x1f; @@ -503,6 +736,131 @@ static vector LinearToCoords(idx_t linear_index, const vector return coordinates; } +static void ByteUnshuffle(const char *input, idx_t input_size, idx_t typesize, vector &output) { + if (typesize <= 1) { + output.assign(input, input + input_size); + return; + } + auto element_count = input_size / typesize; + output.resize(input_size); + for (idx_t element_idx = 0; element_idx < element_count; element_idx++) { + for (idx_t byte_idx = 0; byte_idx < typesize; byte_idx++) { + output[element_idx * typesize + byte_idx] = input[byte_idx * element_count + element_idx]; + } + } +} + +static void BitUnshuffleTypesizeOne(const char *input, idx_t input_size, vector &output) { + if (input_size % 8 != 0) { + throw InvalidInputException("Unsupported Blosc bitshuffle payload size: %llu", input_size); + } + auto plane_size = input_size / 8; + output.assign(input_size, 0); + for (idx_t i = 0; i < input_size; i++) { + auto byte_index = i / 8; + auto bit_index = i % 8; + uint8_t value = 0; + for (idx_t bit = 0; bit < 8; bit++) { + auto bit_value = (static_cast(input[bit * plane_size + byte_index]) >> bit_index) & 0x1; + value |= NumericCast(bit_value << bit); + } + output[i] = NumericCast(value); + } +} + +static vector DecompressZstd(const char *compressed_data, idx_t compressed_size, idx_t expected_size) { + vector result(expected_size); + auto decompressed_size = + duckdb_zstd::ZSTD_decompress(result.data(), result.size(), compressed_data, compressed_size); + if (duckdb_zstd::ZSTD_isError(decompressed_size)) { + throw InvalidInputException("Failed to decompress Zstd payload inside Blosc chunk"); + } + if (decompressed_size != expected_size) { + throw InvalidInputException("Unexpected Zstd decompressed size: expected %llu bytes, got %llu bytes", + expected_size, decompressed_size); + } + return result; +} + +static vector DecompressBloscChunk(const vector &compressed_data, const string &compressor, + idx_t expected_size) { + if (compressed_data.size() < 20) { + throw InvalidInputException("Blosc chunk is too small"); + } + unique_ptr doc(yyjson_read(compressor.c_str(), compressor.size(), 0), + yyjson_doc_free); + if (!doc) { + throw InvalidInputException("Failed to parse Blosc codec metadata"); + } + auto root = yyjson_doc_get_root(doc.get()); + auto configuration = yyjson_obj_get(root, "configuration"); + if (configuration) { + if (!yyjson_is_obj(configuration)) { + throw InvalidInputException("Blosc codec configuration is not an object"); + } + root = configuration; + } + auto cname = OptionalString(root, "cname"); + auto shuffle = OptionalString(root, "shuffle", "noshuffle"); + auto typesize_value = yyjson_obj_get(root, "typesize"); + if (!typesize_value || (!yyjson_is_uint(typesize_value) && !yyjson_is_sint(typesize_value))) { + throw InvalidInputException("Blosc codec metadata is missing integer typesize"); + } + auto typesize = yyjson_is_uint(typesize_value) ? NumericCast(unsafe_yyjson_get_uint(typesize_value)) + : NumericCast(unsafe_yyjson_get_sint(typesize_value)); + if (cname != "zstd") { + throw InvalidInputException("Only Blosc with cname=zstd is currently supported"); + } + + auto nbytes = NumericCast(Load(reinterpret_cast(compressed_data.data() + 4))); + auto blocksize = NumericCast(Load(reinterpret_cast(compressed_data.data() + 8))); + auto cbytes = NumericCast(Load(reinterpret_cast(compressed_data.data() + 12))); + if (nbytes != expected_size) { + throw InvalidInputException("Blosc chunk size mismatch: expected %llu bytes, got %llu bytes", expected_size, + nbytes); + } + if (cbytes != compressed_data.size()) { + throw InvalidInputException("Unexpected Blosc compressed byte count"); + } + auto nblocks = (nbytes + blocksize - 1) / blocksize; + auto offsets_table_size = nblocks * sizeof(uint32_t); + if (compressed_data.size() < 16 + offsets_table_size) { + throw InvalidInputException("Blosc chunk is missing the block offsets table"); + } + + vector result; + result.reserve(expected_size); + for (idx_t block_idx = 0; block_idx < nblocks; block_idx++) { + auto block_offset_ptr = compressed_data.data() + 16 + block_idx * sizeof(uint32_t); + auto block_offset = NumericCast(Load(reinterpret_cast(block_offset_ptr))); + if (block_offset + sizeof(uint32_t) > compressed_data.size()) { + throw InvalidInputException("Invalid Blosc block offset"); + } + auto block_cbytes = NumericCast( + Load(reinterpret_cast(compressed_data.data() + block_offset))); + auto block_payload_offset = block_offset + sizeof(uint32_t); + if (block_payload_offset + block_cbytes > compressed_data.size()) { + throw InvalidInputException("Invalid Blosc block payload length"); + } + auto block_expected_size = MinValue(blocksize, nbytes - block_idx * blocksize); + auto raw_block = + DecompressZstd(compressed_data.data() + block_payload_offset, block_cbytes, block_expected_size); + vector decoded_block; + if (shuffle == "shuffle") { + ByteUnshuffle(raw_block.data(), raw_block.size(), typesize, decoded_block); + } else if (shuffle == "bitshuffle") { + if (typesize != 1) { + throw InvalidInputException("Blosc bitshuffle is currently only supported for typesize=1"); + } + BitUnshuffleTypesizeOne(raw_block.data(), raw_block.size(), decoded_block); + } else { + decoded_block = std::move(raw_block); + } + result.insert(result.end(), decoded_block.begin(), decoded_block.end()); + } + return result; +} + static vector DecompressChunk(const vector &compressed_data, const string &compressor, idx_t expected_size) { auto compressor_id = ParseCompressorId(compressor); @@ -514,6 +872,9 @@ static vector DecompressChunk(const vector &compressed_data, const s return compressed_data; } if (compressor_id != "gzip") { + if (compressor_id == "blosc") { + return DecompressBloscChunk(compressed_data, compressor, expected_size); + } throw InvalidInputException("Unsupported Zarr compressor for zarr_cells: %s", compressor_id); } vector decompressed(expected_size); @@ -797,6 +1158,50 @@ static vector SelectFilteredChunks(const ZarrArrayEntry &array, const vec return chunk_indexes; } +static idx_t ShardingIndexEncodedSize(const ZarrArrayEntry &array) { + auto chunk_count = Product(ShardInnerChunksPerShard(array)); + idx_t index_size = chunk_count * 2 * sizeof(uint64_t); + if (array.index_codecs == "bytes+crc32c") { + index_size += sizeof(uint32_t); + } + return index_size; +} + +static vector DecodeShardedInnerChunk(const vector &shard_data, const ZarrArrayEntry &array, + const ZarrChunkEntry &chunk, idx_t expected_size) { + auto index_size = ShardingIndexEncodedSize(array); + if (shard_data.size() < index_size) { + throw InvalidInputException("Shard file is smaller than its sharding index: %s", chunk.file_path); + } + idx_t index_offset = 0; + if (array.index_location == "end") { + index_offset = shard_data.size() - index_size; + } + auto crc_size = array.index_codecs == "bytes+crc32c" ? sizeof(uint32_t) : 0; + auto index_bytes = reinterpret_cast(shard_data.data() + index_offset); + auto index_payload_size = index_size - crc_size; + auto chunks_per_shard = ShardInnerChunksPerShard(array); + auto flat_index = FlattenCoordsCOrder(chunk.inner_chunk_coords, chunks_per_shard); + auto entry_offset = flat_index * 2 * sizeof(uint64_t); + if (entry_offset + 2 * sizeof(uint64_t) > index_payload_size) { + throw InvalidInputException("Invalid sharding index entry offset for %s", chunk.file_path); + } + auto offset = + ReadUnsignedInteger(reinterpret_cast(index_bytes + entry_offset), sizeof(uint64_t), true); + auto nbytes = ReadUnsignedInteger(reinterpret_cast(index_bytes + entry_offset + sizeof(uint64_t)), + sizeof(uint64_t), true); + if (offset == std::numeric_limits::max() && nbytes == std::numeric_limits::max()) { + return {}; + } + if (offset + nbytes > shard_data.size()) { + throw InvalidInputException("Invalid sharding index offset/length for %s", chunk.file_path); + } + vector encoded_inner_chunk; + encoded_inner_chunk.insert(encoded_inner_chunk.end(), shard_data.begin() + NumericCast(offset), + shard_data.begin() + NumericCast(offset + nbytes)); + return DecompressChunk(encoded_inner_chunk, array.inner_compressor, expected_size); +} + static bool ComputeGlobalCoords(idx_t linear_index, const ZarrArrayEntry &array, const ZarrChunkEntry &chunk, vector &global_coords) { auto local_coords = LinearToCoords(linear_index, array.chunks, array.order); @@ -838,7 +1243,13 @@ static bool LoadNextChunk(FileSystem &fs, const ZarrCellsBindData &bind_data, Za auto &chunk = bind_data.chunks[state.current_chunk_index]; if (chunk.present) { auto raw_data = ReadBinaryFile(fs, chunk.file_path); - state.decoded_chunk = DecompressChunk(raw_data, bind_data.array.compressor, state.expected_chunk_bytes); + if (bind_data.array.is_sharding_indexed || + ParseCompressorId(bind_data.array.compressor) == "sharding_indexed") { + state.decoded_chunk = + DecodeShardedInnerChunk(raw_data, bind_data.array, chunk, state.expected_chunk_bytes); + } else { + state.decoded_chunk = DecompressChunk(raw_data, bind_data.array.compressor, state.expected_chunk_bytes); + } } else { state.decoded_chunk.clear(); } @@ -849,90 +1260,6 @@ static bool LoadNextChunk(FileSystem &fs, const ZarrCellsBindData &bind_data, Za return false; } -static bool TryParseInteger(const string &text, int64_t &value) { - if (text.empty()) { - return false; - } - int64_t parsed_value = 0; - for (auto ch : text) { - if (!StringUtil::CharacterIsDigit(ch)) { - return false; - } - auto digit = NumericCast(ch - '0'); - if (parsed_value > (std::numeric_limits::max() - digit) / 10) { - return false; - } - parsed_value = parsed_value * 10 + digit; - } - value = parsed_value; - return true; -} - -static bool TryParseChunkCoords(const string &chunk_key, char separator, vector &coords) { - auto parts = StringUtil::Split(chunk_key, separator); - if (parts.empty()) { - return false; - } - coords.clear(); - coords.reserve(parts.size()); - for (auto &part : parts) { - int64_t coord; - if (!TryParseInteger(part, coord)) { - return false; - } - coords.push_back(coord); - } - return true; -} - -static void CollectSlashChunks(FileSystem &fs, const ZarrArrayEntry &array, const string &dir_path, - const string &relative_key, vector &entries) { - fs.ListFiles(dir_path, [&](const string &child_name, bool is_dir) { - if (StringUtil::StartsWith(child_name, ".")) { - return; - } - auto child_path = fs.JoinPath(dir_path, child_name); - auto next_key = relative_key.empty() ? child_name : relative_key + "/" + child_name; - if (is_dir) { - CollectSlashChunks(fs, array, child_path, next_key, entries); - return; - } - vector coords; - if (!TryParseChunkCoords(next_key, '/', coords)) { - return; - } - if (coords.size() != NumericCast(array.rank)) { - return; - } - auto handle = fs.OpenFile(child_path, FileFlags::FILE_FLAGS_READ); - entries.push_back({array.store_path, array.array_path, next_key, std::move(coords), child_path, - NumericCast(handle->GetFileSize()), true}); - }); -} - -static void CollectDotChunks(FileSystem &fs, const ZarrArrayEntry &array, const string &dir_path, - vector &entries) { - fs.ListFiles(dir_path, [&](const string &child_name, bool is_dir) { - if (is_dir) { - return; - } - if (StringUtil::StartsWith(child_name, ".")) { - return; - } - auto child_path = fs.JoinPath(dir_path, child_name); - vector coords; - if (!TryParseChunkCoords(child_name, '.', coords)) { - return; - } - if (coords.size() != NumericCast(array.rank)) { - return; - } - auto handle = fs.OpenFile(child_path, FileFlags::FILE_FLAGS_READ); - entries.push_back({array.store_path, array.array_path, child_name, std::move(coords), child_path, - NumericCast(handle->GetFileSize()), true}); - }); -} - static ZarrArrayEntry ParseArrayMetadata(FileSystem &fs, const string &store_path, const string &relative_path, const string &dir_path) { auto metadata_path = fs.JoinPath(dir_path, ".zarray"); @@ -945,25 +1272,21 @@ static ZarrArrayEntry ParseArrayMetadata(FileSystem &fs, const string &store_pat return ParseArrayMetadataObject(yyjson_doc_get_root(doc.get()), store_path, relative_path, metadata_path); } -static string ArrayDirectoryPath(FileSystem &fs, const string &store_path, const string &array_path) { - if (array_path.empty()) { - return store_path; - } - return fs.JoinPath(store_path, array_path); -} - -static void CollectArrayChunks(FileSystem &fs, const ZarrArrayEntry &array, vector &chunks) { - auto dir_path = ArrayDirectoryPath(fs, array.store_path, array.array_path); - if (array.dimension_separator == "/") { - CollectSlashChunks(fs, array, dir_path, "", chunks); - } else { - CollectDotChunks(fs, array, dir_path, chunks); +static ZarrArrayEntry ParseArrayMetadataV3(FileSystem &fs, const string &store_path, const string &relative_path, + const string &dir_path) { + auto metadata_path = fs.JoinPath(dir_path, "zarr.json"); + auto metadata_text = ReadTextFile(fs, metadata_path); + unique_ptr doc(yyjson_read(metadata_text.c_str(), metadata_text.size(), 0), + yyjson_doc_free); + if (!doc) { + throw InvalidInputException("Failed to parse %s", metadata_path); } + return ParseArrayMetadataObjectV3(yyjson_doc_get_root(doc.get()), store_path, relative_path, metadata_path); } -static void TraverseStore(FileSystem &fs, const string &store_path, const string &dir_path, const string &relative_path, - bool collect_chunks, vector &groups, vector &arrays, - vector &chunks) { +static void TraverseStoreV2(FileSystem &fs, const string &store_path, const string &dir_path, + const string &relative_path, bool collect_chunks, vector &groups, + vector &arrays, vector &chunks) { auto group_metadata = fs.JoinPath(dir_path, ".zgroup"); auto array_metadata = fs.JoinPath(dir_path, ".zarray"); @@ -980,7 +1303,8 @@ static void TraverseStore(FileSystem &fs, const string &store_path, const string if (fs.FileExists(array_metadata)) { auto entry = ParseArrayMetadata(fs, store_path, relative_path, dir_path); if (collect_chunks) { - CollectArrayChunks(fs, entry, chunks); + auto array_chunks = GenerateChunkEntries(fs, store_path, entry, false); + chunks.insert(chunks.end(), array_chunks.begin(), array_chunks.end()); } arrays.push_back(std::move(entry)); return; @@ -994,8 +1318,52 @@ static void TraverseStore(FileSystem &fs, const string &store_path, const string return; } auto child_path = fs.JoinPath(dir_path, child_name); - TraverseStore(fs, store_path, child_path, JoinNodePath(relative_path, child_name), collect_chunks, groups, - arrays, chunks); + TraverseStoreV2(fs, store_path, child_path, JoinNodePath(relative_path, child_name), collect_chunks, groups, + arrays, chunks); + }); +} + +static void TraverseStoreV3(FileSystem &fs, const string &store_path, const string &dir_path, + const string &relative_path, bool collect_chunks, vector &groups, + vector &arrays, vector &chunks) { + auto metadata_path = fs.JoinPath(dir_path, "zarr.json"); + if (fs.FileExists(metadata_path)) { + auto metadata_text = ReadTextFile(fs, metadata_path); + unique_ptr doc( + yyjson_read(metadata_text.c_str(), metadata_text.size(), 0), yyjson_doc_free); + if (!doc) { + throw InvalidInputException("Failed to parse %s", metadata_path); + } + auto root = yyjson_doc_get_root(doc.get()); + if (!yyjson_is_obj(root)) { + throw InvalidInputException("%s does not contain a JSON object", metadata_path); + } + auto node_type = RequireNodeType(root, metadata_path); + if (node_type == "group") { + groups.push_back(ParseGroupMetadataV3(root, store_path, relative_path, metadata_path)); + } else if (node_type == "array") { + auto entry = ParseArrayMetadataObjectV3(root, store_path, relative_path, metadata_path); + if (collect_chunks) { + auto array_chunks = GenerateChunkEntries(fs, store_path, entry, false); + chunks.insert(chunks.end(), array_chunks.begin(), array_chunks.end()); + } + arrays.push_back(std::move(entry)); + return; + } else { + throw InvalidInputException("Unsupported Zarr v3 node_type in %s: %s", metadata_path, node_type); + } + } + + fs.ListFiles(dir_path, [&](const string &child_name, bool is_dir) { + if (!is_dir) { + return; + } + if (StringUtil::StartsWith(child_name, ".")) { + return; + } + auto child_path = fs.JoinPath(dir_path, child_name); + TraverseStoreV3(fs, store_path, child_path, JoinNodePath(relative_path, child_name), collect_chunks, groups, + arrays, chunks); }); } @@ -1016,6 +1384,22 @@ static int64_t RequireZarrFormatV2(yyjson_val *root, const string &metadata_path return zarr_format; } +static int64_t RequireZarrFormatV3(yyjson_val *root, const string &metadata_path) { + auto zarr_format = RequireInt64(root, "zarr_format"); + if (zarr_format != 3) { + throw InvalidInputException("Only Zarr v3 metadata is currently supported: %s", metadata_path); + } + return zarr_format; +} + +static string RequireNodeType(yyjson_val *root, const string &metadata_path) { + auto node_type = OptionalString(root, "node_type"); + if (node_type.empty()) { + throw InvalidInputException("Zarr v3 metadata is missing node_type: %s", metadata_path); + } + return node_type; +} + static ZarrGroupEntry ParseGroupMetadata(yyjson_val *root, const string &store_path, const string &relative_path, const string &metadata_path) { if (!yyjson_is_obj(root)) { @@ -1055,18 +1439,328 @@ static ZarrArrayEntry ParseArrayMetadataObject(yyjson_val *root, const string &s throw InvalidInputException("Unsupported Zarr order in %s: %s", metadata_path, order); } - return {store_path, - relative_path, - RequireZarrFormatV2(root, metadata_path), - NumericCast(shape.size()), - std::move(shape), - std::move(chunks), - OptionalString(root, "dtype"), - std::move(order), - std::move(compressor), - std::move(fill_value), - std::move(dimension_separator), - metadata_path}; + return MakeArrayEntry(store_path, relative_path, RequireZarrFormatV2(root, metadata_path), + NumericCast(shape.size()), std::move(shape), std::move(chunks), + OptionalString(root, "dtype"), std::move(order), std::move(compressor), std::move(fill_value), + "v2", std::move(dimension_separator), metadata_path, true, "", false, {}, {}, "", "", ""); +} + +static ZarrGroupEntry ParseGroupMetadataV3(yyjson_val *root, const string &store_path, const string &relative_path, + const string &metadata_path) { + if (!yyjson_is_obj(root)) { + throw InvalidInputException("%s does not contain a JSON object", metadata_path); + } + auto zarr_format = RequireZarrFormatV3(root, metadata_path); + auto node_type = RequireNodeType(root, metadata_path); + if (node_type != "group") { + throw InvalidInputException("Expected a Zarr v3 group metadata document at %s", metadata_path); + } + return {store_path, FormatGroupPath(relative_path), zarr_format, metadata_path}; +} + +static void ParseV3ShardingCodec(yyjson_val *codec_value, const string &metadata_path, vector &outer_chunks, + string &compressor, bool &supports_cells, string &cells_error, + vector &inner_chunks, string &inner_compressor, string &index_codecs, + string &index_location, bool &little_endian) { + auto codec_config = GetExtensionConfiguration(codec_value, metadata_path, "codecs"); + if (!codec_config) { + throw InvalidInputException("Zarr v3 sharding_indexed codec is missing configuration in %s", metadata_path); + } + inner_chunks = RequireInt64Array(codec_config, "chunk_shape"); + if (inner_chunks.size() != outer_chunks.size()) { + throw InvalidInputException("Zarr v3 sharding_indexed inner chunk rank mismatch in %s", metadata_path); + } + for (idx_t i = 0; i < inner_chunks.size(); i++) { + if (inner_chunks[i] <= 0 || outer_chunks[i] <= 0 || outer_chunks[i] % inner_chunks[i] != 0) { + throw InvalidInputException( + "Zarr v3 sharding_indexed requires inner chunk_shape to evenly divide the outer chunk shape in %s", + metadata_path); + } + } + + auto inner_codecs = RequireArray(codec_config, "codecs"); + bool saw_bytes = false; + little_endian = HostIsLittleEndian(); + yyjson_val *inner_codec_value; + yyjson_arr_iter inner_codec_iter = yyjson_arr_iter_with(inner_codecs); + while ((inner_codec_value = yyjson_arr_iter_next(&inner_codec_iter))) { + auto inner_codec_name = GetExtensionName(inner_codec_value, metadata_path, "codecs"); + auto inner_codec_config = GetExtensionConfiguration(inner_codec_value, metadata_path, "codecs"); + if (inner_codec_name == "bytes") { + if (saw_bytes) { + throw InvalidInputException( + "Zarr v3 sharding_indexed metadata contains multiple inner bytes codecs: %s", metadata_path); + } + saw_bytes = true; + if (inner_codec_config) { + auto endian = OptionalString(inner_codec_config, "endian", ""); + if (endian == "little") { + little_endian = true; + } else if (endian == "big") { + little_endian = false; + } else if (!endian.empty()) { + supports_cells = false; + cells_error = "zarr_cells does not yet support Zarr v3 sharding_indexed bytes endian " + endian; + } + } + continue; + } + if (inner_codec_name == "blosc") { + inner_compressor = JsonObjectWithId(inner_codec_value, "blosc"); + compressor = JsonObjectWithId(codec_value, "sharding_indexed"); + if (!inner_codec_config) { + supports_cells = false; + cells_error = "zarr_cells requires Blosc configuration metadata for sharding_indexed arrays"; + continue; + } + auto cname = OptionalString(inner_codec_config, "cname", ""); + auto shuffle = OptionalString(inner_codec_config, "shuffle", "noshuffle"); + auto typesize = NumericCast(RequireInt64(inner_codec_config, "typesize")); + if (cname != "zstd") { + supports_cells = false; + cells_error = "zarr_cells currently supports Blosc only with cname=zstd"; + } else if (!IsSupportedBloscShuffle(shuffle, typesize)) { + supports_cells = false; + cells_error = + "zarr_cells currently supports Blosc shuffle, noshuffle, and bitshuffle only for typesize=1"; + } + continue; + } + if (inner_codec_name == "gzip") { + inner_compressor = "{\"id\":\"gzip\"}"; + compressor = JsonObjectWithId(codec_value, "sharding_indexed"); + if (inner_codec_config) { + auto level_value = yyjson_obj_get(inner_codec_config, "level"); + if (level_value && (yyjson_is_uint(level_value) || yyjson_is_sint(level_value))) { + auto level = yyjson_is_uint(level_value) ? NumericCast(unsafe_yyjson_get_uint(level_value)) + : unsafe_yyjson_get_sint(level_value); + inner_compressor = "{\"id\":\"gzip\",\"level\":" + std::to_string(level) + "}"; + } + } + continue; + } + supports_cells = false; + cells_error = "zarr_cells does not yet support Zarr v3 sharding_indexed inner codec: " + inner_codec_name; + } + if (!saw_bytes) { + supports_cells = false; + if (cells_error.empty()) { + cells_error = "zarr_cells requires an inner bytes codec for Zarr v3 sharding_indexed arrays"; + } + } + auto index_codec_array = RequireArray(codec_config, "index_codecs"); + index_location = OptionalString(codec_config, "index_location", "end"); + if (index_location != "start" && index_location != "end") { + supports_cells = false; + if (cells_error.empty()) { + cells_error = "zarr_cells only supports Zarr v3 sharding_indexed with index_location start or end"; + } + } + yyjson_val *index_codec_value; + yyjson_arr_iter index_codec_iter = yyjson_arr_iter_with(index_codec_array); + bool saw_index_bytes = false; + bool saw_crc32c = false; + while ((index_codec_value = yyjson_arr_iter_next(&index_codec_iter))) { + auto index_codec_name = GetExtensionName(index_codec_value, metadata_path, "index_codecs"); + auto index_codec_config = GetExtensionConfiguration(index_codec_value, metadata_path, "index_codecs"); + if (index_codec_name == "bytes") { + saw_index_bytes = true; + if (index_codec_config) { + auto endian = OptionalString(index_codec_config, "endian", "little"); + if (endian != "little") { + supports_cells = false; + cells_error = "zarr_cells currently requires little-endian sharding index bytes codec"; + } + } + continue; + } + if (index_codec_name == "crc32c") { + saw_crc32c = true; + continue; + } + supports_cells = false; + cells_error = "zarr_cells does not yet support Zarr v3 sharding index codec: " + index_codec_name; + } + if (!saw_index_bytes) { + supports_cells = false; + cells_error = "zarr_cells requires a bytes codec for Zarr v3 sharding indexes"; + } + index_codecs = saw_crc32c ? "bytes+crc32c" : "bytes"; +} + +static ZarrArrayEntry ParseArrayMetadataObjectV3(yyjson_val *root, const string &store_path, + const string &relative_path, const string &metadata_path) { + if (!yyjson_is_obj(root)) { + throw InvalidInputException("%s does not contain a JSON object", metadata_path); + } + auto zarr_format = RequireZarrFormatV3(root, metadata_path); + auto node_type = RequireNodeType(root, metadata_path); + if (node_type != "array") { + throw InvalidInputException("Expected a Zarr v3 array metadata document at %s", metadata_path); + } + auto shape = RequireInt64Array(root, "shape"); + auto chunk_grid = RequireObject(root, "chunk_grid"); + auto chunk_grid_name = GetExtensionName(chunk_grid, metadata_path, "chunk_grid"); + if (chunk_grid_name != "regular") { + throw InvalidInputException("Unsupported Zarr v3 chunk_grid in %s: %s", metadata_path, chunk_grid_name); + } + auto chunk_grid_config = GetExtensionConfiguration(chunk_grid, metadata_path, "chunk_grid"); + if (!chunk_grid_config) { + throw InvalidInputException("Zarr v3 chunk_grid is missing configuration in %s", metadata_path); + } + auto chunks = RequireInt64Array(chunk_grid_config, "chunk_shape"); + if (shape.size() != chunks.size()) { + throw InvalidInputException("%s has mismatched shape/chunk_grid rank", metadata_path); + } + for (idx_t i = 0; i < shape.size(); i++) { + if (shape[i] < 0) { + throw InvalidInputException("%s contains a negative shape dimension", metadata_path); + } + if (chunks[i] <= 0) { + throw InvalidInputException("%s contains a non-positive chunk dimension", metadata_path); + } + } + + auto chunk_key_encoding = RequireObjectKey(root, "chunk_key_encoding"); + auto chunk_key_encoding_name = GetExtensionName(chunk_key_encoding, metadata_path, "chunk_key_encoding"); + if (chunk_key_encoding_name != "default" && chunk_key_encoding_name != "v2") { + throw InvalidInputException("Unsupported Zarr v3 chunk_key_encoding in %s: %s", metadata_path, + chunk_key_encoding_name); + } + string dimension_separator = chunk_key_encoding_name == "default" ? "/" : "."; + auto chunk_key_config = GetExtensionConfiguration(chunk_key_encoding, metadata_path, "chunk_key_encoding"); + if (chunk_key_config) { + dimension_separator = OptionalString(chunk_key_config, "separator", dimension_separator); + } + if (dimension_separator != "." && dimension_separator != "/") { + throw InvalidInputException("Unsupported Zarr v3 chunk key separator in %s: %s", metadata_path, + dimension_separator); + } + + auto codecs = RequireArray(root, "codecs"); + auto codec_count = unsafe_yyjson_get_len(codecs); + if (codec_count == 1) { + auto first_codec = yyjson_arr_get_first(codecs); + auto first_codec_name = GetExtensionName(first_codec, metadata_path, "codecs"); + if (first_codec_name == "sharding_indexed") { + string compressor = JsonObjectWithId(first_codec, "sharding_indexed"); + string inner_compressor; + string index_codecs; + string index_location = "end"; + bool supports_cells = true; + string cells_error; + vector inner_chunks; + bool little_endian = HostIsLittleEndian(); + ParseV3ShardingCodec(first_codec, metadata_path, chunks, compressor, supports_cells, cells_error, + inner_chunks, inner_compressor, index_codecs, index_location, little_endian); + return MakeArrayEntry(store_path, relative_path, zarr_format, NumericCast(shape.size()), + std::move(shape), std::move(chunks), + ParseV3DataType(root, metadata_path, little_endian), "C", std::move(compressor), + JsonToString(yyjson_obj_get(root, "fill_value")), std::move(chunk_key_encoding_name), + std::move(dimension_separator), metadata_path, supports_cells, std::move(cells_error), + true, {}, std::move(inner_chunks), std::move(inner_compressor), + std::move(index_codecs), std::move(index_location)); + } + } + bool saw_bytes = false; + string order = "C"; + string compressor; + bool little_endian = HostIsLittleEndian(); + bool supports_cells = true; + string cells_error; + yyjson_val *codec_value; + yyjson_arr_iter codec_iter = yyjson_arr_iter_with(codecs); + while ((codec_value = yyjson_arr_iter_next(&codec_iter))) { + auto codec_name = GetExtensionName(codec_value, metadata_path, "codecs"); + auto codec_config = GetExtensionConfiguration(codec_value, metadata_path, "codecs"); + if (codec_name == "transpose") { + if (saw_bytes) { + throw InvalidInputException("Zarr v3 transpose codec must appear before bytes in %s", metadata_path); + } + if (!codec_config) { + throw InvalidInputException("Zarr v3 transpose codec is missing configuration in %s", metadata_path); + } + auto transpose_order = RequireInt64Array(codec_config, "order"); + if (transpose_order.size() != shape.size()) { + throw InvalidInputException("Zarr v3 transpose order rank mismatch in %s", metadata_path); + } + if (IsIdentityPermutation(transpose_order)) { + order = "C"; + } else if (IsReversePermutation(transpose_order)) { + order = "F"; + } else { + throw InvalidInputException( + "Only identity and full-reverse Zarr v3 transpose orders are currently supported: %s", + metadata_path); + } + continue; + } + if (codec_name == "bytes") { + if (saw_bytes) { + throw InvalidInputException("Zarr v3 metadata contains multiple bytes codecs: %s", metadata_path); + } + saw_bytes = true; + if (codec_config) { + auto endian = OptionalString(codec_config, "endian", ""); + if (endian == "little") { + little_endian = true; + } else if (endian == "big") { + little_endian = false; + } else if (!endian.empty()) { + throw InvalidInputException("Unsupported Zarr v3 bytes codec endian in %s: %s", metadata_path, + endian); + } + } + continue; + } + if (codec_name == "sharding_indexed") { + supports_cells = false; + compressor = JsonToString(codec_value); + cells_error = + "zarr_cells does not yet support Zarr v3 sharding_indexed codec pipelines (including inner Blosc)"; + continue; + } + if (codec_name == "gzip") { + if (!saw_bytes) { + throw InvalidInputException("Zarr v3 gzip codec must appear after bytes in %s", metadata_path); + } + if (!compressor.empty()) { + throw InvalidInputException("Only one Zarr v3 compression codec is currently supported: %s", + metadata_path); + } + if (!codec_config) { + compressor = "{\"id\":\"gzip\"}"; + continue; + } + auto level_value = yyjson_obj_get(codec_config, "level"); + if (!level_value) { + compressor = "{\"id\":\"gzip\"}"; + continue; + } + if (!yyjson_is_uint(level_value) && !yyjson_is_sint(level_value)) { + throw InvalidInputException("Zarr v3 gzip level must be an integer in %s", metadata_path); + } + auto level = yyjson_is_uint(level_value) ? NumericCast(unsafe_yyjson_get_uint(level_value)) + : unsafe_yyjson_get_sint(level_value); + compressor = "{\"id\":\"gzip\",\"level\":" + std::to_string(level) + "}"; + continue; + } + supports_cells = false; + compressor = JsonToString(codec_value); + cells_error = "zarr_cells does not yet support Zarr v3 codec: " + codec_name; + } + if (!saw_bytes && supports_cells) { + throw InvalidInputException("Zarr v3 arrays currently require a bytes codec in %s", metadata_path); + } + if (!supports_cells && cells_error.empty()) { + cells_error = "zarr_cells does not yet support the Zarr v3 codec pipeline for this array"; + } + + return MakeArrayEntry(store_path, relative_path, zarr_format, NumericCast(shape.size()), std::move(shape), + std::move(chunks), ParseV3DataType(root, metadata_path, little_endian), std::move(order), + std::move(compressor), JsonToString(yyjson_obj_get(root, "fill_value")), + std::move(chunk_key_encoding_name), std::move(dimension_separator), metadata_path, + supports_cells, std::move(cells_error), false, {}, {}, "", "", ""); } static string MetadataPathForKey(const string &store_path, const string &key) { @@ -1084,8 +1778,20 @@ static string RelativePathFromMetadataKey(const string &key, const string &suffi return key.substr(0, key.size() - key_suffix.size()); } -static string ChunkKeyFromCoords(const vector &coords, const string &dimension_separator) { +static string ChunkKeyFromCoords(const vector &coords, const string &chunk_key_encoding, + const string &dimension_separator) { string separator = dimension_separator == "/" ? "/" : "."; + if (chunk_key_encoding == "default") { + if (coords.empty()) { + return "c"; + } + string result = "c"; + for (idx_t i = 0; i < coords.size(); i++) { + result += separator; + result += std::to_string(coords[i]); + } + return result; + } string result; for (idx_t i = 0; i < coords.size(); i++) { if (i > 0) { @@ -1107,11 +1813,37 @@ static string ChunkPath(const string &store_path, const string &array_path, cons return path; } +static idx_t FlattenCoordsCOrder(const vector &coords, const vector &shape) { + if (coords.size() != shape.size()) { + throw InternalException("Coordinate rank mismatch"); + } + idx_t linear_index = 0; + for (idx_t i = 0; i < coords.size(); i++) { + auto dim = NumericCast(shape[i]); + auto coord = NumericCast(coords[i]); + if (coord >= dim) { + throw InternalException("Coordinate out of bounds"); + } + linear_index = linear_index * dim + coord; + } + return linear_index; +} + +static vector ShardInnerChunksPerShard(const ZarrArrayEntry &array) { + auto &storage_chunks = array.storage_chunks.empty() ? array.chunks : array.storage_chunks; + vector chunks_per_shard; + chunks_per_shard.reserve(array.rank); + for (idx_t dim = 0; dim < NumericCast(array.rank); dim++) { + chunks_per_shard.push_back(storage_chunks[dim] / array.inner_chunks[dim]); + } + return chunks_per_shard; +} + static void GenerateChunkEntriesRecursive(FileSystem &fs, const string &store_path, const ZarrArrayEntry &array, vector &coords, idx_t dim, bool include_missing, vector &chunks) { if (dim == coords.size()) { - auto chunk_key = ChunkKeyFromCoords(coords, array.dimension_separator); + auto chunk_key = ChunkKeyFromCoords(coords, array.chunk_key_encoding, array.dimension_separator); auto file_path = ChunkPath(store_path, array.array_path, chunk_key); auto file_exists = fs.FileExists(file_path); if (!file_exists && !include_missing) { @@ -1122,7 +1854,8 @@ static void GenerateChunkEntriesRecursive(FileSystem &fs, const string &store_pa auto handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_READ); file_size = NumericCast(handle->GetFileSize()); } - chunks.push_back({store_path, array.array_path, chunk_key, coords, file_path, file_size, file_exists}); + chunks.push_back( + {store_path, array.array_path, chunk_key, coords, file_path, file_size, file_exists, false, {}, {}}); return; } auto chunk_count = (array.shape[dim] + array.chunks[dim] - 1) / array.chunks[dim]; @@ -1133,16 +1866,59 @@ static void GenerateChunkEntriesRecursive(FileSystem &fs, const string &store_pa } static vector GenerateChunkEntries(FileSystem &fs, const string &store_path, - const ZarrArrayEntry &array, bool include_missing = false) { + const ZarrArrayEntry &array, bool include_missing) { vector chunks; vector coords(NumericCast(array.rank), 0); GenerateChunkEntriesRecursive(fs, store_path, array, coords, 0, include_missing, chunks); return chunks; } -static bool DiscoverConsolidatedStore(FileSystem &fs, const string &store_path, vector &groups, - vector &arrays, vector &chunks, - bool collect_chunks) { +static void GenerateCellChunkEntriesRecursive(FileSystem &fs, const string &store_path, const ZarrArrayEntry &array, + const vector &chunks_per_shard, vector &logical_coords, + idx_t dim, bool include_missing, vector &chunks) { + if (dim == logical_coords.size()) { + vector shard_coords(logical_coords.size(), 0); + vector inner_coords(logical_coords.size(), 0); + for (idx_t i = 0; i < logical_coords.size(); i++) { + shard_coords[i] = logical_coords[i] / chunks_per_shard[i]; + inner_coords[i] = logical_coords[i] % chunks_per_shard[i]; + } + auto storage_chunk_key = ChunkKeyFromCoords(shard_coords, array.chunk_key_encoding, array.dimension_separator); + auto file_path = ChunkPath(store_path, array.array_path, storage_chunk_key); + auto file_exists = fs.FileExists(file_path); + if (!file_exists && !include_missing) { + return; + } + int64_t file_size = 0; + if (file_exists) { + auto handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_READ); + file_size = NumericCast(handle->GetFileSize()); + } + chunks.push_back({store_path, array.array_path, storage_chunk_key, logical_coords, file_path, file_size, + file_exists, true, std::move(shard_coords), std::move(inner_coords)}); + return; + } + auto chunk_count = (array.shape[dim] + array.inner_chunks[dim] - 1) / array.inner_chunks[dim]; + for (int64_t chunk_coord = 0; chunk_coord < chunk_count; chunk_coord++) { + logical_coords[dim] = chunk_coord; + GenerateCellChunkEntriesRecursive(fs, store_path, array, chunks_per_shard, logical_coords, dim + 1, + include_missing, chunks); + } +} + +static vector GenerateCellChunkEntries(FileSystem &fs, const string &store_path, + const ZarrArrayEntry &array, bool include_missing) { + auto chunks_per_shard = ShardInnerChunksPerShard(array); + vector chunks; + vector logical_coords(NumericCast(array.rank), 0); + GenerateCellChunkEntriesRecursive(fs, store_path, array, chunks_per_shard, logical_coords, 0, include_missing, + chunks); + return chunks; +} + +static bool DiscoverConsolidatedStoreV2(FileSystem &fs, const string &store_path, vector &groups, + vector &arrays, vector &chunks, + bool collect_chunks) { auto metadata_path = store_path + "/.zmetadata"; if (!fs.FileExists(metadata_path)) { return false; @@ -1175,7 +1951,7 @@ static bool DiscoverConsolidatedStore(FileSystem &fs, const string &store_path, auto array = ParseArrayMetadataObject(entry_value, store_path, relative_path, MetadataPathForKey(store_path, key)); if (collect_chunks) { - auto array_chunks = GenerateChunkEntries(fs, store_path, array); + auto array_chunks = GenerateChunkEntries(fs, store_path, array, false); chunks.insert(chunks.end(), array_chunks.begin(), array_chunks.end()); } arrays.push_back(std::move(array)); @@ -1184,6 +1960,66 @@ static bool DiscoverConsolidatedStore(FileSystem &fs, const string &store_path, return true; } +static bool HasV2RootMarker(FileSystem &fs, const string &store_path) { + return fs.FileExists(store_path + "/.zmetadata") || fs.FileExists(store_path + "/.zgroup") || + fs.FileExists(store_path + "/.zarray"); +} + +static bool HasV3RootMarker(FileSystem &fs, const string &store_path) { + return fs.FileExists(store_path + "/zarr.json"); +} + +static bool DiscoverStoreV2(FileSystem &fs, const string &store_path, bool is_remote, vector &groups, + vector &arrays, vector &chunks, bool collect_chunks) { + if (DiscoverConsolidatedStoreV2(fs, store_path, groups, arrays, chunks, collect_chunks)) { + return true; + } + if (is_remote) { + if (HasV2RootMarker(fs, store_path)) { + throw InvalidInputException( + "Remote Zarr v2 stores currently require consolidated metadata (.zmetadata): %s", store_path); + } + return false; + } + if (!fs.DirectoryExists(store_path)) { + return false; + } + TraverseStoreV2(fs, store_path, store_path, "", collect_chunks, groups, arrays, chunks); + return !groups.empty() || !arrays.empty(); +} + +static bool DiscoverStoreV3(FileSystem &fs, const string &store_path, bool is_remote, vector &groups, + vector &arrays, vector &chunks, bool collect_chunks) { + if (!HasV3RootMarker(fs, store_path)) { + return false; + } + if (is_remote) { + auto metadata_path = store_path + "/zarr.json"; + auto metadata_text = ReadTextFile(fs, metadata_path); + unique_ptr doc( + yyjson_read(metadata_text.c_str(), metadata_text.size(), 0), yyjson_doc_free); + if (!doc) { + throw InvalidInputException("Failed to parse %s", metadata_path); + } + auto root = yyjson_doc_get_root(doc.get()); + auto node_type = RequireNodeType(root, metadata_path); + if (node_type == "array") { + auto entry = ParseArrayMetadataObjectV3(root, store_path, "", metadata_path); + if (collect_chunks) { + auto array_chunks = GenerateChunkEntries(fs, store_path, entry, false); + chunks.insert(chunks.end(), array_chunks.begin(), array_chunks.end()); + } + arrays.push_back(std::move(entry)); + return true; + } + throw InvalidInputException("Remote non-consolidated Zarr v3 group discovery is not yet supported; query a " + "specific array path or use a local store: %s", + store_path); + } + TraverseStoreV3(fs, store_path, store_path, "", collect_chunks, groups, arrays, chunks); + return true; +} + static void EnsureRemoteFilesystemSupport(ClientContext &context, const string &path) { if (!FileSystem::IsRemoteFile(path)) { return; @@ -1194,43 +2030,41 @@ static void EnsureRemoteFilesystemSupport(ClientContext &context, const string & } } -static vector DiscoverGroups(ClientContext &context, const string &path) { - FileSystem &fs = FileSystem::GetFileSystem(context); - EnsureRemoteFilesystemSupport(context, path); - auto store_path = NormalizeStorePath(fs, path); - vector groups; - vector arrays; - vector chunks; - if (!DiscoverConsolidatedStore(fs, store_path, groups, arrays, chunks, false)) { - if (FileSystem::IsRemoteFile(path)) { - throw InvalidInputException("Remote Zarr stores currently require consolidated metadata (.zmetadata): %s", - store_path); - } - if (!fs.DirectoryExists(store_path)) { - throw InvalidInputException("Zarr store path does not exist or is not a directory: %s", store_path); - } - TraverseStore(fs, store_path, store_path, "", false, groups, arrays, chunks); - } - std::sort(groups.begin(), groups.end(), [](const ZarrGroupEntry &lhs, const ZarrGroupEntry &rhs) { - return std::tie(lhs.group_path, lhs.metadata_path) < std::tie(rhs.group_path, rhs.metadata_path); - }); - return groups; -} - static void DiscoverStore(ClientContext &context, const string &path, vector &groups, - vector &arrays, vector &chunks, bool collect_chunks) { + vector &arrays, vector &chunks, bool collect_chunks, + ZarrVersionOverride version_override) { FileSystem &fs = FileSystem::GetFileSystem(context); EnsureRemoteFilesystemSupport(context, path); auto store_path = NormalizeStorePath(fs, path); - if (!DiscoverConsolidatedStore(fs, store_path, groups, arrays, chunks, collect_chunks)) { - if (FileSystem::IsRemoteFile(path)) { - throw InvalidInputException("Remote Zarr stores currently require consolidated metadata (.zmetadata): %s", - store_path); - } - if (!fs.DirectoryExists(store_path)) { - throw InvalidInputException("Zarr store path does not exist or is not a directory: %s", store_path); - } - TraverseStore(fs, store_path, store_path, "", collect_chunks, groups, arrays, chunks); + auto is_remote = FileSystem::IsRemoteFile(path); + auto has_v2_root = HasV2RootMarker(fs, store_path); + auto has_v3_root = HasV3RootMarker(fs, store_path); + bool discovered = false; + switch (version_override) { + case ZarrVersionOverride::AUTO: + if (has_v2_root && has_v3_root) { + throw InvalidInputException( + "Zarr store contains both v2 and v3 root metadata markers; use an explicit version override: %s", + store_path); + } + discovered = DiscoverStoreV3(fs, store_path, is_remote, groups, arrays, chunks, collect_chunks) || + DiscoverStoreV2(fs, store_path, is_remote, groups, arrays, chunks, collect_chunks); + break; + case ZarrVersionOverride::V2: + discovered = DiscoverStoreV2(fs, store_path, is_remote, groups, arrays, chunks, collect_chunks); + break; + case ZarrVersionOverride::V3: + discovered = DiscoverStoreV3(fs, store_path, is_remote, groups, arrays, chunks, collect_chunks); + break; + default: + throw InternalException("Unexpected zarr version override"); + } + if (!discovered) { + auto version_label = version_override == ZarrVersionOverride::V2 ? "v2" + : version_override == ZarrVersionOverride::V3 ? "v3" + : "v2 or v3"; + throw InvalidInputException("Zarr store path does not contain recognizable %s metadata: %s", version_label, + store_path); } std::sort(groups.begin(), groups.end(), [](const ZarrGroupEntry &lhs, const ZarrGroupEntry &rhs) { return std::tie(lhs.group_path, lhs.metadata_path) < std::tie(rhs.group_path, rhs.metadata_path); @@ -1248,28 +2082,39 @@ static unique_ptr ZarrInit(ClientContext &, TableFunct return make_uniq(); } -static unique_ptr BindGroups(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { - if (input.inputs.empty() || input.inputs[0].IsNull()) { - throw BinderException("zarr_groups requires a non-NULL store path"); - } +static unique_ptr BindGroupsInternal(ClientContext &context, const string &store_path, + ZarrVersionOverride version_override, + vector &return_types, vector &names) { + vector groups; + vector arrays; + vector chunks; + DiscoverStore(context, store_path, groups, arrays, chunks, false, version_override); names = {"store_path", "group_path", "zarr_format", "metadata_path"}; return_types = {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::BIGINT, LogicalType::VARCHAR}; - return make_uniq>(DiscoverGroups(context, StringValue::Get(input.inputs[0]))); + return make_uniq>(std::move(groups)); } -static unique_ptr BindArrays(ClientContext &context, TableFunctionBindInput &input, +static unique_ptr BindGroups(ClientContext &context, TableFunctionBindInput &input, vector &return_types, vector &names) { if (input.inputs.empty() || input.inputs[0].IsNull()) { - throw BinderException("zarr_arrays requires a non-NULL store path"); + throw BinderException("zarr_groups requires a non-NULL store path"); } + auto version_override = + input.inputs.size() > 1 ? ParseZarrVersionOverride(input.inputs[1], "zarr_groups") : ZarrVersionOverride::AUTO; + return BindGroupsInternal(context, StringValue::Get(input.inputs[0]), version_override, return_types, names); +} + +static unique_ptr BindArraysInternal(ClientContext &context, const string &store_path, + ZarrVersionOverride version_override, + vector &return_types, vector &names) { vector groups; vector arrays; vector chunks; - DiscoverStore(context, StringValue::Get(input.inputs[0]), groups, arrays, chunks, false); + DiscoverStore(context, store_path, groups, arrays, chunks, false, version_override); - names = {"store_path", "array_path", "zarr_format", "rank", "shape", "chunk_shape", "dtype", - "order", "compressor", "dimension_separator", "metadata_path"}; + names = {"store_path", "array_path", "zarr_format", "rank", "shape", + "chunk_shape", "dtype", "order", "compressor", "chunk_key_encoding", + "dimension_separator", "metadata_path"}; return_types = {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::BIGINT, @@ -1280,19 +2125,28 @@ static unique_ptr BindArrays(ClientContext &context, TableFunction LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, + LogicalType::VARCHAR, LogicalType::VARCHAR}; return make_uniq>(std::move(arrays)); } -static unique_ptr BindChunks(ClientContext &context, TableFunctionBindInput &input, +static unique_ptr BindArrays(ClientContext &context, TableFunctionBindInput &input, vector &return_types, vector &names) { if (input.inputs.empty() || input.inputs[0].IsNull()) { - throw BinderException("zarr_chunks requires a non-NULL store path"); + throw BinderException("zarr_arrays requires a non-NULL store path"); } + auto version_override = + input.inputs.size() > 1 ? ParseZarrVersionOverride(input.inputs[1], "zarr_arrays") : ZarrVersionOverride::AUTO; + return BindArraysInternal(context, StringValue::Get(input.inputs[0]), version_override, return_types, names); +} + +static unique_ptr BindChunksInternal(ClientContext &context, const string &store_path, + ZarrVersionOverride version_override, + vector &return_types, vector &names) { vector groups; vector arrays; vector chunks; - DiscoverStore(context, StringValue::Get(input.inputs[0]), groups, arrays, chunks, true); + DiscoverStore(context, store_path, groups, arrays, chunks, true, version_override); names = {"store_path", "array_path", "chunk_key", "chunk_coords", "file_path", "file_size_bytes"}; return_types = {LogicalType::VARCHAR, LogicalType::VARCHAR, @@ -1301,28 +2155,49 @@ static unique_ptr BindChunks(ClientContext &context, TableFunction return make_uniq>(std::move(chunks)); } +static unique_ptr BindChunks(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + if (input.inputs.empty() || input.inputs[0].IsNull()) { + throw BinderException("zarr_chunks requires a non-NULL store path"); + } + auto version_override = + input.inputs.size() > 1 ? ParseZarrVersionOverride(input.inputs[1], "zarr_chunks") : ZarrVersionOverride::AUTO; + return BindChunksInternal(context, StringValue::Get(input.inputs[0]), version_override, return_types, names); +} + static unique_ptr BindCellsInternal(ClientContext &context, const string &store_path, - const string &array_path, vector &return_types, - vector &names) { + const string &array_path, ZarrVersionOverride version_override, + vector &return_types, vector &names) { vector groups; vector arrays; vector chunks; - DiscoverStore(context, store_path, groups, arrays, chunks, false); + DiscoverStore(context, store_path, groups, arrays, chunks, false, version_override); auto &array = FindArrayEntry(arrays, array_path); + if (!array.supports_cells) { + throw InvalidInputException("%s: %s", array.cells_error, array.array_path); + } + auto cell_array = array; auto dtype = ParseNumericDType(array.dtype); - chunks = GenerateChunkEntries(FileSystem::GetFileSystem(context), array.store_path, array, - HasMaterializedFillValue(array)); - - names.reserve(array.rank + 1); - return_types.reserve(array.rank + 1); - for (idx_t dim = 0; dim < NumericCast(array.rank); dim++) { + chunks = cell_array.is_sharding_indexed + ? GenerateCellChunkEntries(FileSystem::GetFileSystem(context), array.store_path, array, + HasMaterializedFillValue(array)) + : GenerateChunkEntries(FileSystem::GetFileSystem(context), cell_array.store_path, cell_array, + HasMaterializedFillValue(cell_array)); + if (cell_array.is_sharding_indexed) { + cell_array.storage_chunks = cell_array.chunks; + cell_array.chunks = cell_array.inner_chunks; + } + + names.reserve(cell_array.rank + 1); + return_types.reserve(cell_array.rank + 1); + for (idx_t dim = 0; dim < NumericCast(cell_array.rank); dim++) { names.push_back("dim_" + std::to_string(dim)); return_types.push_back(LogicalType::BIGINT); } names.push_back("value"); return_types.push_back(dtype.logical_type); - return make_uniq(array, std::move(chunks)); + return make_uniq(cell_array, std::move(chunks)); } static unique_ptr BindCells(ClientContext &context, TableFunctionBindInput &input, @@ -1330,13 +2205,20 @@ static unique_ptr BindCells(ClientContext &context, TableFunctionB if (input.inputs.size() < 2 || input.inputs[0].IsNull() || input.inputs[1].IsNull()) { throw BinderException("zarr_cells requires a non-NULL store path and array path"); } + auto version_override = + input.inputs.size() > 2 ? ParseZarrVersionOverride(input.inputs[2], "zarr_cells") : ZarrVersionOverride::AUTO; return BindCellsInternal(context, StringValue::Get(input.inputs[0]), - NormalizeArrayPath(StringValue::Get(input.inputs[1])), return_types, names); + NormalizeArrayPath(StringValue::Get(input.inputs[1])), version_override, return_types, + names); } static unique_ptr BindZarr(ClientContext &context, TableFunctionBindInput &input, vector &return_types, vector &names) { - return BindArrays(context, input, return_types, names); + if (input.inputs.empty() || input.inputs[0].IsNull()) { + throw BinderException("zarr requires a non-NULL store path"); + } + return BindArraysInternal(context, StringValue::Get(input.inputs[0]), ZarrVersionOverride::AUTO, return_types, + names); } static void ScanGroups(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { @@ -1363,14 +2245,15 @@ static void ScanArrays(ClientContext &, TableFunctionInput &data_p, DataChunk &o output.SetValue(0, count, Value(entry.store_path)); output.SetValue(1, count, Value(entry.array_path)); output.SetValue(2, count, Value::BIGINT(entry.zarr_format)); - output.SetValue(3, count, Value::BIGINT(entry.rank)); + output.SetValue(3, count, Value::BIGINT(NumericCast(entry.shape.size()))); output.SetValue(4, count, Value::LIST(LogicalType::BIGINT, ToBigIntValues(entry.shape))); output.SetValue(5, count, Value::LIST(LogicalType::BIGINT, ToBigIntValues(entry.chunks))); output.SetValue(6, count, Value(entry.dtype)); output.SetValue(7, count, Value(entry.order)); output.SetValue(8, count, Value(entry.compressor)); - output.SetValue(9, count, Value(entry.dimension_separator)); - output.SetValue(10, count, Value(entry.metadata_path)); + output.SetValue(9, count, Value(entry.chunk_key_encoding)); + output.SetValue(10, count, Value(entry.dimension_separator)); + output.SetValue(11, count, Value(entry.metadata_path)); count++; } output.SetCardinality(count); @@ -1451,36 +2334,64 @@ static unique_ptr InitCells(ClientContext &context, Ta return std::move(result); } -TableFunction ZarrMetadata::GetGroupsFunction() { - return TableFunction("zarr_groups", {LogicalType::VARCHAR}, ScanGroups, BindGroups, ZarrInit); -} - -TableFunction ZarrMetadata::GetArraysFunction() { - return TableFunction("zarr_arrays", {LogicalType::VARCHAR}, ScanArrays, BindArrays, ZarrInit); -} - -TableFunction ZarrMetadata::GetChunksFunction() { - return TableFunction("zarr_chunks", {LogicalType::VARCHAR}, ScanChunks, BindChunks, ZarrInit); -} - -TableFunction ZarrMetadata::GetCellsFunction() { - TableFunction function("zarr_cells", {LogicalType::VARCHAR, LogicalType::VARCHAR}, ScanCells, BindCells, InitCells); - function.projection_pushdown = true; - function.filter_pushdown = true; - function.filter_prune = true; - return function; -} - -TableFunction ZarrMetadata::GetZarrFunction() { - return TableFunction("zarr", {LogicalType::VARCHAR}, ScanArrays, BindZarr, ZarrInit); -} - -TableFunction ZarrMetadata::GetZarrCellsAliasFunction() { - TableFunction function("zarr", {LogicalType::VARCHAR, LogicalType::VARCHAR}, ScanCells, BindCells, InitCells); - function.projection_pushdown = true; - function.filter_pushdown = true; - function.filter_prune = true; - return function; +TableFunctionSet ZarrMetadata::GetGroupsFunction() { + TableFunctionSet set("zarr_groups"); + set.AddFunction(TableFunction("zarr_groups", {LogicalType::VARCHAR}, ScanGroups, BindGroups, ZarrInit)); + set.AddFunction( + TableFunction("zarr_groups", {LogicalType::VARCHAR, LogicalType::VARCHAR}, ScanGroups, BindGroups, ZarrInit)); + return set; +} + +TableFunctionSet ZarrMetadata::GetArraysFunction() { + TableFunctionSet set("zarr_arrays"); + set.AddFunction(TableFunction("zarr_arrays", {LogicalType::VARCHAR}, ScanArrays, BindArrays, ZarrInit)); + set.AddFunction( + TableFunction("zarr_arrays", {LogicalType::VARCHAR, LogicalType::VARCHAR}, ScanArrays, BindArrays, ZarrInit)); + return set; +} + +TableFunctionSet ZarrMetadata::GetChunksFunction() { + TableFunctionSet set("zarr_chunks"); + set.AddFunction(TableFunction("zarr_chunks", {LogicalType::VARCHAR}, ScanChunks, BindChunks, ZarrInit)); + set.AddFunction( + TableFunction("zarr_chunks", {LogicalType::VARCHAR, LogicalType::VARCHAR}, ScanChunks, BindChunks, ZarrInit)); + return set; +} + +TableFunctionSet ZarrMetadata::GetCellsFunction() { + TableFunctionSet set("zarr_cells"); + TableFunction base("zarr_cells", {LogicalType::VARCHAR, LogicalType::VARCHAR}, ScanCells, BindCells, InitCells); + base.projection_pushdown = true; + base.filter_pushdown = true; + base.filter_prune = true; + set.AddFunction(base); + TableFunction with_override("zarr_cells", {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}, + ScanCells, BindCells, InitCells); + with_override.projection_pushdown = true; + with_override.filter_pushdown = true; + with_override.filter_prune = true; + set.AddFunction(with_override); + return set; +} + +TableFunctionSet ZarrMetadata::GetZarrFunction() { + return TableFunctionSet(TableFunction("zarr", {LogicalType::VARCHAR}, ScanArrays, BindZarr, ZarrInit)); +} + +TableFunctionSet ZarrMetadata::GetZarrCellsAliasFunction() { + TableFunctionSet set("zarr"); + TableFunction base("zarr", {LogicalType::VARCHAR, LogicalType::VARCHAR}, ScanCells, BindCells, InitCells); + base.projection_pushdown = true; + base.filter_pushdown = true; + base.filter_prune = true; + set.AddFunction(base); + TableFunction with_override("zarr", {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}, ScanCells, + BindCells, InitCells); + with_override.projection_pushdown = true; + with_override.filter_pushdown = true; + with_override.filter_prune = true; + set.AddFunction(with_override); + return set; } } // namespace duckdb diff --git a/test/README.md b/test/README.md index 1a7af3d..2590db3 100644 --- a/test/README.md +++ b/test/README.md @@ -8,13 +8,17 @@ The current suite covers the metadata and `zarr_cells` slices: - `zarr_arrays(path)` - `zarr_chunks(path)` - `zarr_cells(path, array_path)` +- `test/python_smoke.py` for a Python-side smoke example using DuckDB 1.5.0 +- real OME-Zarr v3 sharded/Blosc cell scans against `test/data/idr0062A/6001240_labels.zarr` The checked-in fixtures used by `test/sql/duckdb_zarr.test` are: - `test/data/simple_v2.zarr` +- `test/data/simple_v3.zarr` - `test/data/ome_example.ome.zarr` +- `test/data/idr0062A/6001240_labels.zarr` -Both fixtures can be regenerated with: +These fixtures can be regenerated with: ```bash make fixture @@ -31,3 +35,9 @@ or ```bash make test_debug ``` + +The Python smoke example can be run with: + +```bash +uvx --with duckdb==1.5.0 --with numpy --with pandas python test/python_smoke.py +``` diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/0/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/0/0/0 new file mode 100644 index 0000000..02ca7a7 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/0/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/1/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/1/0/0 new file mode 100644 index 0000000..2b7a366 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/1/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/10/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/10/0/0 new file mode 100644 index 0000000..40e6acf Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/10/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/11/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/11/0/0 new file mode 100644 index 0000000..cd77daf Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/11/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/12/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/12/0/0 new file mode 100644 index 0000000..5ec34b3 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/12/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/13/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/13/0/0 new file mode 100644 index 0000000..f776774 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/13/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/14/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/14/0/0 new file mode 100644 index 0000000..fc0e623 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/14/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/15/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/15/0/0 new file mode 100644 index 0000000..d340e3f Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/15/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/16/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/16/0/0 new file mode 100644 index 0000000..b400a54 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/16/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/17/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/17/0/0 new file mode 100644 index 0000000..7701433 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/17/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/18/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/18/0/0 new file mode 100644 index 0000000..fd435e2 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/18/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/19/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/19/0/0 new file mode 100644 index 0000000..7a3d34e Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/19/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/2/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/2/0/0 new file mode 100644 index 0000000..5abc6b8 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/2/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/20/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/20/0/0 new file mode 100644 index 0000000..ff3754c Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/20/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/21/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/21/0/0 new file mode 100644 index 0000000..2f46fed Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/21/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/22/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/22/0/0 new file mode 100644 index 0000000..4dc7354 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/22/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/23/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/23/0/0 new file mode 100644 index 0000000..d265988 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/23/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/3/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/3/0/0 new file mode 100644 index 0000000..f3c6f9e Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/3/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/4/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/4/0/0 new file mode 100644 index 0000000..2ef8cfe Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/4/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/5/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/5/0/0 new file mode 100644 index 0000000..4a059ba Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/5/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/6/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/6/0/0 new file mode 100644 index 0000000..e54f628 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/6/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/7/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/7/0/0 new file mode 100644 index 0000000..b693404 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/7/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/8/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/8/0/0 new file mode 100644 index 0000000..457627e Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/8/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/0/9/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/0/9/0/0 new file mode 100644 index 0000000..9a5b24f Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/0/9/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/0/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/0/0/0 new file mode 100644 index 0000000..e880fdc Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/0/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/1/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/1/0/0 new file mode 100644 index 0000000..486b334 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/1/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/10/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/10/0/0 new file mode 100644 index 0000000..3416fd6 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/10/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/11/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/11/0/0 new file mode 100644 index 0000000..40e1375 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/11/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/12/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/12/0/0 new file mode 100644 index 0000000..d5ef5ef Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/12/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/13/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/13/0/0 new file mode 100644 index 0000000..8240411 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/13/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/14/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/14/0/0 new file mode 100644 index 0000000..f6c026f Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/14/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/15/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/15/0/0 new file mode 100644 index 0000000..b4066f1 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/15/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/16/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/16/0/0 new file mode 100644 index 0000000..b9eb31d Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/16/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/17/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/17/0/0 new file mode 100644 index 0000000..256bb98 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/17/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/18/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/18/0/0 new file mode 100644 index 0000000..e08d85a Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/18/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/19/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/19/0/0 new file mode 100644 index 0000000..d522d01 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/19/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/2/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/2/0/0 new file mode 100644 index 0000000..6fd0b57 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/2/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/20/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/20/0/0 new file mode 100644 index 0000000..d3730f1 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/20/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/21/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/21/0/0 new file mode 100644 index 0000000..0bcc299 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/21/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/22/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/22/0/0 new file mode 100644 index 0000000..96b981c Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/22/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/23/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/23/0/0 new file mode 100644 index 0000000..bb535fc Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/23/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/3/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/3/0/0 new file mode 100644 index 0000000..5201a0e Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/3/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/4/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/4/0/0 new file mode 100644 index 0000000..a87a4e0 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/4/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/5/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/5/0/0 new file mode 100644 index 0000000..913d6ae Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/5/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/6/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/6/0/0 new file mode 100644 index 0000000..dbcf98a Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/6/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/7/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/7/0/0 new file mode 100644 index 0000000..43d5338 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/7/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/8/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/8/0/0 new file mode 100644 index 0000000..361a97c Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/8/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/c/1/9/0/0 b/test/data/idr0062A/6001240_labels.zarr/0/c/1/9/0/0 new file mode 100644 index 0000000..44c1d09 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/0/c/1/9/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/0/zarr.json b/test/data/idr0062A/6001240_labels.zarr/0/zarr.json new file mode 100644 index 0000000..0f5f955 --- /dev/null +++ b/test/data/idr0062A/6001240_labels.zarr/0/zarr.json @@ -0,0 +1 @@ +{"chunk_grid":{"configuration":{"chunk_shape":[1,10,512,512]},"name":"regular"},"chunk_key_encoding":{"name":"default"},"codecs":[{"configuration":{"chunk_shape":[1,1,256,256],"codecs":[{"configuration":{"endian":"little"},"name":"bytes"},{"configuration":{"blocksize":0,"clevel":5,"cname":"zstd","shuffle":"shuffle","typesize":2},"name":"blosc"}],"index_codecs":[{"configuration":{"endian":"little"},"name":"bytes"},{"name":"crc32c"}]},"name":"sharding_indexed"}],"data_type":"uint16","dimension_names":["c","z","y","x"],"fill_value":0,"node_type":"array","shape":[2,236,275,271],"zarr_format":3} diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/0/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/0/0/0 new file mode 100644 index 0000000..8bf0f8d Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/0/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/1/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/1/0/0 new file mode 100644 index 0000000..6c31e9c Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/1/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/10/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/10/0/0 new file mode 100644 index 0000000..dcd8b3e Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/10/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/11/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/11/0/0 new file mode 100644 index 0000000..bdd532c Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/11/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/12/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/12/0/0 new file mode 100644 index 0000000..cbca47b Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/12/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/13/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/13/0/0 new file mode 100644 index 0000000..6ee44b8 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/13/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/14/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/14/0/0 new file mode 100644 index 0000000..1a5d652 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/14/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/15/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/15/0/0 new file mode 100644 index 0000000..5d2b976 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/15/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/16/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/16/0/0 new file mode 100644 index 0000000..6ac580d Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/16/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/17/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/17/0/0 new file mode 100644 index 0000000..ee4f667 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/17/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/18/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/18/0/0 new file mode 100644 index 0000000..2d7d270 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/18/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/19/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/19/0/0 new file mode 100644 index 0000000..7ce1822 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/19/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/2/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/2/0/0 new file mode 100644 index 0000000..59ffae4 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/2/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/20/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/20/0/0 new file mode 100644 index 0000000..9d3308d Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/20/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/21/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/21/0/0 new file mode 100644 index 0000000..e5bac16 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/21/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/22/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/22/0/0 new file mode 100644 index 0000000..360bf60 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/22/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/23/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/23/0/0 new file mode 100644 index 0000000..7d6a220 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/23/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/3/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/3/0/0 new file mode 100644 index 0000000..7a4586a Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/3/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/4/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/4/0/0 new file mode 100644 index 0000000..d7690e8 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/4/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/5/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/5/0/0 new file mode 100644 index 0000000..989ccfe Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/5/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/6/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/6/0/0 new file mode 100644 index 0000000..27cf142 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/6/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/7/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/7/0/0 new file mode 100644 index 0000000..32490a9 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/7/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/8/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/8/0/0 new file mode 100644 index 0000000..c467e5d Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/8/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/0/9/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/0/9/0/0 new file mode 100644 index 0000000..68c318d Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/0/9/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/0/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/0/0/0 new file mode 100644 index 0000000..43a16de Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/0/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/1/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/1/0/0 new file mode 100644 index 0000000..a67cca8 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/1/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/10/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/10/0/0 new file mode 100644 index 0000000..b4d02e2 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/10/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/11/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/11/0/0 new file mode 100644 index 0000000..840b3ab Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/11/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/12/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/12/0/0 new file mode 100644 index 0000000..13bf7d0 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/12/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/13/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/13/0/0 new file mode 100644 index 0000000..fc9aec4 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/13/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/14/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/14/0/0 new file mode 100644 index 0000000..cb74507 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/14/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/15/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/15/0/0 new file mode 100644 index 0000000..59e7bf8 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/15/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/16/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/16/0/0 new file mode 100644 index 0000000..39289c0 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/16/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/17/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/17/0/0 new file mode 100644 index 0000000..5d12a1e Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/17/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/18/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/18/0/0 new file mode 100644 index 0000000..cc26915 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/18/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/19/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/19/0/0 new file mode 100644 index 0000000..2b9b242 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/19/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/2/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/2/0/0 new file mode 100644 index 0000000..423cc27 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/2/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/20/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/20/0/0 new file mode 100644 index 0000000..f3b3e26 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/20/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/21/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/21/0/0 new file mode 100644 index 0000000..fc711e7 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/21/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/22/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/22/0/0 new file mode 100644 index 0000000..4cad8ee Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/22/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/23/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/23/0/0 new file mode 100644 index 0000000..d6005c8 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/23/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/3/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/3/0/0 new file mode 100644 index 0000000..ed187c0 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/3/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/4/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/4/0/0 new file mode 100644 index 0000000..68e60c0 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/4/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/5/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/5/0/0 new file mode 100644 index 0000000..bf981f8 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/5/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/6/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/6/0/0 new file mode 100644 index 0000000..5b71ef2 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/6/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/7/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/7/0/0 new file mode 100644 index 0000000..8ce9f16 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/7/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/8/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/8/0/0 new file mode 100644 index 0000000..6661a4d Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/8/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/c/1/9/0/0 b/test/data/idr0062A/6001240_labels.zarr/1/c/1/9/0/0 new file mode 100644 index 0000000..423c46c Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/1/c/1/9/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/1/zarr.json b/test/data/idr0062A/6001240_labels.zarr/1/zarr.json new file mode 100644 index 0000000..c86dd22 --- /dev/null +++ b/test/data/idr0062A/6001240_labels.zarr/1/zarr.json @@ -0,0 +1 @@ +{"chunk_grid":{"configuration":{"chunk_shape":[1,10,512,512]},"name":"regular"},"chunk_key_encoding":{"name":"default"},"codecs":[{"configuration":{"chunk_shape":[1,1,256,256],"codecs":[{"configuration":{"endian":"little"},"name":"bytes"},{"configuration":{"blocksize":0,"clevel":5,"cname":"zstd","shuffle":"shuffle","typesize":2},"name":"blosc"}],"index_codecs":[{"configuration":{"endian":"little"},"name":"bytes"},{"name":"crc32c"}]},"name":"sharding_indexed"}],"data_type":"uint16","dimension_names":["c","z","y","x"],"fill_value":0,"node_type":"array","shape":[2,236,137,135],"zarr_format":3} diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/0/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/0/0/0 new file mode 100644 index 0000000..2a1f8f8 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/0/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/1/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/1/0/0 new file mode 100644 index 0000000..952fc83 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/1/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/10/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/10/0/0 new file mode 100644 index 0000000..5b45b8b Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/10/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/11/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/11/0/0 new file mode 100644 index 0000000..d658e20 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/11/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/12/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/12/0/0 new file mode 100644 index 0000000..0864a6e Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/12/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/13/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/13/0/0 new file mode 100644 index 0000000..3985c83 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/13/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/14/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/14/0/0 new file mode 100644 index 0000000..e3c0142 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/14/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/15/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/15/0/0 new file mode 100644 index 0000000..06d5a65 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/15/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/16/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/16/0/0 new file mode 100644 index 0000000..60f3018 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/16/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/17/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/17/0/0 new file mode 100644 index 0000000..3651547 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/17/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/18/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/18/0/0 new file mode 100644 index 0000000..8fbe453 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/18/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/19/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/19/0/0 new file mode 100644 index 0000000..4a121cf Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/19/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/2/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/2/0/0 new file mode 100644 index 0000000..bee0412 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/2/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/20/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/20/0/0 new file mode 100644 index 0000000..3d4ac84 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/20/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/21/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/21/0/0 new file mode 100644 index 0000000..c2a0236 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/21/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/22/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/22/0/0 new file mode 100644 index 0000000..87a27f1 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/22/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/23/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/23/0/0 new file mode 100644 index 0000000..d6711b9 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/23/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/3/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/3/0/0 new file mode 100644 index 0000000..3b47508 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/3/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/4/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/4/0/0 new file mode 100644 index 0000000..f3b5c35 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/4/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/5/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/5/0/0 new file mode 100644 index 0000000..9c121fc Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/5/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/6/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/6/0/0 new file mode 100644 index 0000000..1ed3399 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/6/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/7/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/7/0/0 new file mode 100644 index 0000000..a06f9c4 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/7/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/8/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/8/0/0 new file mode 100644 index 0000000..3b7da07 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/8/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/0/9/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/0/9/0/0 new file mode 100644 index 0000000..ad69a96 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/0/9/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/0/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/0/0/0 new file mode 100644 index 0000000..e435c1a Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/0/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/1/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/1/0/0 new file mode 100644 index 0000000..7478b8d Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/1/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/10/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/10/0/0 new file mode 100644 index 0000000..d09d10e Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/10/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/11/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/11/0/0 new file mode 100644 index 0000000..9b4c689 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/11/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/12/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/12/0/0 new file mode 100644 index 0000000..b033a5d Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/12/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/13/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/13/0/0 new file mode 100644 index 0000000..7902601 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/13/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/14/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/14/0/0 new file mode 100644 index 0000000..8935ee8 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/14/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/15/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/15/0/0 new file mode 100644 index 0000000..abd0611 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/15/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/16/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/16/0/0 new file mode 100644 index 0000000..a3a34ec Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/16/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/17/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/17/0/0 new file mode 100644 index 0000000..7f6438d Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/17/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/18/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/18/0/0 new file mode 100644 index 0000000..f560d1d Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/18/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/19/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/19/0/0 new file mode 100644 index 0000000..87320f7 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/19/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/2/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/2/0/0 new file mode 100644 index 0000000..a810d10 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/2/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/20/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/20/0/0 new file mode 100644 index 0000000..f7bac60 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/20/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/21/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/21/0/0 new file mode 100644 index 0000000..1b19786 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/21/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/22/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/22/0/0 new file mode 100644 index 0000000..344dabf Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/22/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/23/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/23/0/0 new file mode 100644 index 0000000..8eaeff7 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/23/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/3/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/3/0/0 new file mode 100644 index 0000000..34354dc Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/3/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/4/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/4/0/0 new file mode 100644 index 0000000..a684fca Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/4/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/5/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/5/0/0 new file mode 100644 index 0000000..d555f82 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/5/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/6/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/6/0/0 new file mode 100644 index 0000000..60acebb Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/6/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/7/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/7/0/0 new file mode 100644 index 0000000..dec3c76 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/7/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/8/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/8/0/0 new file mode 100644 index 0000000..54265da Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/8/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/c/1/9/0/0 b/test/data/idr0062A/6001240_labels.zarr/2/c/1/9/0/0 new file mode 100644 index 0000000..a765e1a Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/2/c/1/9/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/2/zarr.json b/test/data/idr0062A/6001240_labels.zarr/2/zarr.json new file mode 100644 index 0000000..8997285 --- /dev/null +++ b/test/data/idr0062A/6001240_labels.zarr/2/zarr.json @@ -0,0 +1 @@ +{"chunk_grid":{"configuration":{"chunk_shape":[1,10,512,512]},"name":"regular"},"chunk_key_encoding":{"name":"default"},"codecs":[{"configuration":{"chunk_shape":[1,1,256,256],"codecs":[{"configuration":{"endian":"little"},"name":"bytes"},{"configuration":{"blocksize":0,"clevel":5,"cname":"zstd","shuffle":"shuffle","typesize":2},"name":"blosc"}],"index_codecs":[{"configuration":{"endian":"little"},"name":"bytes"},{"name":"crc32c"}]},"name":"sharding_indexed"}],"data_type":"uint16","dimension_names":["c","z","y","x"],"fill_value":0,"node_type":"array","shape":[2,236,68,67],"zarr_format":3} diff --git a/test/data/idr0062A/6001240_labels.zarr/6001240_labels.zarr/ro-crate-metadata.json b/test/data/idr0062A/6001240_labels.zarr/6001240_labels.zarr/ro-crate-metadata.json new file mode 100644 index 0000000..cf032f4 --- /dev/null +++ b/test/data/idr0062A/6001240_labels.zarr/6001240_labels.zarr/ro-crate-metadata.json @@ -0,0 +1,39 @@ +{ + "@context": [ + "https://w3id.org/ro/crate/1.1/context", + { + "organism_classification": "https://schema.org/taxonomicRange", + "BioChemEntity": "https://schema.org/BioChemEntity", + "channel": "https://www.openmicroscopy.org/Schemas/Documentation/Generated/OME-2016-06/ome_xsd.html#Channel", + "obo": "http://purl.obolibrary.org/obo/", + "FBcv": "http://ontobee.org/ontology/FBcv/", + "acquisiton_method": { + "@reverse": "https://schema.org/result", + "@type": "@id" + }, + "biological_entity": "https://schema.org/about", + "biosample": "http://purl.obolibrary.org/obo/OBI_0002648", + "preparation_method": "https://www.wikidata.org/wiki/Property:P1537", + "specimen": "http://purl.obolibrary.org/obo/HSO_0000308" + } + ], + "@graph": [ + { + "@id": "./", + "@type": "Dataset", + "name": "dataset name", + "description": "dataset description", + "licence": "https://creativecommons.org/licenses/by/4.0/" + }, + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "conformsTo": { + "@id": "https://w3id.org/ro/crate/1.1" + }, + "about": { + "@id": "./" + } + } + ] +} diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/10/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/10/0/0 new file mode 100644 index 0000000..d99d4ba Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/10/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/11/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/11/0/0 new file mode 100644 index 0000000..52d9cdf Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/11/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/12/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/12/0/0 new file mode 100644 index 0000000..9b5cd08 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/12/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/13/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/13/0/0 new file mode 100644 index 0000000..4a9eb2b Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/13/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/14/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/14/0/0 new file mode 100644 index 0000000..b63c5a9 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/14/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/15/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/15/0/0 new file mode 100644 index 0000000..71941a1 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/15/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/16/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/16/0/0 new file mode 100644 index 0000000..342e0fa Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/16/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/17/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/17/0/0 new file mode 100644 index 0000000..a3ebc13 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/17/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/18/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/18/0/0 new file mode 100644 index 0000000..95e19fe Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/18/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/19/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/19/0/0 new file mode 100644 index 0000000..8bd8d41 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/19/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/2/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/2/0/0 new file mode 100644 index 0000000..bdaea94 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/2/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/20/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/20/0/0 new file mode 100644 index 0000000..eff5c9f Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/20/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/21/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/21/0/0 new file mode 100644 index 0000000..c56b4d9 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/21/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/22/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/22/0/0 new file mode 100644 index 0000000..9cf75f7 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/22/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/23/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/23/0/0 new file mode 100644 index 0000000..9e64158 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/23/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/3/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/3/0/0 new file mode 100644 index 0000000..00cc1c4 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/3/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/4/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/4/0/0 new file mode 100644 index 0000000..ad2b392 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/4/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/5/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/5/0/0 new file mode 100644 index 0000000..b1b84d0 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/5/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/6/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/6/0/0 new file mode 100644 index 0000000..173675d Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/6/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/7/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/7/0/0 new file mode 100644 index 0000000..95c08d7 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/7/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/8/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/8/0/0 new file mode 100644 index 0000000..9226734 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/8/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/9/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/9/0/0 new file mode 100644 index 0000000..f422c4c Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/c/0/9/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/0/zarr.json b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/zarr.json new file mode 100644 index 0000000..cda58c5 --- /dev/null +++ b/test/data/idr0062A/6001240_labels.zarr/labels/0/0/zarr.json @@ -0,0 +1 @@ +{"chunk_grid":{"configuration":{"chunk_shape":[1,10,512,512]},"name":"regular"},"chunk_key_encoding":{"name":"default"},"codecs":[{"configuration":{"chunk_shape":[1,1,256,256],"codecs":[{"name":"bytes"},{"configuration":{"blocksize":0,"clevel":5,"cname":"zstd","shuffle":"bitshuffle","typesize":1},"name":"blosc"}],"index_codecs":[{"configuration":{"endian":"little"},"name":"bytes"},{"name":"crc32c"}]},"name":"sharding_indexed"}],"data_type":"int8","dimension_names":["c","z","y","x"],"fill_value":0,"node_type":"array","shape":[1,236,275,271],"zarr_format":3} diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/10/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/10/0/0 new file mode 100644 index 0000000..1501557 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/10/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/11/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/11/0/0 new file mode 100644 index 0000000..18914c1 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/11/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/12/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/12/0/0 new file mode 100644 index 0000000..6066a70 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/12/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/13/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/13/0/0 new file mode 100644 index 0000000..7817e7f Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/13/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/14/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/14/0/0 new file mode 100644 index 0000000..efd2520 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/14/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/15/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/15/0/0 new file mode 100644 index 0000000..03c95d3 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/15/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/16/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/16/0/0 new file mode 100644 index 0000000..3250386 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/16/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/17/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/17/0/0 new file mode 100644 index 0000000..4e7bc5f Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/17/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/18/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/18/0/0 new file mode 100644 index 0000000..20a4de4 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/18/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/19/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/19/0/0 new file mode 100644 index 0000000..6bc81c1 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/19/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/2/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/2/0/0 new file mode 100644 index 0000000..779d98e Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/2/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/20/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/20/0/0 new file mode 100644 index 0000000..f75cb6b Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/20/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/21/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/21/0/0 new file mode 100644 index 0000000..c2420fd Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/21/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/22/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/22/0/0 new file mode 100644 index 0000000..c014583 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/22/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/23/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/23/0/0 new file mode 100644 index 0000000..05382dd Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/23/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/3/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/3/0/0 new file mode 100644 index 0000000..6d6cc93 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/3/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/4/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/4/0/0 new file mode 100644 index 0000000..6deb5d7 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/4/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/5/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/5/0/0 new file mode 100644 index 0000000..30cb88f Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/5/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/6/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/6/0/0 new file mode 100644 index 0000000..66fb08e Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/6/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/7/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/7/0/0 new file mode 100644 index 0000000..0f65898 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/7/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/8/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/8/0/0 new file mode 100644 index 0000000..b38dcb5 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/8/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/9/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/9/0/0 new file mode 100644 index 0000000..de90fae Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/c/0/9/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/1/zarr.json b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/zarr.json new file mode 100644 index 0000000..141acfb --- /dev/null +++ b/test/data/idr0062A/6001240_labels.zarr/labels/0/1/zarr.json @@ -0,0 +1 @@ +{"chunk_grid":{"configuration":{"chunk_shape":[1,10,512,512]},"name":"regular"},"chunk_key_encoding":{"name":"default"},"codecs":[{"configuration":{"chunk_shape":[1,1,256,256],"codecs":[{"name":"bytes"},{"configuration":{"blocksize":0,"clevel":5,"cname":"zstd","shuffle":"bitshuffle","typesize":1},"name":"blosc"}],"index_codecs":[{"configuration":{"endian":"little"},"name":"bytes"},{"name":"crc32c"}]},"name":"sharding_indexed"}],"data_type":"int8","dimension_names":["c","z","y","x"],"fill_value":0,"node_type":"array","shape":[1,236,137,135],"zarr_format":3} diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/10/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/10/0/0 new file mode 100644 index 0000000..ea2dbe0 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/10/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/11/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/11/0/0 new file mode 100644 index 0000000..d7cf902 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/11/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/12/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/12/0/0 new file mode 100644 index 0000000..7b6365d Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/12/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/13/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/13/0/0 new file mode 100644 index 0000000..03d7c76 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/13/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/14/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/14/0/0 new file mode 100644 index 0000000..0161c5b Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/14/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/15/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/15/0/0 new file mode 100644 index 0000000..c138c1f Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/15/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/16/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/16/0/0 new file mode 100644 index 0000000..e9c9dec Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/16/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/17/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/17/0/0 new file mode 100644 index 0000000..2d0af72 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/17/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/18/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/18/0/0 new file mode 100644 index 0000000..b9d8d1a Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/18/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/19/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/19/0/0 new file mode 100644 index 0000000..e3ad0f9 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/19/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/2/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/2/0/0 new file mode 100644 index 0000000..d2de4fd Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/2/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/20/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/20/0/0 new file mode 100644 index 0000000..f694dc3 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/20/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/21/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/21/0/0 new file mode 100644 index 0000000..7ab4bcf Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/21/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/22/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/22/0/0 new file mode 100644 index 0000000..16e20ed Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/22/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/23/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/23/0/0 new file mode 100644 index 0000000..075a7aa Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/23/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/3/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/3/0/0 new file mode 100644 index 0000000..f5915c7 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/3/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/4/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/4/0/0 new file mode 100644 index 0000000..d326af0 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/4/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/5/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/5/0/0 new file mode 100644 index 0000000..5606032 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/5/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/6/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/6/0/0 new file mode 100644 index 0000000..903efec Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/6/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/7/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/7/0/0 new file mode 100644 index 0000000..ef188db Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/7/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/8/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/8/0/0 new file mode 100644 index 0000000..85783af Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/8/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/9/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/9/0/0 new file mode 100644 index 0000000..53ba439 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/c/0/9/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/2/zarr.json b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/zarr.json new file mode 100644 index 0000000..e83ca5d --- /dev/null +++ b/test/data/idr0062A/6001240_labels.zarr/labels/0/2/zarr.json @@ -0,0 +1 @@ +{"chunk_grid":{"configuration":{"chunk_shape":[1,10,512,512]},"name":"regular"},"chunk_key_encoding":{"name":"default"},"codecs":[{"configuration":{"chunk_shape":[1,1,256,256],"codecs":[{"name":"bytes"},{"configuration":{"blocksize":0,"clevel":5,"cname":"zstd","shuffle":"bitshuffle","typesize":1},"name":"blosc"}],"index_codecs":[{"configuration":{"endian":"little"},"name":"bytes"},{"name":"crc32c"}]},"name":"sharding_indexed"}],"data_type":"int8","dimension_names":["c","z","y","x"],"fill_value":0,"node_type":"array","shape":[1,236,68,67],"zarr_format":3} diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/10/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/10/0/0 new file mode 100644 index 0000000..c81b0f6 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/10/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/11/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/11/0/0 new file mode 100644 index 0000000..a23e138 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/11/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/12/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/12/0/0 new file mode 100644 index 0000000..bb5562a Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/12/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/13/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/13/0/0 new file mode 100644 index 0000000..bb234dd Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/13/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/14/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/14/0/0 new file mode 100644 index 0000000..a3557d2 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/14/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/15/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/15/0/0 new file mode 100644 index 0000000..7b32528 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/15/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/16/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/16/0/0 new file mode 100644 index 0000000..acd638e Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/16/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/17/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/17/0/0 new file mode 100644 index 0000000..7da1f1e Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/17/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/18/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/18/0/0 new file mode 100644 index 0000000..af7d269 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/18/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/19/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/19/0/0 new file mode 100644 index 0000000..1aafeb5 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/19/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/2/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/2/0/0 new file mode 100644 index 0000000..55c648e Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/2/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/20/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/20/0/0 new file mode 100644 index 0000000..8291e53 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/20/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/21/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/21/0/0 new file mode 100644 index 0000000..04ac230 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/21/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/22/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/22/0/0 new file mode 100644 index 0000000..299f2c0 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/22/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/23/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/23/0/0 new file mode 100644 index 0000000..6ca92dd Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/23/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/3/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/3/0/0 new file mode 100644 index 0000000..a448b80 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/3/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/4/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/4/0/0 new file mode 100644 index 0000000..a2caf31 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/4/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/5/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/5/0/0 new file mode 100644 index 0000000..df83350 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/5/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/6/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/6/0/0 new file mode 100644 index 0000000..ef5d6aa Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/6/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/7/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/7/0/0 new file mode 100644 index 0000000..ba15fab Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/7/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/8/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/8/0/0 new file mode 100644 index 0000000..c65aff5 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/8/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/9/0/0 b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/9/0/0 new file mode 100644 index 0000000..a9cb907 Binary files /dev/null and b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/c/0/9/0/0 differ diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/3/zarr.json b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/zarr.json new file mode 100644 index 0000000..a7742ba --- /dev/null +++ b/test/data/idr0062A/6001240_labels.zarr/labels/0/3/zarr.json @@ -0,0 +1 @@ +{"chunk_grid":{"configuration":{"chunk_shape":[1,10,512,512]},"name":"regular"},"chunk_key_encoding":{"name":"default"},"codecs":[{"configuration":{"chunk_shape":[1,1,256,256],"codecs":[{"name":"bytes"},{"configuration":{"blocksize":0,"clevel":5,"cname":"zstd","shuffle":"bitshuffle","typesize":1},"name":"blosc"}],"index_codecs":[{"configuration":{"endian":"little"},"name":"bytes"},{"name":"crc32c"}]},"name":"sharding_indexed"}],"data_type":"int8","dimension_names":["c","z","y","x"],"fill_value":0,"node_type":"array","shape":[1,236,34,33],"zarr_format":3} diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/0/zarr.json b/test/data/idr0062A/6001240_labels.zarr/labels/0/zarr.json new file mode 100644 index 0000000..91d4569 --- /dev/null +++ b/test/data/idr0062A/6001240_labels.zarr/labels/0/zarr.json @@ -0,0 +1 @@ +{"attributes": {"ome": {"version": "0.5", "image-label": {"colors": [{"label-value": 1, "rgba": [128, 128, 128, 128]}, {"label-value": 2, "rgba": [128, 128, 128, 128]}, {"label-value": 3, "rgba": [128, 128, 128, 128]}, {"label-value": 4, "rgba": [128, 128, 128, 128]}, {"label-value": 5, "rgba": [128, 128, 128, 128]}, {"label-value": 6, "rgba": [128, 128, 128, 128]}, {"label-value": 7, "rgba": [128, 128, 128, 128]}, {"label-value": 8, "rgba": [128, 128, 128, 128]}, {"label-value": 9, "rgba": [128, 128, 128, 128]}, {"label-value": 10, "rgba": [128, 128, 128, 128]}, {"label-value": 11, "rgba": [128, 128, 128, 128]}, {"label-value": 12, "rgba": [128, 128, 128, 128]}, {"label-value": 13, "rgba": [128, 128, 128, 128]}, {"label-value": 14, "rgba": [128, 128, 128, 128]}, {"label-value": 15, "rgba": [128, 128, 128, 128]}, {"label-value": 16, "rgba": [128, 128, 128, 128]}, {"label-value": 17, "rgba": [128, 128, 128, 128]}, {"label-value": 18, "rgba": [128, 128, 128, 128]}, {"label-value": 19, "rgba": [128, 128, 128, 128]}, {"label-value": 20, "rgba": [128, 128, 128, 128]}, {"label-value": 21, "rgba": [128, 128, 128, 128]}, {"label-value": 22, "rgba": [128, 128, 128, 128]}, {"label-value": 23, "rgba": [128, 128, 128, 128]}, {"label-value": 24, "rgba": [128, 128, 128, 128]}, {"label-value": 25, "rgba": [128, 128, 128, 128]}, {"label-value": 26, "rgba": [128, 128, 128, 128]}, {"label-value": 27, "rgba": [128, 128, 128, 128]}, {"label-value": 28, "rgba": [128, 128, 128, 128]}, {"label-value": 29, "rgba": [128, 128, 128, 128]}, {"label-value": 30, "rgba": [128, 128, 128, 128]}, {"label-value": 31, "rgba": [128, 128, 128, 128]}, {"label-value": 32, "rgba": [128, 128, 128, 128]}, {"label-value": 33, "rgba": [128, 128, 128, 128]}, {"label-value": 34, "rgba": [128, 128, 128, 128]}, {"label-value": 35, "rgba": [128, 128, 128, 128]}, {"label-value": 36, "rgba": [128, 128, 128, 128]}, {"label-value": 37, "rgba": [128, 128, 128, 128]}, {"label-value": 38, "rgba": [128, 128, 128, 128]}, {"label-value": 39, "rgba": [128, 128, 128, 128]}, {"label-value": 40, "rgba": [128, 128, 128, 128]}, {"label-value": 41, "rgba": [128, 128, 128, 128]}, {"label-value": 42, "rgba": [128, 128, 128, 128]}, {"label-value": 43, "rgba": [128, 128, 128, 128]}, {"label-value": 44, "rgba": [128, 128, 128, 128]}, {"label-value": 45, "rgba": [128, 128, 128, 128]}, {"label-value": 46, "rgba": [128, 128, 128, 128]}, {"label-value": 47, "rgba": [128, 128, 128, 128]}, {"label-value": 48, "rgba": [128, 128, 128, 128]}, {"label-value": 49, "rgba": [128, 128, 128, 128]}, {"label-value": 50, "rgba": [128, 128, 128, 128]}, {"label-value": 51, "rgba": [128, 128, 128, 128]}, {"label-value": 52, "rgba": [128, 128, 128, 128]}, {"label-value": 53, "rgba": [128, 128, 128, 128]}, {"label-value": 54, "rgba": [128, 128, 128, 128]}, {"label-value": 55, "rgba": [128, 128, 128, 128]}, {"label-value": 56, "rgba": [128, 128, 128, 128]}, {"label-value": 57, "rgba": [128, 128, 128, 128]}, {"label-value": 58, "rgba": [128, 128, 128, 128]}, {"label-value": 59, "rgba": [128, 128, 128, 128]}, {"label-value": 60, "rgba": [128, 128, 128, 128]}, {"label-value": 61, "rgba": [128, 128, 128, 128]}], "properties": [{"label-value": 1, "omero:roiId": 1380887, "omero:shapeId": 1664569}, {"label-value": 2, "omero:roiId": 1380888, "omero:shapeId": 1664617}, {"label-value": 3, "omero:roiId": 1380889, "omero:shapeId": 1664664}, {"label-value": 4, "omero:roiId": 1380890, "omero:shapeId": 1664709}, {"label-value": 5, "omero:roiId": 1380891, "omero:shapeId": 1664749}, {"label-value": 6, "omero:roiId": 1380892, "omero:shapeId": 1664787}, {"label-value": 7, "omero:roiId": 1380893, "omero:shapeId": 1664826}, {"label-value": 8, "omero:roiId": 1380894, "omero:shapeId": 1664859}, {"label-value": 9, "omero:roiId": 1380895, "omero:shapeId": 1664898}, {"label-value": 10, "omero:roiId": 1380896, "omero:shapeId": 1664932}, {"label-value": 11, "omero:roiId": 1380897, "omero:shapeId": 1664967}, {"label-value": 12, "omero:roiId": 1380898, "omero:shapeId": 1665000}, {"label-value": 13, "omero:roiId": 1380899, "omero:shapeId": 1665036}, {"label-value": 14, "omero:roiId": 1380900, "omero:shapeId": 1665078}, {"label-value": 15, "omero:roiId": 1380901, "omero:shapeId": 1665118}, {"label-value": 16, "omero:roiId": 1380902, "omero:shapeId": 1665156}, {"label-value": 17, "omero:roiId": 1380903, "omero:shapeId": 1665200}, {"label-value": 18, "omero:roiId": 1380904, "omero:shapeId": 1665238}, {"label-value": 19, "omero:roiId": 1380905, "omero:shapeId": 1665283}, {"label-value": 20, "omero:roiId": 1380906, "omero:shapeId": 1665319}, {"label-value": 21, "omero:roiId": 1380907, "omero:shapeId": 1665356}, {"label-value": 22, "omero:roiId": 1380908, "omero:shapeId": 1665396}, {"label-value": 23, "omero:roiId": 1380909, "omero:shapeId": 1665428}, {"label-value": 24, "omero:roiId": 1380910, "omero:shapeId": 1665467}, {"label-value": 25, "omero:roiId": 1380911, "omero:shapeId": 1665513}, {"label-value": 26, "omero:roiId": 1380912, "omero:shapeId": 1665541}, {"label-value": 27, "omero:roiId": 1380913, "omero:shapeId": 1665575}, {"label-value": 28, "omero:roiId": 1380914, "omero:shapeId": 1665611}, {"label-value": 29, "omero:roiId": 1380915, "omero:shapeId": 1665662}, {"label-value": 30, "omero:roiId": 1380916, "omero:shapeId": 1665709}, {"label-value": 31, "omero:roiId": 1380917, "omero:shapeId": 1665743}, {"label-value": 32, "omero:roiId": 1380918, "omero:shapeId": 1665777}, {"label-value": 33, "omero:roiId": 1380919, "omero:shapeId": 1665808}, {"label-value": 34, "omero:roiId": 1380920, "omero:shapeId": 1665853}, {"label-value": 35, "omero:roiId": 1380921, "omero:shapeId": 1665891}, {"label-value": 36, "omero:roiId": 1380922, "omero:shapeId": 1665935}, {"label-value": 37, "omero:roiId": 1380923, "omero:shapeId": 1665970}, {"label-value": 38, "omero:roiId": 1380924, "omero:shapeId": 1666013}, {"label-value": 39, "omero:roiId": 1380925, "omero:shapeId": 1666052}, {"label-value": 40, "omero:roiId": 1380926, "omero:shapeId": 1666085}, {"label-value": 41, "omero:roiId": 1380927, "omero:shapeId": 1666118}, {"label-value": 42, "omero:roiId": 1380928, "omero:shapeId": 1666158}, {"label-value": 43, "omero:roiId": 1380929, "omero:shapeId": 1666194}, {"label-value": 44, "omero:roiId": 1380930, "omero:shapeId": 1666235}, {"label-value": 45, "omero:roiId": 1380931, "omero:shapeId": 1666268}, {"label-value": 46, "omero:roiId": 1380932, "omero:shapeId": 1666307}, {"label-value": 47, "omero:roiId": 1380933, "omero:shapeId": 1666338}, {"label-value": 48, "omero:roiId": 1380934, "omero:shapeId": 1666378}, {"label-value": 49, "omero:roiId": 1380935, "omero:shapeId": 1666411}, {"label-value": 50, "omero:roiId": 1380936, "omero:shapeId": 1666446}, {"label-value": 51, "omero:roiId": 1380937, "omero:shapeId": 1666481}, {"label-value": 52, "omero:roiId": 1380938, "omero:shapeId": 1666518}, {"label-value": 53, "omero:roiId": 1380939, "omero:shapeId": 1666558}, {"label-value": 54, "omero:roiId": 1380940, "omero:shapeId": 1666596}, {"label-value": 55, "omero:roiId": 1380941, "omero:shapeId": 1666671}, {"label-value": 56, "omero:roiId": 1380942, "omero:shapeId": 1666711}, {"label-value": 57, "omero:roiId": 1380943, "omero:shapeId": 1666746}, {"label-value": 58, "omero:roiId": 1380944, "omero:shapeId": 1666789}, {"label-value": 59, "omero:roiId": 1380945, "omero:shapeId": 1666826}, {"label-value": 60, "omero:roiId": 1380946, "omero:shapeId": 1666846}, {"label-value": 61, "omero:roiId": 1380947, "omero:shapeId": 1666891}], "source": {"image": "../.."}}, "multiscales": [{"axes": [{"name": "c", "type": "channel"}, {"name": "z", "type": "space", "unit": "micrometer"}, {"name": "y", "type": "space", "unit": "micrometer"}, {"name": "x", "type": "space", "unit": "micrometer"}], "datasets": [{"coordinateTransformations": [{"scale": [1.0, 0.5002025531914894, 0.3603981534640209, 0.3603981534640209], "type": "scale"}], "path": "0"}, {"coordinateTransformations": [{"scale": [1.0, 0.5002025531914894, 0.7207963069280418, 0.7207963069280418], "type": "scale"}], "path": "1"}, {"coordinateTransformations": [{"scale": [1.0, 0.5002025531914894, 1.4415926138560835, 1.4415926138560835], "type": "scale"}], "path": "2"}, {"coordinateTransformations": [{"scale": [1.0, 0.5002025531914894, 2.883185227712167, 2.883185227712167], "type": "scale"}], "path": "3"}], "name": "0"}]}}, "zarr_format": 3, "node_type": "group"} diff --git a/test/data/idr0062A/6001240_labels.zarr/labels/zarr.json b/test/data/idr0062A/6001240_labels.zarr/labels/zarr.json new file mode 100644 index 0000000..cde99cd --- /dev/null +++ b/test/data/idr0062A/6001240_labels.zarr/labels/zarr.json @@ -0,0 +1 @@ +{"attributes": {"ome": {"labels": ["0"]}}, "zarr_format": 3, "node_type": "group"} diff --git a/test/data/idr0062A/6001240_labels.zarr/zarr.json b/test/data/idr0062A/6001240_labels.zarr/zarr.json new file mode 100644 index 0000000..24e6ef0 --- /dev/null +++ b/test/data/idr0062A/6001240_labels.zarr/zarr.json @@ -0,0 +1 @@ +{"attributes": {"ome": {"version": "0.5", "_creator": {"name": "omero-zarr"}, "multiscales": [{"axes": [{"name": "c", "type": "channel"}, {"name": "z", "type": "space", "unit": "micrometer"}, {"name": "y", "type": "space", "unit": "micrometer"}, {"name": "x", "type": "space", "unit": "micrometer"}], "datasets": [{"coordinateTransformations": [{"scale": [1.0, 0.5002025531914894, 0.3603981534640209, 0.3603981534640209], "type": "scale"}], "path": "0"}, {"coordinateTransformations": [{"scale": [1.0, 0.5002025531914894, 0.7207963069280418, 0.7207963069280418], "type": "scale"}], "path": "1"}, {"coordinateTransformations": [{"scale": [1.0, 0.5002025531914894, 1.4415926138560835, 1.4415926138560835], "type": "scale"}], "path": "2"}]}], "omero": {"channels": [{"active": true, "coefficient": 1.0, "color": "0000FF", "family": "linear", "inverted": false, "label": "LaminB1", "window": {"end": 1500.0, "max": 65535.0, "min": 0.0, "start": 0.0}}, {"active": true, "coefficient": 1.0, "color": "FFFF00", "family": "linear", "inverted": false, "label": "Dapi", "window": {"end": 1500.0, "max": 65535.0, "min": 0.0, "start": 0.0}}], "id": 1, "rdefs": {"defaultT": 0, "defaultZ": 118, "model": "color"}}}}, "zarr_format": 3, "node_type": "group"} diff --git a/test/data/simple_v3.zarr/fortran_v3/c/0/0 b/test/data/simple_v3.zarr/fortran_v3/c/0/0 new file mode 100644 index 0000000..67d9bca Binary files /dev/null and b/test/data/simple_v3.zarr/fortran_v3/c/0/0 differ diff --git a/test/data/simple_v3.zarr/fortran_v3/zarr.json b/test/data/simple_v3.zarr/fortran_v3/zarr.json new file mode 100644 index 0000000..07e5575 --- /dev/null +++ b/test/data/simple_v3.zarr/fortran_v3/zarr.json @@ -0,0 +1 @@ +{"zarr_format":3,"node_type":"array","shape":[2,2],"data_type":"int32","chunk_grid":{"name":"regular","configuration":{"chunk_shape":[2,2]}},"chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}},"fill_value":0,"codecs":[{"name":"transpose","configuration":{"order":[1,0]}},{"name":"bytes","configuration":{"endian":"little"}}]} diff --git a/test/data/simple_v3.zarr/mask_v3/0.0 b/test/data/simple_v3.zarr/mask_v3/0.0 new file mode 100644 index 0000000..f03a3bd Binary files /dev/null and b/test/data/simple_v3.zarr/mask_v3/0.0 differ diff --git a/test/data/simple_v3.zarr/mask_v3/0.1 b/test/data/simple_v3.zarr/mask_v3/0.1 new file mode 100644 index 0000000..baea61a Binary files /dev/null and b/test/data/simple_v3.zarr/mask_v3/0.1 differ diff --git a/test/data/simple_v3.zarr/mask_v3/zarr.json b/test/data/simple_v3.zarr/mask_v3/zarr.json new file mode 100644 index 0000000..2585735 --- /dev/null +++ b/test/data/simple_v3.zarr/mask_v3/zarr.json @@ -0,0 +1 @@ +{"zarr_format":3,"node_type":"array","shape":[2,3],"data_type":"bool","chunk_grid":{"name":"regular","configuration":{"chunk_shape":[2,2]}},"chunk_key_encoding":{"name":"v2","configuration":{"separator":"."}},"fill_value":false,"codecs":[{"name":"bytes"}]} diff --git a/test/data/simple_v3.zarr/temperature_v3/c/0/0 b/test/data/simple_v3.zarr/temperature_v3/c/0/0 new file mode 100644 index 0000000..0976d52 Binary files /dev/null and b/test/data/simple_v3.zarr/temperature_v3/c/0/0 differ diff --git a/test/data/simple_v3.zarr/temperature_v3/c/1/0 b/test/data/simple_v3.zarr/temperature_v3/c/1/0 new file mode 100644 index 0000000..26cde57 Binary files /dev/null and b/test/data/simple_v3.zarr/temperature_v3/c/1/0 differ diff --git a/test/data/simple_v3.zarr/temperature_v3/zarr.json b/test/data/simple_v3.zarr/temperature_v3/zarr.json new file mode 100644 index 0000000..38a2b82 --- /dev/null +++ b/test/data/simple_v3.zarr/temperature_v3/zarr.json @@ -0,0 +1 @@ +{"zarr_format":3,"node_type":"array","shape":[3,2],"data_type":"float64","chunk_grid":{"name":"regular","configuration":{"chunk_shape":[2,2]}},"chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}},"fill_value":null,"codecs":[{"name":"bytes","configuration":{"endian":"little"}},{"name":"gzip","configuration":{"level":1}}],"attributes":{"units":"celsius"}} diff --git a/test/data/simple_v3.zarr/zarr.json b/test/data/simple_v3.zarr/zarr.json new file mode 100644 index 0000000..069972d --- /dev/null +++ b/test/data/simple_v3.zarr/zarr.json @@ -0,0 +1 @@ +{"zarr_format":3,"node_type":"group","attributes":{"name":"simple_v3"}} diff --git a/test/python_smoke.py b/test/python_smoke.py new file mode 100644 index 0000000..68e53ec --- /dev/null +++ b/test/python_smoke.py @@ -0,0 +1,191 @@ +import gzip +import os +import platform +import shutil +from pathlib import Path + +import duckdb +import numpy as np +import pandas as pd + + +ROOT = Path(__file__).resolve().parents[1] +FIXTURE_PATH = ROOT / "test" / "data" / "simple_v2.zarr" +FIXTURE_V3_PATH = ROOT / "test" / "data" / "simple_v3.zarr" +IDR_FIXTURE_PATH = ROOT / "test" / "data" / "idr0062A" / "6001240_labels.zarr" +LOCAL_BUILD_EXTENSION = ROOT / "build" / "release" / "extension" / "duckdb_zarr" / "duckdb_zarr.duckdb_extension" +DECOMPRESSED_EXTENSION = ROOT / "duckdb_zarr.duckdb_extension" + + +def expected_release_asset() -> Path: + system = platform.system().lower() + machine = platform.machine().lower() + + if system == "darwin": + platform_name = "osx_arm64" if machine in {"arm64", "aarch64"} else "osx_amd64" + elif system == "linux": + platform_name = "linux_arm64" if machine in {"arm64", "aarch64"} else "linux_amd64" + elif system == "windows": + platform_name = "windows_amd64" + else: + raise RuntimeError(f"Unsupported platform for release asset lookup: {platform.system()} / {platform.machine()}") + + return ROOT / f"duckdb_zarr-v1.5.0-{platform_name}.duckdb_extension.gz" + + +def resolve_extension_path() -> Path: + override = os.environ.get("DUCKDB_ZARR_EXTENSION") + if override: + return Path(override).resolve() + if LOCAL_BUILD_EXTENSION.exists(): + return LOCAL_BUILD_EXTENSION + release_asset = expected_release_asset() + if release_asset.exists(): + with gzip.open(release_asset, "rb") as src, DECOMPRESSED_EXTENSION.open("wb") as dst: + shutil.copyfileobj(src, dst) + return DECOMPRESSED_EXTENSION + raise FileNotFoundError( + "No extension artifact found. Build the repo with `make`, set DUCKDB_ZARR_EXTENSION " + "to an explicit extension path, or download the platform-matching release asset " + f"`{release_asset.name}` into the repo root." + ) + + +def should_allow_unsigned_extensions(extension_path: Path) -> bool: + if os.environ.get("DUCKDB_ZARR_ALLOW_UNSIGNED", "").lower() in {"1", "true", "yes"}: + return True + return extension_path.is_file() and extension_path.resolve().is_relative_to(ROOT.resolve()) + + +def main() -> None: + if not FIXTURE_PATH.exists() or not FIXTURE_V3_PATH.exists(): + raise FileNotFoundError("Fixture stores missing. Run `make fixture` from the repo root first.") + + extension_path = resolve_extension_path() + config = {} + if should_allow_unsigned_extensions(extension_path): + # Local build artifacts and manually downloaded release files are unsigned in this repo. + config["allow_unsigned_extensions"] = "true" + + con = duckdb.connect(config=config) + print("DuckDB version:", con.execute("select version()").fetchone()[0]) + + try: + con.execute(f"LOAD '{extension_path.as_posix()}'") + except duckdb.Error as exc: + raise RuntimeError( + "Failed to load the extension. If you are loading a local unsigned artifact, set " + "DUCKDB_ZARR_ALLOW_UNSIGNED=1." + ) from exc + + overview = con.execute( + """ + SELECT array_path, rank, dtype + FROM zarr(?) + ORDER BY array_path + """, + [FIXTURE_PATH.as_posix()], + ).fetchdf() + print("\nArray overview:") + print(overview) + + temperature = con.execute( + """ + SELECT * + FROM zarr(?, 'temperature') + ORDER BY dim_0, dim_1 + """, + [FIXTURE_PATH.as_posix()], + ).fetchdf() + print("\nTemperature cells:") + print(temperature) + + values = temperature["value"].to_numpy(dtype=np.float64) + print("\nTemperature summary:") + print(pd.Series(values).describe()) + + v3_overview = con.execute( + """ + SELECT array_path, rank, zarr_format + FROM zarr(?) + ORDER BY array_path + """, + [FIXTURE_V3_PATH.as_posix()], + ).fetchdf() + print("\nV3 array overview:") + print(v3_overview) + + v3_temperature = con.execute( + """ + SELECT * + FROM zarr(?, 'temperature_v3', 'v3') + ORDER BY dim_0, dim_1 + """, + [FIXTURE_V3_PATH.as_posix()], + ).fetchdf() + print("\nV3 temperature cells:") + print(v3_temperature) + + if IDR_FIXTURE_PATH.exists(): + idr_overview = con.execute( + """ + SELECT array_path, rank + FROM zarr(?) + ORDER BY array_path + """, + [IDR_FIXTURE_PATH.as_posix()], + ).fetchdf() + print("\nIDR OME-Zarr overview:") + print(idr_overview) + + idr_head = con.execute( + """ + SELECT * + FROM zarr(?, '0') + LIMIT 5 + """, + [IDR_FIXTURE_PATH.as_posix()], + ).fetchdf() + print("\nIDR OME-Zarr first cells:") + print(idr_head) + + idr_label_sum = con.execute( + """ + SELECT SUM(value) AS label_sum + FROM zarr(?, 'labels/0/0') + WHERE dim_0 = 0 AND dim_1 = 0 AND dim_2 < 4 AND dim_3 < 4 + """, + [IDR_FIXTURE_PATH.as_posix()], + ).fetchone()[0] + print("\nIDR label block sum:", idr_label_sum) + + assert idr_overview["array_path"].tolist() == [ + "0", + "1", + "2", + "labels/0/0", + "labels/0/1", + "labels/0/2", + "labels/0/3", + ] + assert idr_head["value"].tolist() == [8, 9, 8, 10, 8] + assert idr_label_sum == 0 + + assert len(temperature) == 12 + assert np.isclose(values.sum(), 78.0) + assert overview["array_path"].tolist() == [ + "half_precision", + "mask", + "sparse_fill", + "stations/elevation", + "temperature", + ] + assert v3_overview["array_path"].tolist() == ["fortran_v3", "mask_v3", "temperature_v3"] + assert v3_overview["zarr_format"].tolist() == [3, 3, 3] + assert np.isclose(v3_temperature["value"].to_numpy(dtype=np.float64).sum(), 21.0) + + print("\nPython smoke test passed.") + + +if __name__ == "__main__": + main() diff --git a/test/sql/duckdb_zarr.test b/test/sql/duckdb_zarr.test index 6d0b382..137c1b2 100644 --- a/test/sql/duckdb_zarr.test +++ b/test/sql/duckdb_zarr.test @@ -10,12 +10,12 @@ Catalog Error: Table Function with name zarr_groups does not exist! require duckdb_zarr query III -SELECT group_path, zarr_format, metadata_path LIKE '%simple_v2.zarr/%' +SELECT group_path, zarr_format, metadata_path LIKE '%simple_v2.zarr%' FROM zarr_groups('test/data/simple_v2.zarr') ORDER BY group_path; ---- -/ 2 true -stations 2 true +/ 2 1 +stations 2 1 query I SELECT COUNT(*) @@ -34,7 +34,7 @@ SELECT "order", COALESCE(NULLIF(compressor, ''), ''), dimension_separator, - metadata_path LIKE '%simple_v2.zarr/%' + metadata_path LIKE '%simple_v2.zarr%' FROM zarr_arrays('test/data/simple_v2.zarr') ORDER BY array_path; ---- @@ -50,7 +50,7 @@ SELECT chunk_key, chunk_coords::VARCHAR, file_size_bytes > 0, - file_path LIKE '%simple_v2.zarr/%' + file_path LIKE '%simple_v2.zarr%' FROM zarr_chunks('test/data/simple_v2.zarr') ORDER BY array_path, chunk_key; ---- @@ -178,11 +178,11 @@ WHERE dim_0 = 1; 200 query III -SELECT group_path, zarr_format, metadata_path LIKE '%ome_example.ome.zarr/%' +SELECT group_path, zarr_format, metadata_path LIKE '%ome_example.ome.zarr%' FROM zarr_groups('test/data/ome_example.ome.zarr') ORDER BY group_path; ---- -/ 2 true +/ 2 1 query IIIIIIII SELECT @@ -232,3 +232,191 @@ FROM zarr('test/data/ome_example.ome.zarr') ORDER BY array_path; ---- 0 3 + +query III +SELECT group_path, zarr_format, metadata_path LIKE '%simple_v3.zarr%' +FROM zarr_groups('test/data/simple_v3.zarr') +ORDER BY group_path; +---- +/ 3 1 + +query IIIIIIII +SELECT + array_path, + rank, + shape::VARCHAR, + chunk_shape::VARCHAR, + dtype, + "order", + COALESCE(NULLIF(compressor, ''), ''), + chunk_key_encoding +FROM zarr_arrays('test/data/simple_v3.zarr') +ORDER BY array_path; +---- +fortran_v3 2 [2, 2] [2, 2] default +mask_v3 2 [2, 3] [2, 2] |b1 C v2 +temperature_v3 2 [3, 2] [2, 2]