From 20f2ac887a147471f0cc8e9780d6ef9b8bcdad26 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 25 Jul 2026 16:10:44 -0700 Subject: [PATCH 01/20] =?UTF-8?q?feat:=20url-pipeline=20core=20=E2=80=94?= =?UTF-8?q?=20parser,=20adapter=20ABC,=20registry,=20store=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements URL pipeline support (https://github.com/jbms/url-pipeline): '|'-chained URLs resolve through pluggable adapters registered under the 'zarr.url_adapters' entry-point group (entry-point name = URL scheme). - zarr.abc.url_pipeline: PipelineSegment, AdapterResolution, PipelineContext, URLPipelineAdapter (single-classmethod contract) - zarr.storage._url_pipeline: parse_pipeline / resolve_pipeline; the root sub-URL delegates to make_store so existing file/memory/fsspec routing is unchanged - registry: register_url_adapter / get_url_adapter / list_url_adapter_schemes (name check only; no adapter imports) - make_store/make_store_path route strings containing '|' (or a registered root scheme) through the resolver; residual store paths combine with the user-supplied path - StorePath gains a zarr_format attribute (populated by format segments in a follow-up) Co-Authored-By: Claude Fable 5 --- changes/4192.feature.md | 25 + docs/api/zarr/abc/index.md | 1 + docs/api/zarr/abc/url_pipeline.md | 5 + docs/user-guide/storage.md | 25 + mkdocs.yml | 1 + src/zarr/abc/url_pipeline.py | 256 ++++++++ src/zarr/api/asynchronous.py | 39 +- src/zarr/errors.py | 7 + src/zarr/registry.py | 77 ++- src/zarr/storage/__init__.py | 3 + src/zarr/storage/_common.py | 49 +- src/zarr/storage/_url_pipeline.py | 311 ++++++++++ .../entry_points.txt | 4 +- tests/package_with_entrypoint/__init__.py | 15 + tests/test_api.py | 5 +- tests/test_url_pipeline/__init__.py | 0 tests/test_url_pipeline/conftest.py | 22 + tests/test_url_pipeline/test_parser.py | 232 +++++++ tests/test_url_pipeline/test_resolver.py | 569 ++++++++++++++++++ tests/test_url_pipeline/test_spec_examples.py | 258 ++++++++ 20 files changed, 1887 insertions(+), 17 deletions(-) create mode 100644 changes/4192.feature.md create mode 100644 docs/api/zarr/abc/url_pipeline.md create mode 100644 src/zarr/abc/url_pipeline.py create mode 100644 src/zarr/storage/_url_pipeline.py create mode 100644 tests/test_url_pipeline/__init__.py create mode 100644 tests/test_url_pipeline/conftest.py create mode 100644 tests/test_url_pipeline/test_parser.py create mode 100644 tests/test_url_pipeline/test_resolver.py create mode 100644 tests/test_url_pipeline/test_spec_examples.py diff --git a/changes/4192.feature.md b/changes/4192.feature.md new file mode 100644 index 0000000000..757aa13895 --- /dev/null +++ b/changes/4192.feature.md @@ -0,0 +1,25 @@ +Added core support for URL pipelines (https://github.com/jbms/url-pipeline): +`|`-chained URLs that address zarr data through nested storage layers, e.g. +`s3://bucket/data.zip|zip:|zarr3:`. This PR adds the parser, the single-method +`zarr.abc.url_pipeline.URLPipelineAdapter` interface, and the +`zarr.url_adapters` entry-point group through which third-party packages +(e.g. Icechunk) register adapters for their own schemes. Adapters for a scheme +are loaded lazily and individually. Builtin adapters (`zip:`, +`zarr2:`/`zarr3:`) follow in separate pull requests. + +Behavior notes: + +- The `|` character is now reserved as the pipeline delimiter in every string + store specification, and no percent-escape is decoded; pass a `pathlib.Path` + to address a local file whose name contains `|`. URLs without a `|` (and + without a registered root adapter scheme) are handled exactly as before — + registered adapters cannot intercept zarr's native `file:`/`memory:` + routing, and fsspec chained URLs (`zip::s3://...`) keep flowing to fsspec. +- Inside a pipeline, `memory:` and `file:` roots follow the URL pipeline + spec's semantics (spelling equivalences; `file:` must be absolute, with at + most a `localhost` authority). +- Mode `"a"` (open-or-create, the `zarr.open` default) on a *read-only* store + now serves the "open" half instead of raising upfront, for all stores; + unambiguous write modes (`"w"`, `"w-"`, `"r+"`) still raise. +- For root-adapter URLs (e.g. `gh://org/repo`), `storage_options` are handed + to the adapter and are not validated as used by `make_store`. diff --git a/docs/api/zarr/abc/index.md b/docs/api/zarr/abc/index.md index 7e15cb2a51..d155f60a5a 100644 --- a/docs/api/zarr/abc/index.md +++ b/docs/api/zarr/abc/index.md @@ -11,3 +11,4 @@ Abstract base classes for extending Zarr-Python. - **[zarr.abc.metadata](./metadata.md)** - Creating metadata classes compatible with the Zarr API - **[zarr.abc.numcodec](./numcodec.md)** - Protocols and classes for modeling codec interface used by numcodecs - **[zarr.abc.store](./store.md)** - ABC for implementing Zarr stores and managing getting and setting bytes in a store +- **[zarr.abc.url_pipeline](./url_pipeline.md)** - ABC for implementing [URL pipeline](https://github.com/jbms/url-pipeline) adapters diff --git a/docs/api/zarr/abc/url_pipeline.md b/docs/api/zarr/abc/url_pipeline.md new file mode 100644 index 0000000000..2606d4d5d4 --- /dev/null +++ b/docs/api/zarr/abc/url_pipeline.md @@ -0,0 +1,5 @@ +--- +title: url_pipeline +--- + +::: zarr.abc.url_pipeline diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index a34e2e2874..0b86bef2b0 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -103,6 +103,31 @@ print(group) - a [`Store`][zarr.abc.store.Store] or [`StorePath`][zarr.storage.StorePath] - see explicit store creation below. +## URL Pipelines {#user-guide-url-pipelines} + +Zarr supports [URL pipelines](https://github.com/jbms/url-pipeline): `|`-chained URLs +that address zarr data through nested storage layers, read left to right. The first +sub-URL locates a resource with a conventional URL; each subsequent sub-URL names an +*adapter* that reinterprets everything to its left (e.g. +`s3://bucket/data.zip|zip:|zarr3:`). Adapters are provided by packages through the +`zarr.url_adapters` entry-point group — see +[`zarr.abc.url_pipeline`][zarr.abc.url_pipeline] for the adapter interface. Builtin +adapters (`zip:`, `zarr2:`/`zarr3:`) are under development and will expand this +section. URLs without a `|` (and without a registered root scheme) are handled +exactly as before. + +`storage_options` passed to `zarr.open` apply to the *root* sub-URL (e.g. fsspec +options for `s3://...`); adapters may consume adapter-specific, namespaced keys. +Non-dict forms of `storage_options` are reserved for future per-segment +configuration. + +The `|` character is reserved as the pipeline delimiter in every string store +specification, and no percent-escape is decoded: to address a local file whose +*name* contains `|` (or `#`), pass a `pathlib.Path` instead of a string. +Registered adapters cannot intercept zarr's native `file:` and `memory:` root +schemes, and fsspec's chained-URL syntax (`zip::s3://...`) keeps flowing to +fsspec. + ## Explicit Store Creation In some cases, it may be helpful to create a store instance directly. Zarr-Python offers diff --git a/mkdocs.yml b/mkdocs.yml index 4d06701a87..58b5d3344a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -43,6 +43,7 @@ nav: - ' zarr.abc.metadata': api/zarr/abc/metadata.md - ' zarr.abc.numcodec': api/zarr/abc/numcodec.md - ' zarr.abc.store': api/zarr/abc/store.md + - ' zarr.abc.url_pipeline': api/zarr/abc/url_pipeline.md - ' zarr.api': - api/zarr/api/index.md - ' zarr.api.asynchronous': api/zarr/api/asynchronous.md diff --git a/src/zarr/abc/url_pipeline.py b/src/zarr/abc/url_pipeline.py new file mode 100644 index 0000000000..0e0363bcf6 --- /dev/null +++ b/src/zarr/abc/url_pipeline.py @@ -0,0 +1,256 @@ +""" +Abstract base class and data model for URL pipeline adapters. + +A URL pipeline is a `|`-separated chain of sub-URLs, read outer-to-inner, +as specified by https://github.com/jbms/url-pipeline. The first sub-URL (the +*root*) locates a resource using a conventional URL, and each subsequent +sub-URL names an *adapter* that reinterprets everything to its left: + + s3://bucket/data.zip|zip:path/inside|zarr3: + +Third-party packages provide adapters by subclassing +[`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] and +registering the class under the `zarr.url_adapters` entry-point group, +using the URL scheme as the entry-point name. +""" + +from __future__ import annotations + +import enum +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from zarr.errors import URLPipelineError + +if TYPE_CHECKING: + from zarr.abc.store import Store + from zarr.core.common import AccessModeLiteral, ZarrFormat + +__all__ = [ + "AdapterResolution", + "PipelineContext", + "PipelineSegment", + "URLPipelineAdapter", +] + + +class _Unset(enum.Enum): + token = 0 + + +_UNSET = _Unset.token + + +@dataclass(frozen=True) +class PipelineSegment: + """ + One `|`-delimited sub-URL of a URL pipeline. + + Attributes + ---------- + scheme : str + The lowercased URL scheme. Empty string only for a schemeless root + (a bare local path), which is treated as opaque text. + body : str + The text after `scheme:` and before any `?`. Interpretation is + scheme-defined; it is **not** URL-normalized, so case-significant + content (e.g. icechunk snapshot IDs) is preserved. + query : str | None + The raw query string after `?`, or None. Interpretation is + scheme-defined. + raw : str + The exact original sub-URL text, preserved for lossless + reconstruction of the pipeline. + """ + + scheme: str + body: str + query: str | None + raw: str + + def __str__(self) -> str: + return self.raw + + +@dataclass(frozen=True) +class AdapterResolution: + """ + The result of resolving a URL pipeline (or a prefix of one). + + Attributes + ---------- + store : Store + The resolved store. + path : str + Residual path *within* the store that the pipeline addresses + (e.g. `"path/to/node"` for `...|icechunk://tag.v1/path/to/node`). + Empty string when the pipeline addresses the store root. + zarr_format : ZarrFormat | None + Zarr format selected by a format segment (`zarr2:`/`zarr3:`), + or None if unspecified. A wrapper adapter that re-wraps a preceding + resolution must carry every field it does not change forward — + prefer `dataclasses.replace(preceding, store=..., path=...)` over + reconstructing, so fields added later are never silently dropped. + """ + + store: Store + path: str = "" + zarr_format: ZarrFormat | None = None + + +@dataclass(frozen=True) +class PipelineContext: + """ + Context handed to a [`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] + describing the pipeline to the left of its segment. + + Attributes + ---------- + preceding : tuple[PipelineSegment, ...] + The parsed sub-URLs to the left of the adapter's segment, outer to + inner. Empty when the adapter's segment is the pipeline root. + mode : AccessModeLiteral | None + The access mode requested by the caller (e.g. `zarr.open(mode=...)`), + or None when unspecified. Adapters for read-only resources should + raise for unambiguous write modes (`"w"`, `"w-"`, `"r+"`) and + open read-only otherwise. `"a"` (the `zarr.open` default) means + open-or-create: read-only adapters serve the "open" half, and any + subsequent write fails at the store level. + storage_options : dict[str, Any] | None + Options passed by the caller. By convention these configure the + *root* sub-URL (e.g. fsspec options); adapters may consume + adapter-specific keys, and should namespace them (e.g. + `myscheme_credentials`) to avoid collisions with other segments' + backends. An adapter that consumes keys should strip them before + resolving the rest of the pipeline, by passing the reduced mapping + to [`resolve_preceding`][zarr.abc.url_pipeline.PipelineContext.resolve_preceding]. + Non-dict forms of the caller-facing `storage_options` argument are + reserved for future per-segment configuration (one mapping per + pipeline segment); this attribute will remain a single mapping — + the one addressed to this adapter's segment. + """ + + preceding: tuple[PipelineSegment, ...] + mode: AccessModeLiteral | None + storage_options: dict[str, Any] | None + + @property + def read_only(self) -> bool: + """ + True when the caller requires a read-only store (`mode == "r"`). + + Adapters must construct their store read-only when this is set + (the resolver enforces it afterwards); when it is False, they may + construct a writable store if the underlying resource supports + writing. + """ + return self.mode == "r" + + @property + def preceding_url(self) -> str: + """ + The pipeline to the left of this segment, reconstructed exactly. + + An adapter that consumes this string instead of calling + [`resolve_preceding`][zarr.abc.url_pipeline.PipelineContext.resolve_preceding] + takes ownership of the *entire* preceding pipeline: it must + validate every preceding segment itself and raise + [`URLPipelineError`][zarr.errors.URLPipelineError] for segments it + does not understand, so that no segment is ever silently ignored. + """ + return "|".join(segment.raw for segment in self.preceding) + + async def resolve_preceding( + self, + *, + mode: AccessModeLiteral | _Unset | None = _UNSET, + storage_options: dict[str, Any] | _Unset | None = _UNSET, + ) -> AdapterResolution: + """ + Resolve the preceding pipeline into a store. + + This is the entry point for *wrapper* adapters (e.g. `zip:`) that + operate on the resource produced by the segments to their left. It + composes with any preceding adapters, because each segment is + resolved by its own adapter. Adapters backed by their own I/O + machinery (e.g. `icechunk:`) may instead consume + [`preceding_url`][zarr.abc.url_pipeline.PipelineContext.preceding_url] + and never materialize the intermediate store — subject to the + ownership contract documented there. + + Parameters + ---------- + mode : AccessModeLiteral | None, optional + Override the mode used to resolve the preceding pipeline. + Wrapper adapters that only read the preceding resource should + pass `mode="r"` so the root is opened read-only and without + create-on-open side effects, regardless of the caller's mode. + When omitted, the caller's mode is used. + storage_options : dict | None, optional + Override the options forwarded to the preceding pipeline. An + adapter that consumed adapter-specific keys should pass the + remaining mapping here (or None when nothing remains), so the + root store never sees keys that were not addressed to it. + When omitted, the caller's options are forwarded unchanged. + """ + from zarr.storage._url_pipeline import _resolve + + if not self.preceding: + raise URLPipelineError( + "this adapter segment is at the pipeline root; " + "there is no preceding sub-URL to resolve" + ) + return await _resolve( + self.preceding, + mode=self.mode if isinstance(mode, _Unset) else mode, + storage_options=( + self.storage_options if isinstance(storage_options, _Unset) else storage_options + ), + ) + + +class URLPipelineAdapter(ABC): + """ + Handler for one URL pipeline scheme. + + Subclasses implement a single classmethod, + [`open_pipeline_segment`][zarr.abc.url_pipeline.URLPipelineAdapter.open_pipeline_segment], + and are registered under the `zarr.url_adapters` entry-point group with + the URL scheme as the entry-point name: + + [project.entry-points."zarr.url_adapters"] + mypackage.myscheme = "mypackage.zarr_adapter:MyAdapter" + + Nonstandard schemes should be vendor-prefixed (`vendor.scheme`) per the + URL pipeline specification. + + An adapter is used in two positions: + + - as an *adapter segment*: `s3://bucket/repo|icechunk://tag.v1` — the + context carries the preceding sub-URLs; + - as a *root scheme*: `gh://org/repo` — `context.preceding` is empty. + """ + + @classmethod + @abstractmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + """ + Resolve `segment` (in the context of the pipeline to its left) + into a store and an optional residual path within that store. + + The returned store must already be open and must honor + `context.read_only` (the resolver additionally enforces it by + downgrading — or rejecting — a writable store when the caller + required read-only). + + This coroutine runs on zarr's internal I/O event loop. It must not + block (do I/O through async APIs or a thread executor) and must not + call zarr's synchronous API (`zarr.open`, `Group.open`, or anything + else that uses `zarr.core.sync.sync`) — doing so raises + `SyncError`. To open the preceding pipeline, use + [`resolve_preceding`][zarr.abc.url_pipeline.PipelineContext.resolve_preceding]. + """ + ... diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 3bdc254ea5..2ae05bef6f 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -108,6 +108,24 @@ def _infer_overwrite(mode: AccessModeLiteral) -> bool: return mode in _OVERWRITE_MODES +def _merge_pipeline_zarr_format( + store_path: StorePath, zarr_format: ZarrFormat | None +) -> ZarrFormat | None: + """ + Combine a zarr format selected by a URL pipeline segment (`zarr2:` / + `zarr3:`) with the caller's `zarr_format` argument. Explicitly + conflicting selections raise. + """ + if store_path.zarr_format is None: + return zarr_format + if zarr_format is not None and zarr_format != store_path.zarr_format: + raise ValueError( + f"zarr_format={zarr_format} conflicts with the 'zarr{store_path.zarr_format}:' " + "segment of the URL pipeline" + ) + return store_path.zarr_format + + def _warn_unimplemented_kwargs(kwargs: dict[str, Any]) -> None: """ Emit a "not yet implemented" warning for each provided keyword argument that is not None. @@ -389,6 +407,7 @@ async def open( else: mode = "a" store_path = await make_store_path(store, mode=mode, path=path, storage_options=storage_options) + zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) # TODO: the mode check below seems wrong! if "shape" not in kwargs and mode in {"a", "r", "r+", "w"}: @@ -494,13 +513,14 @@ async def save_array( **kwargs Passed through to [`create`][zarr.api.asynchronous.create], e.g., compressor. """ - if zarr_format is None: - zarr_format = _default_zarr_format() if not isinstance(arr, NDArrayLike): raise TypeError("arr argument must be numpy or other NDArrayLike array") mode = kwargs.pop("mode", "a") store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) + zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + if zarr_format is None: + zarr_format = _default_zarr_format() if np.isscalar(arr): arr = np.array(arr) shape = arr.shape @@ -550,6 +570,7 @@ async def save_group( """ store_path = await make_store_path(store, path=path, mode="w", storage_options=storage_options) + zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() @@ -762,12 +783,12 @@ async def create_group( The new group. """ - if zarr_format is None: - zarr_format = _default_zarr_format() - mode: Literal["a"] = "a" store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) + zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + if zarr_format is None: + zarr_format = _default_zarr_format() return await AsyncGroup.from_store( store=store_path, @@ -857,6 +878,7 @@ async def open_group( ) store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path) + zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) if attributes is None: attributes = {} @@ -1041,9 +1063,6 @@ async def create( z : array The array. """ - if zarr_format is None: - zarr_format = _default_zarr_format() - _warn_unimplemented_kwargs( { "synchronizer": synchronizer, @@ -1063,6 +1082,9 @@ async def create( if mode is None: mode = "a" store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) + zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + if zarr_format is None: + zarr_format = _default_zarr_format() config_parsed = parse_array_config(config) @@ -1262,6 +1284,7 @@ async def open_array( mode = kwargs.pop("mode", None) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) + zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) if "write_empty_chunks" in kwargs: _warn_write_empty_chunks_kwarg() diff --git a/src/zarr/errors.py b/src/zarr/errors.py index 781bebe534..de04a5bdaa 100644 --- a/src/zarr/errors.py +++ b/src/zarr/errors.py @@ -12,6 +12,7 @@ "MetadataValidationError", "NegativeStepError", "NodeTypeValidationError", + "URLPipelineError", "UnstableSpecificationWarning", "VindexInvalidSelectionError", "ZarrDeprecationWarning", @@ -100,6 +101,12 @@ class UnknownCodecError(BaseZarrError): """ +class URLPipelineError(BaseZarrError): + """ + Raised when a URL pipeline cannot be parsed or resolved. + """ + + class NodeTypeValidationError(MetadataValidationError): """ Specialized exception when the node_type of the metadata document is incorrect. diff --git a/src/zarr/registry.py b/src/zarr/registry.py index c2c0eb2921..53875c015c 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -1,5 +1,6 @@ from __future__ import annotations +import threading import warnings from collections import defaultdict from importlib.metadata import entry_points as get_entry_points @@ -7,7 +8,7 @@ from zarr.core.config import BadConfigError, config from zarr.core.dtype import data_type_registry -from zarr.errors import ZarrUserWarning +from zarr.errors import URLPipelineError, ZarrUserWarning if TYPE_CHECKING: from importlib.metadata import EntryPoint @@ -21,6 +22,7 @@ CodecPipeline, ) from zarr.abc.numcodec import Numcodec + from zarr.abc.url_pipeline import URLPipelineAdapter from zarr.core.buffer import Buffer, NDBuffer from zarr.core.chunk_key_encodings import ChunkKeyEncoding from zarr.core.common import JSON @@ -32,11 +34,14 @@ "get_codec_class", "get_ndbuffer_class", "get_pipeline_class", + "get_url_adapter", + "list_url_adapter_schemes", "register_buffer", "register_chunk_key_encoding", "register_codec", "register_ndbuffer", "register_pipeline", + "register_url_adapter", ] @@ -62,6 +67,7 @@ def register(self, cls: type[T], qualname: str | None = None) -> None: _buffer_registry: Registry[Buffer] = Registry() _ndbuffer_registry: Registry[NDBuffer] = Registry() _chunk_key_encoding_registry: Registry[ChunkKeyEncoding] = Registry() +_url_adapter_registry: Registry[URLPipelineAdapter] = Registry() """ The registry module is responsible for managing implementations of codecs, @@ -108,6 +114,8 @@ def _collect_entrypoints() -> list[Registry[Any]]: entry_points.select(group="zarr", name="chunk_key_encoding") ) + _url_adapter_registry.lazy_load_list.extend(entry_points.select(group="zarr.url_adapters")) + _pipeline_registry.lazy_load_list.extend(entry_points.select(group="zarr.codec_pipeline")) _pipeline_registry.lazy_load_list.extend( entry_points.select(group="zarr", name="codec_pipeline") @@ -124,6 +132,7 @@ def _collect_entrypoints() -> list[Registry[Any]]: _buffer_registry, _ndbuffer_registry, _chunk_key_encoding_registry, + _url_adapter_registry, ] @@ -303,6 +312,72 @@ def get_chunk_key_encoding_class(key: str) -> type[ChunkKeyEncoding]: return _chunk_key_encoding_registry[key] +def register_url_adapter(scheme: str, cls: type[URLPipelineAdapter]) -> None: + """ + Register a [`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] + class for a URL scheme. + + Registering a scheme that already has an adapter replaces it and emits a + [`ZarrUserWarning`][zarr.errors.ZarrUserWarning]. + """ + key = scheme.lower() + previous = _url_adapter_registry.get(key) + if previous is not None and previous is not cls: + warnings.warn( + f"URL pipeline adapter for scheme {scheme!r} is being replaced: " + f"{fully_qualified_name(previous)} -> {fully_qualified_name(cls)}", + category=ZarrUserWarning, + stacklevel=2, + ) + _url_adapter_registry.register(cls, key) + + +def list_url_adapter_schemes() -> set[str]: + """ + The set of URL schemes with a registered URL pipeline adapter. + + Includes adapters advertised via not-yet-loaded `zarr.url_adapters` + entry points; consulting this does not import any adapter code. + Schemes are case-insensitive and reported lowercased. + """ + return set(_url_adapter_registry) | { + e.name.lower() for e in _url_adapter_registry.lazy_load_list + } + + +_url_adapter_lock = threading.Lock() + + +def get_url_adapter(scheme: str) -> type[URLPipelineAdapter]: + """ + Get the URL pipeline adapter class registered for `scheme`. + + Loads pending `zarr.url_adapters` entry points for this scheme only, so + resolving one scheme never imports other providers' packages. + """ + key = scheme.lower() + # The lock keeps concurrent first-time resolutions of different schemes + # from clobbering each other's rebuild of the pending entry-point list. + with _url_adapter_lock: + if key not in _url_adapter_registry: + remaining = [] + for entry_point in _url_adapter_registry.lazy_load_list: + if entry_point.name.lower() == key: + _url_adapter_registry.register(entry_point.load(), qualname=key) + else: + remaining.append(entry_point) + _url_adapter_registry.lazy_load_list[:] = remaining + try: + return _url_adapter_registry[key] + except KeyError: + registered = sorted(list_url_adapter_schemes()) + raise URLPipelineError( + f"no URL pipeline adapter is registered for scheme {scheme!r}. " + f"Registered schemes: {registered}. Adapters are provided by " + "packages via the 'zarr.url_adapters' entry-point group." + ) from None + + _collect_entrypoints() diff --git a/src/zarr/storage/__init__.py b/src/zarr/storage/__init__.py index f1bd1724af..edc01f1a5b 100644 --- a/src/zarr/storage/__init__.py +++ b/src/zarr/storage/__init__.py @@ -10,6 +10,7 @@ from zarr.storage._logging import LoggingStore from zarr.storage._memory import GpuMemoryStore, ManagedMemoryStore, MemoryStore from zarr.storage._obstore import ObjectStore +from zarr.storage._url_pipeline import parse_pipeline, resolve_pipeline from zarr.storage._wrapper import WrapperStore from zarr.storage._zip import ZipStore @@ -25,6 +26,8 @@ "StorePath", "WrapperStore", "ZipStore", + "parse_pipeline", + "resolve_pipeline", ] diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index 72b5fc8a40..7bf294bfd3 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -21,9 +21,15 @@ AccessModeLiteral, ZarrFormat, ) -from zarr.errors import ContainsArrayAndGroupError, ContainsArrayError, ContainsGroupError +from zarr.errors import ( + ContainsArrayAndGroupError, + ContainsArrayError, + ContainsGroupError, + URLPipelineError, +) from zarr.storage._local import LocalStore from zarr.storage._memory import ManagedMemoryStore, MemoryStore +from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline from zarr.storage._utils import UPath, _join_paths, normalize_path, parse_store_url _has_fsspec = importlib.util.find_spec("fsspec") @@ -47,14 +53,22 @@ class StorePath: The store to use. path : str The path within the store. + + Attributes + ---------- + zarr_format : ZarrFormat | None + Zarr format selected by a `zarr2:`/`zarr3:` URL pipeline segment, + or None. Not part of the StorePath's identity (ignored by `__eq__`). """ store: Store path: str + zarr_format: ZarrFormat | None def __init__(self, store: Store, path: str = "") -> None: self.store = store self.path = normalize_path(path) + self.zarr_format = None @property def read_only(self) -> bool: @@ -72,7 +86,10 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No Open StorePath based on the provided mode. * If the mode is None, return an opened version of the store with no changes. - * If the mode is 'r+', 'w-', 'w', or 'a' and the store is read-only, raise a ValueError. + * If the mode is 'r+', 'w-', or 'w' and the store is read-only, raise a ValueError. + * If the mode is 'a' (open-or-create) and the store is read-only, serve the + "open" half: the StorePath is opened read-only, and any subsequent write + fails at the store level. * If the mode is 'r' and the store is not read-only, return a copy of the store with read_only set to True. * If the mode is 'w-' and the store is not read-only and the StorePath contains keys, raise a FileExistsError. * If the mode is 'w' and the store is not read-only, delete all keys nested within the StorePath. @@ -95,7 +112,7 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No FileExistsError If the mode is 'w-' and the store path already exists. ValueError - If the mode is not "r" and the store is read-only, or + If the mode is "r+", "w-", or "w" and the store is read-only, or """ # fastpath if mode is None @@ -106,8 +123,9 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") if store.read_only: - # Don't allow write operations on a read-only store - if mode != "r": + # mode "a" (open-or-create) on a read-only store means open-only; + # unambiguous write modes are rejected outright. + if mode not in ("r", "a"): raise ValueError( f"Store is read-only but mode is {mode!r}. Create a writable store or use 'r' mode." ) @@ -273,7 +291,9 @@ def delete_sync(self) -> None: def __truediv__(self, other: str) -> StorePath: """Combine this store path with another path""" - return self.__class__(self.store, _join_paths([self.path, other])) + result = self.__class__(self.store, _join_paths([self.path, other])) + result.zarr_format = self.zarr_format + return result def __str__(self) -> str: return _join_paths([str(self.store), self.path]) @@ -348,6 +368,16 @@ async def make_store( """ from zarr.storage._fsspec import FsspecStore # circular import + if isinstance(store_like, str) and is_url_pipeline(store_like): + result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options) + if normalize_path(result.path): + result.store.close() + raise URLPipelineError( + f"the URL pipeline {store_like!r} resolves to a path inside a store; " + "use zarr.open() or make_store_path() instead of make_store()" + ) + return result.store + # Parse URL early so we can reuse the result for both validation and routing parsed = parse_store_url(store_like) if isinstance(store_like, str) else None @@ -464,6 +494,13 @@ async def make_store_path( """ path_normalized = normalize_path(path) + if isinstance(store_like, str) and is_url_pipeline(store_like): + result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options) + combined_path = _join_paths([normalize_path(result.path), path_normalized]) + store_path = await StorePath.open(result.store, path=combined_path, mode=mode) + store_path.zarr_format = result.zarr_format + return store_path + if isinstance(store_like, StorePath): # Already a StorePath if storage_options: diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py new file mode 100644 index 0000000000..0161bdee6a --- /dev/null +++ b/src/zarr/storage/_url_pipeline.py @@ -0,0 +1,311 @@ +""" +Parsing and resolution of URL pipelines (https://github.com/jbms/url-pipeline). + +Importing this module is cheap and has no side effects: third-party adapters +(registered through the `zarr.url_adapters` entry-point group) are loaded +only when a pipeline URL naming their scheme is actually resolved. + +The `|` character is reserved as the pipeline delimiter in every string +store specification: a string containing `|` is always routed through the +pipeline machinery, and no percent-escape is decoded. To address a local +file whose *name* contains `|` (or `#`), pass a `pathlib.Path` instead of a +string. + +As a zarr-python extension to the specification (which requires the root +sub-URL of an absolute pipeline to carry a scheme), the root sub-URL may be +a schemeless local filesystem path, e.g. `data/example.zip|zip:`. Schemeless +roots are treated as opaque text — no query or fragment splitting is applied +to them. Such pipelines are not portable to other URL pipeline +implementations; portable pipelines should spell the root as a `file:` URL. +""" + +from __future__ import annotations + +import dataclasses +import re +from typing import TYPE_CHECKING, Any + +from zarr.abc.url_pipeline import ( + AdapterResolution, + PipelineContext, + PipelineSegment, +) +from zarr.errors import URLPipelineError +from zarr.registry import get_url_adapter, list_url_adapter_schemes +from zarr.storage._memory import ManagedMemoryStore +from zarr.storage._utils import parse_store_url + +if TYPE_CHECKING: + from zarr.abc.store import Store + from zarr.core.common import AccessModeLiteral + +__all__ = ["is_url_pipeline", "parse_pipeline", "resolve_pipeline"] + +# Adapter scheme per RFC 3986 plus "." to permit vendor-prefixed +# nonstandard schemes (e.g. "earthmover.myscheme"). +_SCHEME_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.\-]*$") + +# Root scheme at the very start of the sub-URL. The negative lookahead +# excludes fsspec's chained-URL syntax (``zip::file://...``), which is not a +# URL pipeline and keeps flowing through the fsspec machinery. +_ROOT_SCHEME_RE = re.compile(r"([a-zA-Z][a-zA-Z0-9+.\-]*):(?!:)") + +# Schemes zarr resolves natively at the pipeline root. These are never +# dispatched to a registered root adapter, so an installed package cannot +# intercept zarr's own local-path and in-memory routing. +_NATIVE_ROOT_SCHEMES = frozenset({"file", "memory"}) + +# An absolute Windows drive path (C:\... or C:/...), which counts as an +# absolute path in a `file:` pipeline root. +_WINDOWS_DRIVE_RE = re.compile(r"[A-Za-z]:[/\\]") + + +def _root_scheme(root_sub_url: str) -> str: + """ + Detect the scheme of the root sub-URL, or `""` for an opaque root. + + Scheme extraction happens on the raw string (so nothing `urlparse` + would strip or reject — whitespace, exotic authorities — can desync the + detected scheme from the body). Single-letter candidates are delegated + to `parse_store_url`, which knows Windows drive letters are not schemes. + """ + match = _ROOT_SCHEME_RE.match(root_sub_url) + if match is None: + return "" + scheme = match.group(1) + if len(scheme) == 1: + try: + return parse_store_url(root_sub_url).scheme.lower() + except ValueError: + return scheme.lower() + return scheme.lower() + + +def _split_query(sub_url: str) -> tuple[str, str | None]: + """Split a sub-URL on the first `?`. Fragments are not supported.""" + if "#" in sub_url: + raise URLPipelineError( + f"URL pipeline sub-URLs do not support fragments: {sub_url!r}. " + "Percent-encode '#' as '%23' if it is part of the path." + ) + body, sep, query = sub_url.partition("?") + return body, query if sep else None + + +def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]: + """ + Parse a URL pipeline into its `|`-delimited segments. + + The first segment is the *root* sub-URL; its scheme is detected with the + same rules as ordinary store URLs (Windows drive letters are not + schemes). Subsequent segments are *adapter* sub-URLs of the form + `scheme:body` where the trailing colon is optional when the body is + empty (`zip` is equivalent to `zip:`). + + A schemeless root (a bare local path) is accepted as a zarr-python + extension to the specification and treated as opaque text: no query or + fragment splitting applies, since `?` and `#` are ordinary filename + characters there. See the module docstring. + + Segment text is preserved verbatim (no case or percent-encoding + normalization) except that schemes are lowercased. + """ + parts = url.split("|") + if any(not part for part in parts): + raise URLPipelineError(f"URL pipeline contains an empty sub-URL: {url!r}") + + segments: list[PipelineSegment] = [] + for index, part in enumerate(parts): + if index == 0: + scheme = _root_scheme(part) + if not scheme: + # opaque root (bare local path, fsspec chained URL, ...) + segments.append(PipelineSegment(scheme="", body=part, query=None, raw=part)) + continue + body_and_scheme, query = _split_query(part) + body = body_and_scheme[len(scheme) + 1 :] + segments.append(PipelineSegment(scheme=scheme, body=body, query=query, raw=part)) + else: + body_and_scheme, query = _split_query(part) + scheme, _, body = body_and_scheme.partition(":") + scheme = scheme.lower() + if not _SCHEME_RE.match(scheme): + raise URLPipelineError( + f"invalid adapter scheme {scheme!r} in pipeline segment {part!r}" + ) + segments.append(PipelineSegment(scheme=scheme, body=body, query=query, raw=part)) + return tuple(segments) + + +def _root_routes_to_adapter(scheme: str) -> bool: + """ + Whether a root sub-URL with this scheme is dispatched to a registered + root adapter. Schemes zarr resolves natively (`file:`, `memory:`) and + opaque roots are excluded; the registry check inspects entry-point + names only — no adapter code is imported here. + """ + return ( + bool(scheme) and scheme not in _NATIVE_ROOT_SCHEMES and scheme in list_url_adapter_schemes() + ) + + +def is_url_pipeline(url: str) -> bool: + """ + Whether `url` should be routed through the URL pipeline machinery. + + True when the URL contains a `|` separator, or when its scheme has a + registered URL pipeline adapter (a *root adapter* such as `gh:`). + fsspec chained URLs (`zip::file://...`) and zarr's native `file:` / + `memory:` schemes are never routed to a root adapter. + """ + if "|" in url: + return True + return _root_routes_to_adapter(_root_scheme(url)) + + +async def resolve_pipeline( + url: str, + *, + mode: AccessModeLiteral | None = None, + storage_options: dict[str, Any] | None = None, +) -> AdapterResolution: + """ + Resolve a URL pipeline into a store and a residual path. + + Parameters + ---------- + url : str + A URL pipeline, e.g. `"s3://bucket/data.zip|zip:|zarr3:"`. + mode : AccessModeLiteral | None + The caller's access mode. `"r"` requires a read-only store; the + resolver enforces this on whatever the final adapter returns. + storage_options : dict | None + Options forwarded to the root sub-URL's store (and visible to + adapters via the context). Non-dict forms are reserved for future + per-segment configuration (one mapping per pipeline segment). + """ + segments = parse_pipeline(url) + if len(segments) == 1 and not _root_routes_to_adapter(segments[0].scheme): + raise URLPipelineError( + f"{url!r} is not a URL pipeline: it has no '|' separator and no " + f"URL pipeline adapter is registered for scheme {segments[0].scheme!r}" + ) + return await _resolve(segments, mode=mode, storage_options=storage_options) + + +async def _resolve( + segments: tuple[PipelineSegment, ...], + *, + mode: AccessModeLiteral | None, + storage_options: dict[str, Any] | None, +) -> AdapterResolution: + if len(segments) == 1 and not _root_routes_to_adapter(segments[0].scheme): + return AdapterResolution( + store=await _resolve_root(segments[0], mode=mode, storage_options=storage_options) + ) + + *preceding, last = segments + try: + adapter_cls = get_url_adapter(last.scheme) + except URLPipelineError as exc: + raise URLPipelineError( + f"{exc} Note: '|' is reserved as the URL pipeline delimiter; to " + "address a local file whose name contains '|', pass a " + "pathlib.Path instead of a string." + ) from None + + context = PipelineContext( + preceding=tuple(preceding), + mode=mode, + storage_options=storage_options, + ) + resolution = await adapter_cls.open_pipeline_segment(last, context) + if mode == "r" and not resolution.store.read_only: + # The caller required read-only; enforce it rather than trusting + # the adapter to have honored context.read_only. + try: + read_only_store = resolution.store.with_read_only(True) + except NotImplementedError as exc: + raise URLPipelineError( + f"adapter {last.scheme!r} returned a writable store for mode 'r', " + "and the store does not support read-only conversion via " + ".with_read_only()" + ) from exc + await read_only_store._ensure_open() + resolution = dataclasses.replace(resolution, store=read_only_store) + return resolution + + +async def _resolve_root( + segment: PipelineSegment, + *, + mode: AccessModeLiteral | None, + storage_options: dict[str, Any] | None, +) -> Store: + """ + Resolve the root sub-URL of a pipeline into a store. + + `memory:` and `file:` roots are resolved here with the URL pipeline + spec's semantics (spelling equivalences, mandatory absolute `file:` + paths); everything else — bare local paths, fsspec URLs — delegates to + the existing `StoreLike` machinery unchanged. + """ + if segment.scheme == "memory": + return _resolve_memory_root(segment, mode=mode, storage_options=storage_options) + from zarr.storage._common import make_store # circular import + + if segment.scheme == "file": + # Per the spec: file://localhost/p and file:///p are equivalent to + # file:/p; other authorities are unsupported; relative paths are + # forbidden; file: URLs carry no query. + if segment.query is not None: + raise URLPipelineError(f"'file:' pipeline roots do not accept a query: {segment.raw!r}") + body = segment.body + if body.startswith("//"): + authority, sep, rest = body[2:].partition("/") + if authority not in ("", "localhost"): + raise URLPipelineError( + f"unsupported authority {authority!r} in 'file:' pipeline " + f"root {segment.raw!r}; only an empty authority or " + "'localhost' is allowed" + ) + body = f"/{rest}" if sep else "" + # a file URL spells a Windows drive path as /C:/...; strip the + # leading slash so the local-path machinery sees the drive + if body.startswith("/") and _WINDOWS_DRIVE_RE.match(body[1:]): + body = body[1:] + if not (body.startswith("/") or _WINDOWS_DRIVE_RE.match(body)): + raise URLPipelineError( + f"'file:' pipeline roots must carry an absolute path: {segment.raw!r}" + ) + return await make_store(f"file:{body}", mode=mode, storage_options=storage_options) + + try: + return await make_store(segment.raw, mode=mode, storage_options=storage_options) + except ValueError as exc: + raise URLPipelineError( + f"could not resolve the pipeline root {segment.raw!r}: {exc}" + ) from exc + + +def _resolve_memory_root( + segment: PipelineSegment, + *, + mode: AccessModeLiteral | None, + storage_options: dict[str, Any] | None, +) -> Store: + """ + Resolve a `memory:` pipeline root per the spec's spelling equivalences: + `memory:` ≡ `memory:/` ≡ `memory://`, and `memory:a` ≡ `memory:/a` ≡ + `memory://a`. The first path component names the managed store; the + remainder is a path within it. + """ + if segment.query is not None: + raise URLPipelineError(f"'memory:' pipeline roots do not accept a query: {segment.raw!r}") + if storage_options: + raise TypeError( + "'storage_options' was provided but unused. " + "'storage_options' is only used when the store is passed as an FSSpec URI string.", + ) + name, _, path = segment.body.lstrip("/").partition("/") + return ManagedMemoryStore(name=name, path=path, read_only=mode == "r") diff --git a/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt b/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt index 7eb0eb7c86..456c725b4e 100644 --- a/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt +++ b/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt @@ -13,4 +13,6 @@ another_ndbuffer = package_with_entrypoint:TestEntrypointGroup.NDBuffer [zarr.codec_pipeline] another_pipeline = package_with_entrypoint:TestEntrypointGroup.Pipeline [zarr.data_type] -new_data_type = package_with_entrypoint:TestDataType \ No newline at end of file +new_data_type = package_with_entrypoint:TestDataType +[zarr.url_adapters] +example-pkg.entrypoint-scheme = package_with_entrypoint:TestEntrypointURLAdapter diff --git a/tests/package_with_entrypoint/__init__.py b/tests/package_with_entrypoint/__init__.py index 23afcf1dc2..cad73d061d 100644 --- a/tests/package_with_entrypoint/__init__.py +++ b/tests/package_with_entrypoint/__init__.py @@ -7,6 +7,7 @@ import zarr.core.buffer from zarr.abc.codec import ArrayBytesCodec, CodecInput, CodecPipeline +from zarr.abc.url_pipeline import AdapterResolution, URLPipelineAdapter from zarr.codecs import BytesCodec from zarr.core.buffer import Buffer, NDBuffer from zarr.core.dtype.npy.bool import Bool @@ -16,6 +17,7 @@ from collections.abc import Iterable from typing import Any, ClassVar, Literal, Self + from zarr.abc.url_pipeline import PipelineContext, PipelineSegment from zarr.core.array_spec import ArraySpec from zarr.core.common import ZarrFormat from zarr.core.dtype.common import DTypeJSON, DTypeSpec_V2 @@ -100,3 +102,16 @@ def to_json(self, zarr_format: ZarrFormat) -> str | DTypeSpec_V2: # type: ignor if zarr_format == 3: return self._zarr_v3_name raise ValueError("zarr_format must be 2 or 3") + + +class TestEntrypointURLAdapter(URLPipelineAdapter): + """URL pipeline adapter discovered via the zarr.url_adapters entry point.""" + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + from zarr.storage import MemoryStore + + store = await MemoryStore.open(read_only=context.read_only) + return AdapterResolution(store=store, path=segment.body) diff --git a/tests/test_api.py b/tests/test_api.py index 45d0c0dee4..c4e561c180 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -198,6 +198,9 @@ async def test_open_like_creates_array_by_default( async def test_open_like_default_mode_rejects_read_only_store( zarr_format: ZarrFormat, ) -> None: + # mode "a" (open-or-create) on a read-only store serves the "open" half: + # since the node does not exist and creating is impossible, the open + # fails with a not-found error rather than rejecting the mode upfront. ref_arr = zarr.create_array( store={}, shape=(11, 12), @@ -206,7 +209,7 @@ async def test_open_like_default_mode_rejects_read_only_store( zarr_format=zarr_format, ) - with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"): + with pytest.raises(ValueError, match="No array found"): await zarr.api.asynchronous.open_like( ref_arr, path="foo", diff --git a/tests/test_url_pipeline/__init__.py b/tests/test_url_pipeline/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_url_pipeline/conftest.py b/tests/test_url_pipeline/conftest.py new file mode 100644 index 0000000000..13c717e04e --- /dev/null +++ b/tests/test_url_pipeline/conftest.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +import zarr.registry + +if TYPE_CHECKING: + from collections.abc import Generator + + +@pytest.fixture +def clean_url_adapter_registry() -> Generator[None, None, None]: + """Snapshot and restore the URL adapter registry around a test.""" + registry = zarr.registry._url_adapter_registry + saved = dict(registry) + saved_lazy = list(registry.lazy_load_list) + yield + registry.clear() + registry.update(saved) + registry.lazy_load_list[:] = saved_lazy diff --git a/tests/test_url_pipeline/test_parser.py b/tests/test_url_pipeline/test_parser.py new file mode 100644 index 0000000000..13d7846e85 --- /dev/null +++ b/tests/test_url_pipeline/test_parser.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import sys + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from zarr.errors import URLPipelineError +from zarr.storage._url_pipeline import parse_pipeline + + +def test_single_root_url() -> None: + (segment,) = parse_pipeline("s3://bucket/key") + assert segment.scheme == "s3" + assert segment.body == "//bucket/key" + assert segment.query is None + assert segment.raw == "s3://bucket/key" + assert str(segment) == "s3://bucket/key" + + +def test_schemeless_root() -> None: + (segment,) = parse_pipeline("/local/path") + assert segment.scheme == "" + assert segment.body == "/local/path" + assert segment.raw == "/local/path" + + +def test_adapter_chain() -> None: + segments = parse_pipeline("s3://bucket/data.zip|zip:inner/path|zarr3:") + assert [s.scheme for s in segments] == ["s3", "zip", "zarr3"] + assert segments[1].body == "inner/path" + assert segments[2].body == "" + + +def test_trailing_colon_optional() -> None: + with_colon = parse_pipeline("file:/tmp/x.zip|zip:") + without_colon = parse_pipeline("file:/tmp/x.zip|zip") + assert with_colon[1].scheme == without_colon[1].scheme == "zip" + assert with_colon[1].body == without_colon[1].body == "" + + +def test_scheme_case_insensitive() -> None: + segments = parse_pipeline("FILE:/tmp/x.zip|ZIP:Inner/Path") + assert segments[0].scheme == "file" + assert segments[1].scheme == "zip" + # bodies are case-preserved + assert segments[1].body == "Inner/Path" + + +def test_case_preserved_in_raw() -> None: + # e.g. icechunk snapshot IDs are case-significant + segments = parse_pipeline("file:/tmp/repo|icechunk://ABCDEFGH12345678ABCD/x") + assert segments[1].raw == "icechunk://ABCDEFGH12345678ABCD/x" + assert segments[1].body == "//ABCDEFGH12345678ABCD/x" + + +def test_vendor_prefixed_scheme() -> None: + segments = parse_pipeline("file:/data|vendor-1.custom+adapter:sub/path") + assert segments[1].scheme == "vendor-1.custom+adapter" + assert segments[1].body == "sub/path" + + +def test_query_strings() -> None: + segments = parse_pipeline("https://example.com/d.zip?token=abc|zip:x?opt=1") + assert segments[0].query == "token=abc" + assert segments[0].body == "//example.com/d.zip" + assert segments[1].query == "opt=1" + assert segments[1].body == "x" + + +def test_empty_query() -> None: + (segment,) = parse_pipeline("https://example.com/d?") + assert segment.query == "" + + +def test_windows_drive_path_is_not_a_scheme() -> None: + # On Windows parse_store_url treats C:\... as a local path; elsewhere the + # single-letter scheme is preserved but must not crash the parser. + (segment,) = parse_pipeline(r"C:\data\store") + assert segment.raw == r"C:\data\store" + if sys.platform == "win32": + assert segment.scheme == "" + assert segment.body == r"C:\data\store" + else: + assert segment.scheme == "c" + + +@pytest.mark.parametrize("url", ["a||b:", "|zip:", "file:/tmp|", ""]) +def test_empty_sub_url_rejected(url: str) -> None: + with pytest.raises(URLPipelineError, match="empty sub-URL"): + parse_pipeline(url) + + +def test_fragment_rejected() -> None: + with pytest.raises(URLPipelineError, match="fragment"): + parse_pipeline("file:/tmp/x.zip|zip:inner#frag") + + +@pytest.mark.parametrize("segment", ["1zip:", "zi p:x", "zip@:x"]) +def test_invalid_adapter_scheme_rejected(segment: str) -> None: + with pytest.raises(URLPipelineError, match="invalid adapter scheme"): + parse_pipeline(f"file:/tmp/x|{segment}") + + +def test_single_letter_scheme_with_exotic_authority() -> None: + # single-letter candidates are delegated to parse_store_url, which may + # reject an exotic authority; the raw scheme is used as the fallback + segments = parse_pipeline("x://[authority]/path|zip:") + assert segments[0].scheme in ("", "x") + assert segments[0].raw == "x://[authority]/path" + + +def test_fsspec_chained_root_is_opaque() -> None: + # fsspec's ``scheme::`` chaining is not pipeline syntax; such roots are + # schemeless/opaque and flow through the ordinary store machinery + (segment,) = parse_pipeline("zip::file:///tmp/data.zip") + assert segment.scheme == "" + assert segment.body == "zip::file:///tmp/data.zip" + + +def test_whitespace_prefixed_root_is_opaque() -> None: + # urlparse strips leading whitespace when sniffing a scheme; the parser + # must not, or the detected scheme desyncs from the body slice + segments = parse_pipeline("\tfile:/tmp/x|zip:") + assert segments[0].scheme == "" + assert segments[0].body == "\tfile:/tmp/x" + + +def test_schemeless_root_keeps_query_and_fragment_chars() -> None: + # ? and # are ordinary filename characters in a bare local path + segments = parse_pipeline("/tmp/d?v=1|zip:") + assert segments[0].scheme == "" + assert segments[0].body == "/tmp/d?v=1" + assert segments[0].query is None + segments = parse_pipeline("/tmp/d#frag|zip:") + assert segments[0].body == "/tmp/d#frag" + + +def test_round_trip() -> None: + url = "s3://bucket/a.zip?v=2|zip:b/inner.zip|zip:c|zarr3:" + segments = parse_pipeline(url) + assert "|".join(s.raw for s in segments) == url + + +# --- property-based tests ----------------------------------------------- +# +# These exercise the parser's *splitting invariants*, not URL pipeline spec +# validity: parse_pipeline is a permissive segment splitter (bodies and +# queries are opaque text to it), and semantic validation is left to the +# resolver and the adapters. Rather than enumerating registered schemes, +# the strategies sample the full scheme grammar the parser accepts +# (RFC 3986 plus "." for vendor prefixes), so every valid root/adapter +# scheme is reachable. Bodies and queries draw from printable ASCII minus +# the characters the parser itself splits on ("|", "#", and "?" for +# bodies); the generated text is not required to be a well-formed URI. + +_ADAPTER_SCHEME = st.from_regex(r"[a-zA-Z][a-zA-Z0-9+.\-]{0,15}", fullmatch=True) +# two or more characters, so Windows drive-letter handling cannot reclassify +# the root scheme as a local path on one platform but not another +_ROOT_SCHEME = st.from_regex(r"[a-zA-Z]{2}[a-zA-Z0-9+.\-]{0,14}", fullmatch=True) +_BODY = st.text(st.characters(min_codepoint=32, max_codepoint=126, exclude_characters="|#?")) +# a root body starting with ":" would spell fsspec's chained-URL syntax +# ("scheme::..."), which the parser deliberately treats as an opaque root +_ROOT_BODY = _BODY.filter(lambda body: not body.startswith(":")) +_QUERY = st.text(st.characters(min_codepoint=32, max_codepoint=126, exclude_characters="|#")) + + +@st.composite +def pipelines( + draw: st.DrawFn, min_depth: int = 1 +) -> tuple[str, list[tuple[str, str, str | None, str]]]: + """A parseable pipeline URL up to depth 8, with its expected split.""" + depth = draw(st.integers(min_value=min_depth, max_value=8)) + expected = [] + parts = [] + for index in range(depth): + scheme = draw(_ROOT_SCHEME if index == 0 else _ADAPTER_SCHEME) + body = draw(_ROOT_BODY if index == 0 else _BODY) + query = draw(st.none() | _QUERY) + raw = f"{scheme}:{body}" + (f"?{query}" if query is not None else "") + parts.append(raw) + expected.append((scheme.lower(), body, query, raw)) + return "|".join(parts), expected + + +@given(pipelines()) +def test_valid_pipeline_invariants( + case: tuple[str, list[tuple[str, str, str | None, str]]], +) -> None: + url, expected = case + segments = parse_pipeline(url) + # schemes lowercased; bodies and queries preserved verbatim + assert [(s.scheme, s.body, s.query, s.raw) for s in segments] == expected + # lossless round trip + assert "|".join(s.raw for s in segments) == url + + +@given(pipelines(), st.data()) +def test_property_empty_sub_url_rejected( + case: tuple[str, list[tuple[str, str, str | None, str]]], data: st.DataObject +) -> None: + url, _ = case + parts = url.split("|") + parts.insert(data.draw(st.integers(0, len(parts))), "") + with pytest.raises(URLPipelineError, match="empty sub-URL"): + parse_pipeline("|".join(parts)) + + +@given(pipelines(min_depth=2), st.data()) +def test_property_invalid_adapter_scheme_rejected( + case: tuple[str, list[tuple[str, str, str | None, str]]], data: st.DataObject +) -> None: + url, _ = case + parts = url.split("|") + # corrupt one adapter segment's scheme with a leading character the + # scheme grammar forbids + index = data.draw(st.integers(1, len(parts) - 1)) + parts[index] = data.draw(st.sampled_from(["1", " ", "@", "~"])) + parts[index] + with pytest.raises(URLPipelineError, match="invalid adapter scheme"): + parse_pipeline("|".join(parts)) + + +@given(pipelines(), st.data()) +def test_property_fragment_rejected( + case: tuple[str, list[tuple[str, str, str | None, str]]], data: st.DataObject +) -> None: + url, _ = case + parts = url.split("|") + parts[data.draw(st.integers(0, len(parts) - 1))] += "#frag" + with pytest.raises(URLPipelineError, match="fragment"): + parse_pipeline("|".join(parts)) diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py new file mode 100644 index 0000000000..198c178764 --- /dev/null +++ b/tests/test_url_pipeline/test_resolver.py @@ -0,0 +1,569 @@ +from __future__ import annotations + +import dataclasses +import sys +from typing import TYPE_CHECKING, ClassVar + +import pytest + +import zarr +import zarr.registry +from zarr.abc.store import Store +from zarr.abc.url_pipeline import ( + AdapterResolution, + PipelineContext, + PipelineSegment, + URLPipelineAdapter, +) +from zarr.errors import URLPipelineError, ZarrUserWarning +from zarr.registry import ( + get_url_adapter, + list_url_adapter_schemes, + register_url_adapter, +) +from zarr.storage import ManagedMemoryStore, MemoryStore, WrapperStore +from zarr.storage._common import make_store, make_store_path +from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline +from zarr.storage._utils import _join_paths + +if TYPE_CHECKING: + from pathlib import Path + +pytestmark = pytest.mark.usefixtures("clean_url_adapter_registry") + + +class TracingStore(WrapperStore[Store]): + """Wrapper that records the context it was created from.""" + + context: PipelineContext + segment: PipelineSegment + + +class WrapperAdapter(URLPipelineAdapter): + """ + A wrapper-style adapter: resolves the preceding pipeline into a store. + + Follows the wrapper contract: the preceding resolution's residual path + is carried forward (joined with this segment's own path), and unchanged + fields survive via `dataclasses.replace`. + """ + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + preceding = await context.resolve_preceding() + store = TracingStore(preceding.store) + store.context = context + store.segment = segment + return dataclasses.replace( + preceding, store=store, path=_join_paths([preceding.path, segment.body]) + ) + + +class NativeAdapter(URLPipelineAdapter): + """A native-style adapter: consumes the preceding URL as a string.""" + + seen_urls: ClassVar[list[str]] = [] + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + cls.seen_urls.append(context.preceding_url) + store = await MemoryStore.open(read_only=context.read_only) + return AdapterResolution(store=store, path=segment.body) + + +class RootAdapter(URLPipelineAdapter): + """A root-scheme adapter (no preceding segments), like al://.""" + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + assert context.preceding == () + if context.mode in ("w", "w-", "r+"): + raise ValueError("read-only scheme") + store = await MemoryStore.open(read_only=True) + return AdapterResolution(store=store, path=segment.body.lstrip("/")) + + +class DisobedientAdapter(URLPipelineAdapter): + """An adapter that ignores `context.read_only` (a contract violation).""" + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + return AdapterResolution(store=await MemoryStore.open(read_only=False)) + + +class FormatAdapter(URLPipelineAdapter): + """A dummy format-selecting adapter, like the builtin `zarr2:`.""" + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + preceding = await context.resolve_preceding() + return dataclasses.replace(preceding, zarr_format=2) + + +class TestRegistry: + def test_register_and_get(self) -> None: + register_url_adapter("demo", WrapperAdapter) + assert get_url_adapter("demo") is WrapperAdapter + assert get_url_adapter("DEMO") is WrapperAdapter + assert "demo" in list_url_adapter_schemes() + + def test_unknown_scheme(self) -> None: + with pytest.raises(URLPipelineError, match="no URL pipeline adapter is registered"): + get_url_adapter("nonexistent-scheme") + + def test_reregistering_scheme_warns(self) -> None: + register_url_adapter("demo", WrapperAdapter) + with pytest.warns(ZarrUserWarning, match="is being replaced"): + register_url_adapter("demo", NativeAdapter) + assert get_url_adapter("demo") is NativeAdapter + # re-registering the same class is not a collision + register_url_adapter("demo", NativeAdapter) + + @pytest.mark.usefixtures("set_path") + def test_entrypoint_discovery(self) -> None: + assert "example-pkg.entrypoint-scheme" in list_url_adapter_schemes() + cls = get_url_adapter("example-pkg.entrypoint-scheme") + assert cls.__name__ == "TestEntrypointURLAdapter" + + @pytest.mark.usefixtures("set_path") + async def test_entrypoint_end_to_end(self) -> None: + result = await resolve_pipeline("memory://src|example-pkg.entrypoint-scheme:sub/path") + assert result.path == "sub/path" + + @pytest.mark.usefixtures("set_path") + def test_entrypoint_scheme_lookup_is_case_insensitive(self) -> None: + # entry-point names are matched case-insensitively, like schemes + cls = get_url_adapter("Example-PKG.Entrypoint-Scheme") + assert cls.__name__ == "TestEntrypointURLAdapter" + + @pytest.mark.usefixtures("set_path") + def test_loading_one_scheme_leaves_others_pending(self) -> None: + # resolving one scheme must not import other providers' entry points + registry = zarr.registry._url_adapter_registry + assert any(e.name == "example-pkg.entrypoint-scheme" for e in registry.lazy_load_list) + with pytest.raises(URLPipelineError, match="no URL pipeline adapter"): + get_url_adapter("some-other-scheme") + assert any(e.name == "example-pkg.entrypoint-scheme" for e in registry.lazy_load_list) + assert get_url_adapter("example-pkg.entrypoint-scheme").__name__ == ( + "TestEntrypointURLAdapter" + ) + + +class TestIsURLPipeline: + def test_pipe_routes(self) -> None: + assert is_url_pipeline("memory://x|demo:") + + def test_registered_root_scheme_routes(self) -> None: + register_url_adapter("rooty", RootAdapter) + assert is_url_pipeline("rooty://org/repo") + + @pytest.mark.parametrize("url", ["s3://bucket/key", "/local/path", "memory://x", "C:.zarr"]) + def test_plain_urls_do_not_route(self, url: str) -> None: + assert not is_url_pipeline(url) + + def test_fsspec_chained_urls_do_not_route(self) -> None: + # fsspec's ``scheme::`` chaining is not a URL pipeline: even with a + # same-named root adapter registered, these keep flowing to fsspec. + register_url_adapter("zip", RootAdapter) + assert not is_url_pipeline("zip::file:///tmp/data.zip") + assert not is_url_pipeline("simplecache::s3://bucket/key") + + def test_native_schemes_do_not_route_to_adapters(self) -> None: + # an installed package must not be able to intercept zarr's own + # local-path and in-memory routing by registering file:/memory: + register_url_adapter("file", RootAdapter) + register_url_adapter("memory", RootAdapter) + assert not is_url_pipeline("file:/tmp/x") + assert not is_url_pipeline("memory://x") + + def test_exotic_authority_root_scheme(self) -> None: + # The spec's own root-URL example: urlparse rejects the bracketed + # non-IP authority, so scheme detection must work on the raw string + # rather than raising before routing. + url = "vendor1-2.custom-1+proto.ext://[authority]/path?query/part?x" + assert not is_url_pipeline(url) + register_url_adapter("vendor1-2.custom-1+proto.ext", RootAdapter) + assert is_url_pipeline(url) + + +class TestResolve: + async def test_wrapper_adapter_chain(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{tmp_path}|wrap:inner/path") + assert isinstance(result.store, TracingStore) + assert result.path == "inner/path" + assert result.store.context.preceding_url == str(tmp_path) + + async def test_nested_wrappers_preserve_residual_paths(self, tmp_path: Path) -> None: + # each wrapper joins the preceding residual path with its own, so + # no segment's path is lost in root|wrap:a|wrap:b + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{tmp_path}|wrap:a|wrap:b") + assert result.path == "a/b" + + async def test_native_adapter_gets_preceding_url(self) -> None: + register_url_adapter("native", NativeAdapter) + NativeAdapter.seen_urls.clear() + await resolve_pipeline("s3://bucket/repo|native:") + assert NativeAdapter.seen_urls == ["s3://bucket/repo"] + + async def test_multi_segment_preceding_url(self) -> None: + register_url_adapter("wrap", WrapperAdapter) + register_url_adapter("native", NativeAdapter) + NativeAdapter.seen_urls.clear() + await resolve_pipeline("memory://base|wrap:a|native:x") + assert NativeAdapter.seen_urls == ["memory://base|wrap:a"] + + async def test_root_adapter(self) -> None: + register_url_adapter("rooty", RootAdapter) + result = await resolve_pipeline("rooty://org/repo") + assert result.path == "org/repo" + assert result.store.read_only + + async def test_root_adapter_composes_with_chain(self) -> None: + register_url_adapter("rooty", RootAdapter) + register_url_adapter("native", NativeAdapter) + NativeAdapter.seen_urls.clear() + await resolve_pipeline("rooty://org/repo|native:x") + assert NativeAdapter.seen_urls == ["rooty://org/repo"] + + async def test_read_only_flag(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{tmp_path}|wrap:", mode="r") + assert isinstance(result.store, TracingStore) + assert result.store.context.read_only + assert result.store.context.mode == "r" + result = await resolve_pipeline(f"{tmp_path}|wrap:") + assert isinstance(result.store, TracingStore) + assert not result.store.context.read_only + assert result.store.context.mode is None + + async def test_read_only_is_enforced_on_disobedient_adapters(self) -> None: + # the resolver downgrades a writable store returned under mode "r" + # instead of trusting the adapter to have honored context.read_only + register_url_adapter("bad", DisobedientAdapter) + result = await resolve_pipeline("memory://base|bad:", mode="r") + assert result.store.read_only + store = await make_store("memory://base|bad:", mode="r") + assert store.read_only + + async def test_read_only_enforcement_without_conversion_raises(self) -> None: + # a disobedient adapter whose store cannot be converted read-only + class StubbornStore(MemoryStore): + def with_read_only(self, read_only: bool = False) -> MemoryStore: + raise NotImplementedError + + class StubbornAdapter(URLPipelineAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + return AdapterResolution(store=await StubbornStore.open(read_only=False)) + + register_url_adapter("stubborn", StubbornAdapter) + with pytest.raises(URLPipelineError, match="does not support read-only conversion"): + await resolve_pipeline("memory://base|stubborn:", mode="r") + + async def test_resolve_preceding_mode_override(self, tmp_path: Path) -> None: + # a wrapper that only reads the preceding resource can open it + # read-only regardless of the caller's mode + class ReadOnlyRootWrapper(WrapperAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + preceding = await context.resolve_preceding(mode="r") + assert preceding.store.read_only + return dataclasses.replace(preceding, path=segment.body) + + register_url_adapter("rowrap", ReadOnlyRootWrapper) + result = await resolve_pipeline(f"{tmp_path}|rowrap:x", mode="w") + assert result.path == "x" + + async def test_resolve_preceding_storage_options_override(self, tmp_path: Path) -> None: + # an adapter that consumed its namespaced keys strips them before + # the root store ever sees them + class ConsumingWrapper(WrapperAdapter): + seen: ClassVar[object | None] = None + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + options = dict(context.storage_options or {}) + cls.seen = options.pop("consuming_secret", None) + preceding = await context.resolve_preceding(storage_options=options or None) + return dataclasses.replace(preceding, path=segment.body) + + register_url_adapter("consuming", ConsumingWrapper) + # the local-path root rejects any surviving storage_options, so this + # passing proves the adapter's keys were stripped before resolution + result = await resolve_pipeline( + f"{tmp_path}|consuming:x", storage_options={"consuming_secret": "s3cr3t"} + ) + assert result.path == "x" + assert ConsumingWrapper.seen == "s3cr3t" + + async def test_storage_options_visible_to_adapter(self) -> None: + register_url_adapter("native", NativeAdapter) + + class OptionsProbe(NativeAdapter): + seen_options: dict[str, object] | None = None + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + cls.seen_options = context.storage_options + return await super().open_pipeline_segment(segment, context) + + register_url_adapter("probe", OptionsProbe) + opts = {"anon": True} + await resolve_pipeline("s3://bucket/x|probe:", storage_options=opts) + assert OptionsProbe.seen_options == opts + + async def test_wrapper_at_root_position_raises(self) -> None: + # a wrapper adapter used as the pipeline root has nothing to wrap + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="no preceding sub-URL"): + await resolve_pipeline("wrap:whatever") + + async def test_not_a_pipeline_raises(self) -> None: + with pytest.raises(URLPipelineError, match="is not a URL pipeline"): + await resolve_pipeline("s3://bucket/plain") + + async def test_unknown_adapter_scheme_raises(self) -> None: + with pytest.raises(URLPipelineError, match="no URL pipeline adapter is registered"): + await resolve_pipeline("memory://base|no-such-adapter:") + + async def test_unknown_adapter_error_hints_at_reserved_pipe(self, tmp_path: Path) -> None: + # a filename containing '|' routes here; the error points at the fix + with pytest.raises(URLPipelineError, match="pass a\\s+pathlib.Path"): + await resolve_pipeline(f"{tmp_path}/a|b") + + async def test_base_case_value_error_is_wrapped(self) -> None: + # a root that only the fallback scheme detection accepts must not + # leak urlparse's ValueError out of the base case + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="could not resolve the pipeline root"): + await resolve_pipeline("vendor.x://[authority]/path|wrap:") + + async def test_zarr_format_flows_from_adapter(self, tmp_path: Path) -> None: + register_url_adapter("fmt2", FormatAdapter) + result = await resolve_pipeline(f"{tmp_path}|fmt2:") + assert result.zarr_format == 2 + # wrapper adapters carry the format through (dataclasses.replace) + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{tmp_path}|fmt2:|wrap:") + assert result.zarr_format == 2 + + +class TestSpecRoots: + """The spec's `memory:` / `file:` root semantics inside pipelines.""" + + @pytest.mark.parametrize("root", ["memory:", "memory:/", "memory://"]) + async def test_memory_root_spellings_equivalent(self, root: str) -> None: + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{root}|wrap:") + assert isinstance(result.store, TracingStore) + inner = result.store._store + assert isinstance(inner, ManagedMemoryStore) + assert inner._name == "" + assert inner.path == "" + + @pytest.mark.parametrize("root", ["memory:a/b", "memory:/a/b", "memory://a/b"]) + async def test_memory_root_named_spellings_equivalent(self, root: str) -> None: + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{root}|wrap:") + assert isinstance(result.store, TracingStore) + inner = result.store._store + assert isinstance(inner, ManagedMemoryStore) + assert inner._name == "a" + assert inner.path == "b" + + async def test_memory_root_shares_data_across_spellings(self) -> None: + register_url_adapter("wrap", WrapperAdapter) + group = zarr.open_group("memory:pipe-share|wrap:", mode="w") + group.create_array("x", shape=(2,), dtype="i4") + reopened = zarr.open_group("memory://pipe-share|wrap:", mode="r") + assert "x" in reopened + + async def test_memory_root_rejects_query(self) -> None: + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="do not accept a query"): + await resolve_pipeline("memory:a?opt=1|wrap:") + + async def test_memory_root_rejects_storage_options(self) -> None: + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(TypeError, match="'storage_options' was provided but unused"): + await resolve_pipeline("memory:a|wrap:", storage_options={"anon": True}) + + async def test_file_root_localhost_authority(self, tmp_path: Path) -> None: + # file://localhost/p and file:///p are equivalent to file:/p + register_url_adapter("wrap", WrapperAdapter) + # spell the path URL-style: on Windows C:/... gains a leading slash + posix = tmp_path.as_posix() + url_path = posix if posix.startswith("/") else f"/{posix}" + zarr.open_group(f"file://localhost{url_path}|wrap:", mode="w") + opened = zarr.open_group(f"file:{tmp_path}|wrap:", mode="r") + assert isinstance(opened, zarr.Group) + zarr.open_group(f"file://{url_path}|wrap:", mode="r") + + async def test_file_root_windows_drive_spellings( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # Windows drive paths count as absolute, and the URL spelling + # file:///C:/... has its leading slash stripped before the drive + register_url_adapter("wrap", WrapperAdapter) + if sys.platform == "win32": + drive_path = tmp_path.as_posix() # C:/Users/... + else: + # off Windows a drive path is just an odd relative directory; + # chdir into tmp so it is created there + monkeypatch.chdir(tmp_path) + drive_path = "C:/drive/data" + r1 = await resolve_pipeline(f"file:{drive_path}|wrap:") + r2 = await resolve_pipeline(f"file:///{drive_path}|wrap:") + assert isinstance(r1.store, TracingStore) + assert isinstance(r2.store, TracingStore) + + async def test_file_root_rejects_other_authority(self) -> None: + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="unsupported authority"): + await resolve_pipeline("file://example.com/tmp/x|wrap:") + + @pytest.mark.parametrize("root", ["file:relative/path", "file://localhost"]) + async def test_file_root_rejects_relative_paths(self, root: str) -> None: + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="absolute path"): + await resolve_pipeline(f"{root}|wrap:") + + async def test_file_root_rejects_query(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="do not accept a query"): + await resolve_pipeline(f"file:{tmp_path}?v=1|wrap:") + + +class TestMakeStoreIntegration: + async def test_make_store_path_combines_paths(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + store_path = await make_store_path(f"{tmp_path}|wrap:residual", path="user/sub") + assert store_path.path == "residual/user/sub" + + async def test_make_store_rejects_residual_path(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): + await make_store(f"{tmp_path}|wrap:residual") + + async def test_make_store_rejects_slash_root_path(self, tmp_path: Path) -> None: + # an adapter returning path="/" addresses the store root; the raw + # value is normalized before the residual-path rejection + class SlashRootAdapter(WrapperAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + preceding = await context.resolve_preceding() + return dataclasses.replace(preceding, path="/") + + register_url_adapter("slashy", SlashRootAdapter) + store = await make_store(f"{tmp_path}|slashy:") + assert store is not None + + async def test_make_store_closes_store_on_residual_path_error(self) -> None: + closed: list[bool] = [] + + class ClosingStore(MemoryStore): + def close(self) -> None: + closed.append(True) + super().close() + + class LeakProbe(URLPipelineAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + return AdapterResolution(store=await ClosingStore.open(), path="residual") + + register_url_adapter("leaky", LeakProbe) + with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): + await make_store("memory://base|leaky:") + assert closed == [True] + + async def test_make_store_no_residual_path(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + store = await make_store(f"{tmp_path}|wrap:") + assert isinstance(store, TracingStore) + + async def test_zarr_open_end_to_end(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + group = zarr.open_group(f"{tmp_path}|wrap:", mode="w") + array = group.create_array("x", shape=(4,), dtype="i4") + array[:] = [1, 2, 3, 4] + assert zarr.open_array(f"{tmp_path}|wrap:x")[2] == 3 + + async def test_pipeline_store_read_only_mode(self) -> None: + register_url_adapter("native", NativeAdapter) + store_path = await make_store_path("memory://base|native:", mode="r") + assert store_path.read_only + + async def test_mode_a_downgrades_to_read_only_open(self) -> None: + # mode "a" (the zarr.open default) is open-or-create: a pipeline + # that resolves to a read-only store serves the "open" half instead + # of failing outright. + register_url_adapter("rooty", RootAdapter) + store_path = await make_store_path("rooty://org/repo", mode="a") + assert store_path.read_only + + async def test_mode_a_downgrade_applies_to_plain_stores_too(self) -> None: + # the open-or-create downgrade lives in StorePath.open, so it is + # uniform across pipeline and non-pipeline stores + store = await MemoryStore.open(read_only=True) + store_path = await make_store_path(store, mode="a") + assert store_path.read_only + + async def test_explicit_write_mode_reaches_adapter(self) -> None: + register_url_adapter("rooty", RootAdapter) + with pytest.raises(ValueError, match="read-only scheme"): + await make_store_path("rooty://org/repo", mode="w") + + async def test_storage_options_forwarded_to_root(self) -> None: + # storage_options reach the root sub-URL via resolve_preceding -> + # make_store. A local-path root does not accept storage_options, so + # forwarding them must raise the same TypeError as a non-pipeline open. + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(TypeError, match="'storage_options' was provided but unused"): + await make_store("/tmp/some/path|wrap:", storage_options={"anon": True}) + + async def test_store_path_truediv_keeps_format(self, tmp_path: Path) -> None: + # StorePath.__truediv__ constructs a new instance; the pipeline- + # selected format must survive the division + register_url_adapter("fmt2", FormatAdapter) + store_path = await make_store_path(f"{tmp_path}|fmt2:") + assert store_path.zarr_format == 2 + assert (store_path / "sub/node").zarr_format == 2 + + async def test_zarr_format_merges_into_open(self, tmp_path: Path) -> None: + register_url_adapter("fmt2", FormatAdapter) + group = zarr.open_group(f"{tmp_path}|fmt2:", mode="w") + assert group.metadata.zarr_format == 2 + + async def test_conflicting_explicit_format_raises(self, tmp_path: Path) -> None: + register_url_adapter("fmt2", FormatAdapter) + with pytest.raises(ValueError, match="conflicts with"): + zarr.open_group(f"{tmp_path}|fmt2:", mode="w", zarr_format=3) + + async def test_matching_explicit_format_ok(self, tmp_path: Path) -> None: + register_url_adapter("fmt2", FormatAdapter) + group = zarr.open_group(f"{tmp_path}|fmt2:", mode="w", zarr_format=2) + assert group.metadata.zarr_format == 2 diff --git a/tests/test_url_pipeline/test_spec_examples.py b/tests/test_url_pipeline/test_spec_examples.py new file mode 100644 index 0000000000..5538074bea --- /dev/null +++ b/tests/test_url_pipeline/test_spec_examples.py @@ -0,0 +1,258 @@ +""" +Conformance tests against the URL pipeline specification. + +Every entry in `SPEC_EXAMPLES` is an example URL from the specification +repository (https://github.com/jbms/url-pipeline @ 1a01ce6), extracted +with the same rule as the spec's own linter (`scripts/lint.py`): backticked +examples on bullet lines, skipping spans without a `:` or `|`. The spec +validates each example (and an uppercased-scheme variant) against its ABNF +grammar, so this corpus is grammar-valid by construction. + +The parser must accept every example — including schemes zarr-python has no +adapter for, since parsing is independent of adapter availability — split it +into the expected sub-URL schemes, and preserve the text losslessly. + +To regenerate after a spec update: extract examples per the rule above and +recompute the expected scheme tuples with `parse_pipeline`, reviewing any +changes against the spec diff. +""" + +from __future__ import annotations + +import re + +import pytest + +from zarr.storage._url_pipeline import parse_pipeline + +SPEC_EXAMPLES: list[tuple[str, tuple[str, ...]]] = [ + # README.md + ("s3://bucket/path/to/archive.zip|zip:path/within/zip.zarr/|zarr3:", ("s3", "zip", "zarr3")), + ( + "file:///tmp/dataset.ocdbt/|ocdbt://2025-01-01T01:23:45.678Z/path/within/database", + ("file", "ocdbt"), + ), + ( + "s3+https://example.com/path/to/database.icechunk/|icechunk://tag.v5/path/to/node/|zarr3:", + ("s3+https", "icechunk", "zarr3"), + ), + ( + "vendor1-2.custom-1+proto.ext://[authority]/path?query/part?x", + ("vendor1-2.custom-1+proto.ext",), + ), + ( + "http://example.com/file|vendor1-2.custom+adapter:/path/within/adapter", + ("http", "vendor1-2.custom+adapter"), + ), + ("a.b:?", ("a.b",)), + ("http://somehost/downloads/somefile.zip|zip:", ("http", "zip")), + ("http://example.com/archive.jar|zip:path/to/file.txt", ("http", "zip")), + ("https://host/archive.zip|zip:path/in/outer.zip|zip:path/in/inner", ("https", "zip", "zip")), + ("file:///path/to/archive.zip|zip:path/within/archive", ("file", "zip")), + # avif.md + ("file:/path/to/image.avif|avif:", ("file", "avif")), + ("file:/path/to/image.avif|avif", ("file", "avif")), + # bmp.md + ("file:/path/to/image.bmp|bmp:", ("file", "bmp")), + ("file:/path/to/image.bmp|bmp", ("file", "bmp")), + # byte-range.md + ("file:/path/to/data|byte-range:1000-2000", ("file", "byte-range")), + ("file:/path/to/data|byte-range:0-1", ("file", "byte-range")), + # file.md + ("file:/", ("file",)), + ("file:/a", ("file",)), + ("file:/path/to/file.txt", ("file",)), + ("file://localhost/path/to/file.txt", ("file",)), + ("file://LOCALHOST/path/to/file.txt", ("file",)), + ("file:///path/to/file.txt", ("file",)), + ("file://somehost/sharename/path/to/file.txt", ("file",)), + # gs.md + ("gs://bucket", ("gs",)), + ("gs://bucket/", ("gs",)), + ("gs://bucket/path/within/bucket", ("gs",)), + ("gs://aaa", ("gs",)), + ( + "gs://label1-is-sixty-two-characters-long-xxxxxxxxxxxxxxxxxxxxxxxxxx.label2-is-sixty-two-characters-long-yyyyyyyyyyyyyyyyyyyyyyyyyy.label3-is-sixty-two-characters-long-zzzzzzzzzzzzzzzzzzzzzzzzzz.label4-is-thirty-one-characters", + ("gs",), + ), + # gzip.md + ("file:/path/to/data.gz|gzip:", ("file", "gzip")), + ("file:/path/to/data.gz|gzip", ("file", "gzip")), + # hdf5.md + ("s3://bucket/path/to/file.h5|hdf5:/path/to/dataset", ("s3", "hdf5")), + ("s3://bucket/path/to/file.h5|hdf5:a", ("s3", "hdf5")), + ("file:///path/to/file.h5|hdf5:", ("file", "hdf5")), + ("file:///path/to/file.h5|hdf5", ("file", "hdf5")), + # http.md + ("https://example.com/path/to/resource%20name", ("https",)), + ("https://example.com/path/to/resource?query=value", ("https",)), + ("https://example.com/path/to/?query=value", ("https",)), + ("https://example.com/path/with:colon", ("https",)), + ("https://server.example.com:1234/path/to/array", ("https",)), + ("http://a", ("http",)), + ("http://example.com", ("http",)), + ("http://local%68ost", ("http",)), + ("http://example.com:", ("http",)), + ("http://192.168.10.1:1234", ("http",)), + ("http://[::1]", ("http",)), + ("http://[::ffff:127.0.0.1]", ("http",)), + ("http://example.com/", ("http",)), + # icechunk.md + ("file:///path/to/repo.zarr.icechunk/|icechunk:", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk://branch.main/", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk:path/to/node/", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk:a", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk:/path/to/node/", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk:path/to/node/zarr.json", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk:path/to/node/c/0/0/1", ("file", "icechunk")), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://branch.mybranch/path/to/node/", + ("file", "icechunk"), + ), + ("file:///path/to/repo.zarr.icechunk/|icechunk://branch.a", ("file", "icechunk")), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://tag.mytag/path/to/node/", + ("file", "icechunk"), + ), + ("file:///path/to/repo.zarr.icechunk/|icechunk://tag.a", ("file", "icechunk")), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://FWWFQGAW742XMX0F5MF0/path/to/node/", + ("file", "icechunk"), + ), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://ABCDEFGHJKMNPQRSTVWX/path/to/node/", + ("file", "icechunk"), + ), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk:|zarr3:path/to/array/", + ("file", "icechunk", "zarr3"), + ), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://branch.other/|zarr3:path/to/array/", + ("file", "icechunk", "zarr3"), + ), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://tag.v5/|zarr3:path/to/array/", + ("file", "icechunk", "zarr3"), + ), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://4N0217AZA4VNPYD0HR0G/|zarr3:path/to/array/", + ("file", "icechunk", "zarr3"), + ), + # jpeg.md + ("file:/path/to/image.jpeg|jpeg:", ("file", "jpeg")), + ("file:/path/to/image.jpeg|jpeg", ("file", "jpeg")), + # json.md + ("file:/path/to/data.json|json:", ("file", "json")), + ("file:/path/to/data.json|json", ("file", "json")), + ("file:/path/to/data.json|json:/path/to/node", ("file", "json")), + ("file:/path/to/data.json|json:/abc~0def", ("file", "json")), + ("file:/path/to/data.json|json:/abc~1def", ("file", "json")), + ("file:/path/to/data.json|json:/", ("file", "json")), + # memory.md + ("memory:", ("memory",)), + ("memory:/", ("memory",)), + ("memory://", ("memory",)), + ("memory:path/to/resource", ("memory",)), + ("memory:a", ("memory",)), + ("memory:another@path+with,lots&of(special;characters)*_!-$'", ("memory",)), + ("memory:/path/to/resource", ("memory",)), + ("memory://path/to/resource", ("memory",)), + # n5.md + ("file:///tmp/data.n5/|n5:path/to/array", ("file", "n5")), + ("file:///tmp/data.n5/|n5:a", ("file", "n5")), + ("file:///tmp/data.n5/|n5:/path/to/array", ("file", "n5")), + ("file:///tmp/data.n5/|n5:", ("file", "n5")), + ("file:///tmp/data.n5/|n5", ("file", "n5")), + # neuroglancer-precomputed.md + ( + "file:///tmp/dataset.precomputed/|neuroglancer-precomputed:", + ("file", "neuroglancer-precomputed"), + ), + ( + "file:///tmp/dataset.precomputed/|neuroglancer-precomputed", + ("file", "neuroglancer-precomputed"), + ), + # ocdbt.md + ("file:///path/to/repo.ocdbt/|ocdbt:", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt:path/within/repo/", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt:path/within/repo", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt:a", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt:/path/within/repo", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt://v123/", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt://v1", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt://v123/path/within/repo", ("file", "ocdbt")), + ( + "file:///path/to/repo.ocdbt/|ocdbt://2025-01-01T01:23:45.678Z/path/within/database", + ("file", "ocdbt"), + ), + ("file:///path/to/repo.ocdbt/|ocdbt://2025-01-01T01:23:45Z", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt://2025-01-01T01:23:45.1Z", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt://2025-01-01T01:23:45.123456789Z", ("file", "ocdbt")), + # png.md + ("file:/path/to/image.png|png:", ("file", "png")), + ("file:/path/to/image.png|png", ("file", "png")), + # s3+http.md + ("s3+https://endpoint/path/within/bucket", ("s3+https",)), + ("s3+https://endpoint/bucket/path/within/bucket", ("s3+https",)), + ("s3+https://mybucket.s3.amazonaws.com/path/to/file", ("s3+https",)), + ("s3+https://s3.amazonaws.com/mybucket/path/to/file", ("s3+https",)), + ("s3+http://example.com", ("s3+http",)), + ("s3+http://example.com/", ("s3+http",)), + # s3.md + ("s3://bucket", ("s3",)), + ("s3://bucket/", ("s3",)), + ("s3://bucket/path/within/bucket", ("s3",)), + ("s3://aaa", ("s3",)), + ( + "s3://aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ("s3",), + ), + # tiff.md + ("file:/path/to/image.tiff|tiff:", ("file", "tiff")), + ("file:/path/to/image.tiff|tiff", ("file", "tiff")), + # webp.md + ("file:/path/to/image.webp|webp:", ("file", "webp")), + ("file:/path/to/image.webp|webp", ("file", "webp")), + # zarr.md + ("file:///path/to/node.zarr/|zarr3:", ("file", "zarr3")), + ("file:///path/to/node.zarr/|zarr3", ("file", "zarr3")), + ("file:///path/to/node.zarr/|zarr2:", ("file", "zarr2")), + ("file:///path/to/node.zarr/|zarr2", ("file", "zarr2")), + ("file:///path/to/node.zarr/|zarr:", ("file", "zarr")), + ("file:///path/to/node.zarr/|zarr", ("file", "zarr")), + # zip.md + ("file:/path/to/archive.zip|zip:path/to/file.txt", ("file", "zip")), + ("file:/path/to/archive.zip|zip", ("file", "zip")), + ("file:/path/to/archive.zip|zip:/path/to/file.txt", ("file", "zip")), + ("file:/path/to/outer.zip|zip:path/to/inner.zip|zip:path/to/file.txt", ("file", "zip", "zip")), + # zstd.md + ("file:/path/to/data.zstd|zstd:", ("file", "zstd")), + ("file:/path/to/data.zstd|zstd", ("file", "zstd")), +] + + +def _uppercase_schemes(example: str) -> str: + """Uppercase every sub-URL scheme, as the spec's linter does.""" + + def repl(match: re.Match[str]) -> str: + return match.group(1) + match.group(2).upper() + + return re.sub(r"((?:^|\|)\s*)([a-zA-Z][a-zA-Z0-9+.-]*)", repl, example) + + +@pytest.mark.parametrize(("url", "expected_schemes"), SPEC_EXAMPLES, ids=lambda v: str(v)) +def test_spec_example_parses(url: str, expected_schemes: tuple[str, ...]) -> None: + segments = parse_pipeline(url) + assert tuple(segment.scheme for segment in segments) == expected_schemes + # lossless round-trip of the original text + assert "|".join(segment.raw for segment in segments) == url + + +@pytest.mark.parametrize(("url", "expected_schemes"), SPEC_EXAMPLES, ids=lambda v: str(v)) +def test_spec_example_schemes_case_insensitive(url: str, expected_schemes: tuple[str, ...]) -> None: + upper = _uppercase_schemes(url) + segments = parse_pipeline(upper) + assert tuple(segment.scheme for segment in segments) == expected_schemes From 581c5349403610d7062983cd0d54f35269bd68b0 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 16 Sep 2026 19:39:56 -0700 Subject: [PATCH 02/20] polish: validate mode before pipeline resolution, document adapter authoring - make_store validates the access mode before routing a URL pipeline string to adapters, matching every other StoreLike branch - make_store docstring lists URL pipeline strings; StorePath.open documents the invalid-mode ValueError - user guide: StoreLike bullet for pipeline strings and an adapter authoring section in extending.md (runnable example) - changelog: drop PR-relative wording, name the public parser entry points Co-Authored-By: Claude Fable 5.1 --- changes/4192.feature.md | 7 ++-- docs/user-guide/extending.md | 48 ++++++++++++++++++++++++ docs/user-guide/storage.md | 3 ++ src/zarr/storage/_common.py | 14 ++++--- tests/test_url_pipeline/test_resolver.py | 7 ++++ 5 files changed, 71 insertions(+), 8 deletions(-) diff --git a/changes/4192.feature.md b/changes/4192.feature.md index 757aa13895..30f11a559d 100644 --- a/changes/4192.feature.md +++ b/changes/4192.feature.md @@ -1,11 +1,12 @@ Added core support for URL pipelines (https://github.com/jbms/url-pipeline): `|`-chained URLs that address zarr data through nested storage layers, e.g. -`s3://bucket/data.zip|zip:|zarr3:`. This PR adds the parser, the single-method -`zarr.abc.url_pipeline.URLPipelineAdapter` interface, and the +`s3://bucket/data.zip|zip:|zarr3:`. This includes the parser +(`zarr.storage.parse_pipeline` / `zarr.storage.resolve_pipeline`), the +single-method `zarr.abc.url_pipeline.URLPipelineAdapter` interface, and the `zarr.url_adapters` entry-point group through which third-party packages (e.g. Icechunk) register adapters for their own schemes. Adapters for a scheme are loaded lazily and individually. Builtin adapters (`zip:`, -`zarr2:`/`zarr3:`) follow in separate pull requests. +`zarr2:`/`zarr3:`) will follow separately. Behavior notes: diff --git a/docs/user-guide/extending.md b/docs/user-guide/extending.md index 507afedea7..db5769d348 100644 --- a/docs/user-guide/extending.md +++ b/docs/user-guide/extending.md @@ -85,6 +85,54 @@ implementation. Custom stores can be created by implementing the [`zarr.abc.store.Store`][] interface. See [developing custom stores](storage.md#developing-custom-stores) for more information. +## Custom URL pipeline adapters + +[URL pipelines](storage.md#user-guide-url-pipelines) are `|`-chained URLs such as +`s3://bucket/data.zip|zip:|zarr3:`. Each sub-URL after the first names an *adapter* +that reinterprets everything to its left. A package provides an adapter for a scheme by +subclassing [`zarr.abc.url_pipeline.URLPipelineAdapter`][] and implementing its single +classmethod, which resolves the adapter's segment into an open store and a residual path: + +```python test="true" session="url-adapter" +import dataclasses + +from zarr.abc.url_pipeline import ( + AdapterResolution, + PipelineContext, + PipelineSegment, + URLPipelineAdapter, +) +from zarr.storage import WrapperStore + + +class MyAdapter(URLPipelineAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + # A *wrapper* adapter opens the resource to its left and wraps it. Carry the + # preceding residual path forward and keep unchanged fields via dataclasses.replace. + preceding = await context.resolve_preceding() + store = WrapperStore(preceding.store) + return dataclasses.replace(preceding, store=store, path=segment.body) +``` + +Register the class under the `zarr.url_adapters` entry-point group, using the URL +scheme as the entry-point name. Nonstandard schemes should be vendor-prefixed +(`vendor.scheme`), per the URL pipeline specification: + +```toml +[project.entry-points."zarr.url_adapters"] +"mypackage.myscheme" = "mypackage.zarr_adapter:MyAdapter" +``` + +Adapters are loaded lazily, only when a pipeline naming their scheme is resolved. An +adapter may also be used as a *root* scheme (e.g. `mypackage.myscheme://org/repo`), in +which case `context.preceding` is empty. The returned store must honor +`context.read_only`, and `open_pipeline_segment` runs on zarr's internal event loop, so +it must not block or call zarr's synchronous API. See +[`zarr.abc.url_pipeline`][] for the full contract. + ## Custom array buffers Zarr-python provides control over where and how arrays are stored in memory through diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index 2e84f90ed7..d572c5bc1d 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -100,6 +100,9 @@ print(group) group = zarr.open_group(UPath('s3://noaa-nwm-retro-v2-zarr-pds', anon=True), mode='r') ``` +- a [URL pipeline](#user-guide-url-pipelines) string containing `|`, such as + `s3://bucket/data.zip|zip:`, which is resolved through registered adapters. + - a [`Store`][zarr.abc.store.Store] or [`StorePath`][zarr.storage.StorePath] - see explicit store creation below. diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index 7a93141c60..f6cdfee8d7 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -112,7 +112,8 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No FileExistsError If the mode is 'w-' and the store path already exists. ValueError - If the mode is "r+", "w-", or "w" and the store is read-only, or + If the mode is "r+", "w-", or "w" and the store is read-only, or if the + mode is not a valid access mode. """ # fastpath if mode is None @@ -342,6 +343,9 @@ async def make_store( - `None` = `MemoryStore` object. - `FSMap` = `FsspecStore` object. - `UPath` = `FsspecStore` object, or `LocalStore` for a local `UPath`. + - `str` containing `|` (or whose scheme has a registered root adapter) = the store + resolved by the [URL pipeline][user-guide-url-pipelines]; the pipeline must address + the store root (use `make_store_path` for pipelines that address a path within a store). Parameters ---------- @@ -368,6 +372,10 @@ async def make_store( """ from zarr.storage._fsspec import FsspecStore # circular import + if mode is not None and mode not in ANY_ACCESS_MODE: + raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") + _read_only = mode == "r" + if isinstance(store_like, str) and is_url_pipeline(store_like): result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options) if normalize_path(result.path): @@ -390,10 +398,6 @@ async def make_store( "'storage_options' is only used when the store is passed as an FSSpec URI string.", ) - if mode is not None and mode not in ANY_ACCESS_MODE: - raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") - _read_only = mode == "r" - if isinstance(store_like, StorePath): # Get underlying store return store_like.store diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index 198c178764..75582cdc75 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -505,6 +505,13 @@ async def test_make_store_no_residual_path(self, tmp_path: Path) -> None: store = await make_store(f"{tmp_path}|wrap:") assert isinstance(store, TracingStore) + async def test_make_store_rejects_invalid_mode_before_resolving(self, tmp_path: Path) -> None: + # an invalid mode is rejected upfront, like every other StoreLike, + # rather than being handed to adapters + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(ValueError, match="Invalid mode"): + await make_store(f"{tmp_path}|wrap:", mode="invalid") # type: ignore[arg-type] + async def test_zarr_open_end_to_end(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) group = zarr.open_group(f"{tmp_path}|wrap:", mode="w") From 3ed2bb8835aa89488ed5af2a288585f687a5586a Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 17 Sep 2026 14:17:39 -0700 Subject: [PATCH 03/20] fix: load URL adapter entry points outside the registry lock get_url_adapter took a non-reentrant lock across entry_point.load(), so an adapter module that resolved another scheme at import time deadlocked the process. Matching entry points are now taken off the pending list under the lock and imported outside it; import failures are wrapped in URLPipelineError and leave the entry point discoverable for a retry. A same-named entry point is discarded with a ZarrUserWarning when the scheme is already registered (e.g. by a builtin), instead of staying pending forever; duplicate entry-point names warn and the first wins. Co-Authored-By: Claude Fable 5.1 --- src/zarr/registry.py | 67 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 13 deletions(-) diff --git a/src/zarr/registry.py b/src/zarr/registry.py index 3487f87268..990ee51572 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -554,27 +554,68 @@ def get_url_adapter(scheme: str) -> type[URLPipelineAdapter]: Get the URL pipeline adapter class registered for `scheme`. Loads pending `zarr.url_adapters` entry points for this scheme only, so - resolving one scheme never imports other providers' packages. + resolving one scheme never imports other providers' packages. When the + scheme is already registered (e.g. a builtin adapter), a same-named entry + point is discarded with a [`ZarrUserWarning`][zarr.errors.ZarrUserWarning]; + when several entry points share the name, the first one wins and a + warning names the ones ignored. + + Raises + ------ + URLPipelineError + If no adapter is registered for the scheme, or if the entry point + providing it fails to import. """ key = scheme.lower() - # The lock keeps concurrent first-time resolutions of different schemes - # from clobbering each other's rebuild of the pending entry-point list. + # Take the matching entry points out of the pending list under the lock, + # so concurrent first-time resolutions of different schemes never clobber + # each other's view of the list. The import itself runs *outside* the lock: + # it may take arbitrarily long and may itself resolve URL adapters. with _url_adapter_lock: - if key not in _url_adapter_registry: - remaining = [] - for entry_point in _url_adapter_registry.lazy_load_list: - if entry_point.name.lower() == key: - _url_adapter_registry.register(entry_point.load(), qualname=key) - else: - remaining.append(entry_point) - _url_adapter_registry.lazy_load_list[:] = remaining + registered = _url_adapter_registry.get(key) + pending = [e for e in _url_adapter_registry.lazy_load_list if e.name.lower() == key] + if pending: + _url_adapter_registry.lazy_load_list[:] = [ + e for e in _url_adapter_registry.lazy_load_list if e.name.lower() != key + ] + if pending and registered is not None: + warnings.warn( + f"URL pipeline adapter for scheme {scheme!r} is already registered " + f"({fully_qualified_name(registered)}); ignoring entry point(s) " + f"{[e.value for e in pending]} of the same name", + category=ZarrUserWarning, + stacklevel=2, + ) + elif pending: + entry_point, *duplicates = pending + if duplicates: + warnings.warn( + f"multiple 'zarr.url_adapters' entry points are named {entry_point.name!r}; " + f"using {entry_point.value!r} and ignoring {[e.value for e in duplicates]}", + category=ZarrUserWarning, + stacklevel=2, + ) + try: + cls = entry_point.load() + except Exception as exc: + # leave the entry points discoverable so a transient import + # failure is not permanent + with _url_adapter_lock: + _url_adapter_registry.lazy_load_list.extend(pending) + raise URLPipelineError( + f"the 'zarr.url_adapters' entry point {entry_point.name!r} " + f"({entry_point.value!r}) could not be loaded: {exc!r}" + ) from exc + with _url_adapter_lock: + if key not in _url_adapter_registry: + _url_adapter_registry.register(cls, qualname=key) try: return _url_adapter_registry[key] except KeyError: - registered = sorted(list_url_adapter_schemes()) + registered_schemes = sorted(list_url_adapter_schemes()) raise URLPipelineError( f"no URL pipeline adapter is registered for scheme {scheme!r}. " - f"Registered schemes: {registered}. Adapters are provided by " + f"Registered schemes: {registered_schemes}. Adapters are provided by " "packages via the 'zarr.url_adapters' entry-point group." ) from None From 9bf86cebb333b837002bd7c77607cfb4ad14e205 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 17 Sep 2026 14:17:46 -0700 Subject: [PATCH 04/20] fix: close the adapter's store when read-only conversion fails; document contracts - The resolver closes the store an adapter returned when it cannot be made read-only for mode 'r', instead of leaking it. - resolve_pipeline documents every error type it lets through; the resolve_preceding docstring states the local-file-root limitation. - AdapterResolution and PipelineContext compare and hash by identity, since a Store is not hashable and frozen=True implied otherwise. Co-Authored-By: Claude Fable 5.1 --- src/zarr/abc/url_pipeline.py | 16 +++++++++++++--- src/zarr/storage/_url_pipeline.py | 19 ++++++++++++++++++- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/zarr/abc/url_pipeline.py b/src/zarr/abc/url_pipeline.py index 0e0363bcf6..337563685d 100644 --- a/src/zarr/abc/url_pipeline.py +++ b/src/zarr/abc/url_pipeline.py @@ -73,11 +73,14 @@ def __str__(self) -> str: return self.raw -@dataclass(frozen=True) +@dataclass(frozen=True, eq=False) class AdapterResolution: """ The result of resolving a URL pipeline (or a prefix of one). + Instances are immutable but compare and hash by identity, since a + [`Store`][zarr.abc.store.Store] is not hashable. + Attributes ---------- store : Store @@ -99,12 +102,14 @@ class AdapterResolution: zarr_format: ZarrFormat | None = None -@dataclass(frozen=True) +@dataclass(frozen=True, eq=False) class PipelineContext: """ Context handed to a [`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] describing the pipeline to the left of its segment. + Instances are immutable but compare and hash by identity. + Attributes ---------- preceding : tuple[PipelineSegment, ...] @@ -186,7 +191,12 @@ async def resolve_preceding( Wrapper adapters that only read the preceding resource should pass `mode="r"` so the root is opened read-only and without create-on-open side effects, regardless of the caller's mode. - When omitted, the caller's mode is used. + When omitted, the caller's mode is used. Note that a local + *file* root (e.g. `data.zip|zip:`) can currently only be + resolved in mode `"r"`: every other mode creates a directory at + the root and fails with `FileExistsError`, because there is no + file-resource primitive yet. Writing *through* a container file + is therefore not supported by wrapper adapters at this time. storage_options : dict | None, optional Override the options forwarded to the preceding pipeline. An adapter that consumed adapter-specific keys should pass the diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py index 0161bdee6a..41a2114daf 100644 --- a/src/zarr/storage/_url_pipeline.py +++ b/src/zarr/storage/_url_pipeline.py @@ -183,6 +183,19 @@ async def resolve_pipeline( Options forwarded to the root sub-URL's store (and visible to adapters via the context). Non-dict forms are reserved for future per-segment configuration (one mapping per pipeline segment). + + Raises + ------ + URLPipelineError + If the URL cannot be parsed, is not a pipeline, names a scheme with + no registered adapter, names an adapter entry point that fails to + import, or has a root sub-URL that cannot be resolved. + TypeError + If `storage_options` are passed to a root that does not accept them + (local paths and `memory:`), as for non-pipeline stores. + OSError + Errors from opening the root resource (e.g. a missing local directory + in mode `"r"`) propagate unchanged, as for non-pipeline stores. """ segments = parse_pipeline(url) if len(segments) == 1 and not _root_routes_to_adapter(segments[0].scheme): @@ -225,13 +238,17 @@ async def _resolve( # the adapter to have honored context.read_only. try: read_only_store = resolution.store.with_read_only(True) + await read_only_store._ensure_open() except NotImplementedError as exc: + resolution.store.close() raise URLPipelineError( f"adapter {last.scheme!r} returned a writable store for mode 'r', " "and the store does not support read-only conversion via " ".with_read_only()" ) from exc - await read_only_store._ensure_open() + except Exception: + resolution.store.close() + raise resolution = dataclasses.replace(resolution, store=read_only_store) return resolution From c1499610ccdc3720be491e539a02737852cfad3f Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 17 Sep 2026 14:17:49 -0700 Subject: [PATCH 05/20] fix: honor the pipeline-selected zarr format at every make_store_path caller The zarr2:/zarr3: format carried on StorePath.zarr_format was merged only in zarr.api.asynchronous; create_array, from_array, Group.from_store, Group.open, Array.open and the deprecated AsyncArray._create dropped it and swallowed explicit conflicts. The merge now lives on StorePath.resolve_zarr_format and is applied at each site. To let the pipeline's format apply when the caller does not specify one, create_array (sync and async) and Group.from_store default zarr_format to None, resolving to the configured default (3) as before; Array.open gains a zarr_format parameter mirroring Group.open. Tests: registry deadlock/race/import-failure/shadowing, resolver leak, zarr_format merge at each core site, and pins for the documented memory:/ file: divergences and the local-file-root limitation (xfail, strict). Co-Authored-By: Claude Fable 5.1 --- src/zarr/api/asynchronous.py | 32 +-- src/zarr/api/synchronous.py | 2 +- src/zarr/core/array.py | 14 +- src/zarr/core/group.py | 14 +- src/zarr/storage/_common.py | 33 +++ tests/test_api.py | 2 +- tests/test_url_pipeline/test_resolver.py | 297 ++++++++++++++++++++++- 7 files changed, 359 insertions(+), 35 deletions(-) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 0648769727..c471934b6f 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -108,24 +108,6 @@ def _infer_overwrite(mode: AccessModeLiteral) -> bool: return mode in _OVERWRITE_MODES -def _merge_pipeline_zarr_format( - store_path: StorePath, zarr_format: ZarrFormat | None -) -> ZarrFormat | None: - """ - Combine a zarr format selected by a URL pipeline segment (`zarr2:` / - `zarr3:`) with the caller's `zarr_format` argument. Explicitly - conflicting selections raise. - """ - if store_path.zarr_format is None: - return zarr_format - if zarr_format is not None and zarr_format != store_path.zarr_format: - raise ValueError( - f"zarr_format={zarr_format} conflicts with the 'zarr{store_path.zarr_format}:' " - "segment of the URL pipeline" - ) - return store_path.zarr_format - - def _warn_unimplemented_kwargs(kwargs: dict[str, Any]) -> None: """ Emit a "not yet implemented" warning for each provided keyword argument that is not None. @@ -407,7 +389,7 @@ async def open( else: mode = "a" store_path = await make_store_path(store, mode=mode, path=path, storage_options=storage_options) - zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + zarr_format = store_path.resolve_zarr_format(zarr_format) # TODO: the mode check below seems wrong! if "shape" not in kwargs and mode in {"a", "r", "r+", "w"}: @@ -518,7 +500,7 @@ async def save_array( mode = kwargs.pop("mode", "a") store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + zarr_format = store_path.resolve_zarr_format(zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() if np.isscalar(arr): @@ -570,7 +552,7 @@ async def save_group( """ store_path = await make_store_path(store, path=path, mode="w", storage_options=storage_options) - zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + zarr_format = store_path.resolve_zarr_format(zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() @@ -786,7 +768,7 @@ async def create_group( mode: Literal["a"] = "a" store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + zarr_format = store_path.resolve_zarr_format(zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() @@ -878,7 +860,7 @@ async def open_group( ) store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path) - zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + zarr_format = store_path.resolve_zarr_format(zarr_format) if attributes is None: attributes = {} @@ -1082,7 +1064,7 @@ async def create( if mode is None: mode = "a" store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + zarr_format = store_path.resolve_zarr_format(zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() @@ -1284,7 +1266,7 @@ async def open_array( mode = kwargs.pop("mode", None) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - zarr_format = _merge_pipeline_zarr_format(store_path, zarr_format) + zarr_format = store_path.resolve_zarr_format(zarr_format) if "write_empty_chunks" in kwargs: _warn_write_empty_chunks_kwarg() diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 6975f6d953..4532f4ad54 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -829,7 +829,7 @@ def create_array( serializer: SerializerLike = "auto", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, - zarr_format: ZarrFormat | None = 3, + zarr_format: ZarrFormat | None = None, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 5a8d6bf57e..2ad99791be 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -453,8 +453,9 @@ async def _create( Deprecated in favor of [`zarr.api.asynchronous.create_array`][]. """ - dtype_parsed = parse_dtype(dtype, zarr_format=zarr_format) store_path = await make_store_path(store) + zarr_format = store_path.resolve_zarr_format(zarr_format) or zarr_format + dtype_parsed = parse_dtype(dtype, zarr_format=zarr_format) shape = parse_shapelike(shape) @@ -813,6 +814,7 @@ async def example(): ``` """ store_path = await make_store_path(store) + zarr_format = store_path.resolve_zarr_format(zarr_format) metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format) # TODO: remove this cast when we have better type hints _metadata_dict = cast("ArrayMetadataJSON_V3", metadata_dict) @@ -2003,6 +2005,7 @@ def from_dict( def open( cls, store: StoreLike, + zarr_format: ZarrFormat | None = 3, ) -> Self: """Opens an existing Array from a store. @@ -2012,13 +2015,16 @@ def open( Store containing the Array. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. + zarr_format : {2, 3, None}, optional + The zarr format to expect. If None, the format selected by a URL pipeline + segment is used, otherwise the format is inferred from the store contents. Returns ------- Array Array opened from the store. """ - async_array = sync(AsyncArray.open(store)) + async_array = sync(AsyncArray.open(store, zarr_format=zarr_format)) return cls(async_array) @property @@ -4337,6 +4343,7 @@ async def from_array( mode: Literal["a"] = "a" config_parsed = parse_array_config(config) store_path = await make_store_path(store, path=name, mode=mode, storage_options=storage_options) + zarr_format = store_path.resolve_zarr_format(zarr_format) ( chunks, @@ -4663,7 +4670,7 @@ async def create_array( serializer: SerializerLike = "auto", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, - zarr_format: ZarrFormat | None = 3, + zarr_format: ZarrFormat | None = None, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, @@ -4826,6 +4833,7 @@ async def create_array( store_path = await make_store_path( store, path=name, mode=mode, storage_options=storage_options ) + zarr_format = store_path.resolve_zarr_format(zarr_format) return await init_array( store_path=store_path, shape=shape_parsed, diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index d734e6b7cd..e46927d678 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -42,6 +42,7 @@ NodeType, ShapeLike, ZarrFormat, + _default_zarr_format, parse_shapelike, ) from zarr.core.config import config @@ -474,9 +475,12 @@ async def from_store( *, attributes: dict[str, Any] | None = None, overwrite: bool = False, - zarr_format: ZarrFormat = 3, + zarr_format: ZarrFormat | None = None, ) -> AsyncGroup: store_path = await make_store_path(store) + zarr_format = store_path.resolve_zarr_format(zarr_format) + if zarr_format is None: + zarr_format = _default_zarr_format() if overwrite: if store_path.store.supports_deletes: @@ -525,6 +529,7 @@ async def open( to load consolidated metadata from a non-default key. """ store_path = await make_store_path(store) + zarr_format = store_path.resolve_zarr_format(zarr_format) if not store_path.store.supports_consolidated_metadata: # Fail if consolidated metadata was requested but the Store doesn't support it if use_consolidated: @@ -1816,7 +1821,7 @@ def from_store( store: StoreLike, *, attributes: dict[str, Any] | None = None, - zarr_format: ZarrFormat = 3, + zarr_format: ZarrFormat | None = None, overwrite: bool = False, ) -> Group: """Instantiate a group from an initialized store. @@ -1829,8 +1834,9 @@ def from_store( for a description of all valid StoreLike values. attributes : dict, optional A dictionary of JSON-serializable values with user-defined attributes. - zarr_format : {2, 3}, optional - Zarr storage format version. + zarr_format : {2, 3, None}, optional + Zarr storage format version. If None, the format selected by a URL + pipeline segment is used, falling back to the configured default. overwrite : bool, optional If True, do not raise an error if the group already exists. diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index f6cdfee8d7..fa470e1e3b 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -74,6 +74,36 @@ def __init__(self, store: Store, path: str = "") -> None: def read_only(self) -> bool: return self.store.read_only + def resolve_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat | None: + """ + Combine the zarr format selected by a URL pipeline segment (`zarr2:` / + `zarr3:`) with a caller-supplied `zarr_format`. + + Parameters + ---------- + zarr_format : ZarrFormat | None + The format requested by the caller, or None when unspecified. + + Returns + ------- + ZarrFormat | None + The pipeline's format when the caller did not specify one, otherwise the + caller's format. None when neither is set. + + Raises + ------ + ValueError + If the caller's format differs from the one selected by the pipeline. + """ + if self.zarr_format is None: + return zarr_format + if zarr_format is not None and zarr_format != self.zarr_format: + raise ValueError( + f"zarr_format={zarr_format} conflicts with the 'zarr{self.zarr_format}:' " + "segment of the URL pipeline; pass zarr_format=None to use the pipeline's format" + ) + return self.zarr_format + @classmethod async def _create_open_instance(cls, store: Store, path: str) -> Self: """Helper to create and return a StorePath instance.""" @@ -492,6 +522,9 @@ async def make_store_path( If the StoreLike object is not one of the supported types, or if storage_options is provided but not used. ValueError If path is provided for a store that does not support it. + URLPipelineError + If `store_like` is a [URL pipeline][user-guide-url-pipelines] string that + cannot be resolved. See Also -------- diff --git a/tests/test_api.py b/tests/test_api.py index c4e561c180..294f245e9b 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -195,7 +195,7 @@ async def test_open_like_creates_array_by_default( assert np.all(Array(new_arr)[:] == ref_arr.fill_value) -async def test_open_like_default_mode_rejects_read_only_store( +async def test_open_like_default_mode_on_read_only_store_raises_not_found( zarr_format: ZarrFormat, ) -> None: # mode "a" (open-or-create) on a read-only store serves the "open" half: diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index 75582cdc75..66b9246a48 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -2,8 +2,12 @@ import dataclasses import sys +import threading +import warnings +from importlib.metadata import EntryPoint from typing import TYPE_CHECKING, ClassVar +import numpy as np import pytest import zarr @@ -22,13 +26,15 @@ register_url_adapter, ) from zarr.storage import ManagedMemoryStore, MemoryStore, WrapperStore -from zarr.storage._common import make_store, make_store_path +from zarr.storage._common import _has_fsspec, make_store, make_store_path from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline from zarr.storage._utils import _join_paths if TYPE_CHECKING: from pathlib import Path + from zarr.core.common import AccessModeLiteral + pytestmark = pytest.mark.usefixtures("clean_url_adapter_registry") @@ -159,6 +165,152 @@ def test_loading_one_scheme_leaves_others_pending(self) -> None: ) +def _fake_entry_point( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, scheme: str, module: str, source: str +) -> EntryPoint: + """ + Write `source` to an importable module and return a `zarr.url_adapters` + entry point for `scheme` pointing at its `Adapter` attribute. + """ + (tmp_path / f"{module}.py").write_text(source) + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.delitem(sys.modules, module, raising=False) + return EntryPoint(name=scheme, value=f"{module}:Adapter", group="zarr.url_adapters") + + +_ADAPTER_SOURCE = """ +from zarr.abc.url_pipeline import AdapterResolution, URLPipelineAdapter +from zarr.storage import MemoryStore +{prelude} + +class Adapter(URLPipelineAdapter): + @classmethod + async def open_pipeline_segment(cls, segment, context): + return AdapterResolution(store=await MemoryStore.open(read_only=context.read_only)) +""" + + +class TestEntryPointLoading: + """Lazy loading of `zarr.url_adapters` entry points.""" + + def test_adapter_import_may_resolve_other_adapters( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # an adapter module that itself resolves a URL adapter at import + # time must not deadlock on the registry lock + pending = zarr.registry._url_adapter_registry.lazy_load_list + pending.append( + _fake_entry_point( + tmp_path, + monkeypatch, + scheme="reentrant", + module="reentrant_adapter", + source=_ADAPTER_SOURCE.format( + prelude=( + "import zarr.registry\n" + "try:\n" + " zarr.registry.get_url_adapter('no-such-scheme-at-import')\n" + "except Exception:\n" + " pass\n" + ) + ), + ) + ) + result: list[object] = [] + thread = threading.Thread(target=lambda: result.append(get_url_adapter("reentrant"))) + thread.start() + thread.join(timeout=10) + assert not thread.is_alive(), "get_url_adapter deadlocked on a re-entrant lookup" + assert getattr(result[0], "__name__", None) == "Adapter" + + def test_concurrent_loads_keep_other_schemes_pending( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # two threads resolving different schemes at once, one with a slow + # import, must not drop a third, unrelated pending entry point + pending = zarr.registry._url_adapter_registry.lazy_load_list + for scheme, prelude in [ + ("slow", "import time\ntime.sleep(0.3)\n"), + ("quick", ""), + ("untouched", ""), + ]: + pending.append( + _fake_entry_point( + tmp_path, + monkeypatch, + scheme=scheme, + module=f"{scheme}_adapter", + source=_ADAPTER_SOURCE.format(prelude=prelude), + ) + ) + threads = [ + threading.Thread(target=get_url_adapter, args=(scheme,)) for scheme in ("slow", "quick") + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + assert not any(thread.is_alive() for thread in threads) + assert "untouched" in list_url_adapter_schemes() + assert [e.name for e in pending] == ["untouched"] + assert get_url_adapter("untouched").__name__ == "Adapter" + + def test_import_failure_is_wrapped_and_not_permanent(self) -> None: + pending = zarr.registry._url_adapter_registry.lazy_load_list + pending.append( + EntryPoint(name="broken", value="no_such_module_xyz:Adapter", group="zarr.url_adapters") + ) + with pytest.raises(URLPipelineError, match="could not be loaded"): + get_url_adapter("broken") + # still advertised, so a transient failure can be retried + assert "broken" in list_url_adapter_schemes() + with pytest.raises(URLPipelineError, match="could not be loaded"): + get_url_adapter("broken") + + def test_registered_adapter_shadows_entry_point_with_warning( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # a builtin registered under a scheme wins over a same-named entry + # point, which is discarded loudly rather than left pending forever + pending = zarr.registry._url_adapter_registry.lazy_load_list + pending.append( + _fake_entry_point( + tmp_path, + monkeypatch, + scheme="shadowed", + module="shadowed_adapter", + source=_ADAPTER_SOURCE.format(prelude="raise RuntimeError('must not import')\n"), + ) + ) + register_url_adapter("shadowed", NativeAdapter) + with pytest.warns(ZarrUserWarning, match="already registered"): + assert get_url_adapter("shadowed") is NativeAdapter + assert not any(e.name == "shadowed" for e in pending) + # a second lookup is silent + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert get_url_adapter("shadowed") is NativeAdapter + + def test_duplicate_entry_points_warn_and_first_wins( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + pending = zarr.registry._url_adapter_registry.lazy_load_list + for module in ("dup_first", "dup_second"): + pending.append( + _fake_entry_point( + tmp_path, + monkeypatch, + scheme="dup", + module=module, + source=_ADAPTER_SOURCE.format(prelude=f"ORIGIN = {module!r}\n"), + ) + ) + with pytest.warns(ZarrUserWarning, match="multiple 'zarr.url_adapters' entry points"): + cls = get_url_adapter("dup") + assert getattr(sys.modules[cls.__module__], "ORIGIN", None) == "dup_first" + assert not any(e.name == "dup" for e in pending) + + class TestIsURLPipeline: def test_pipe_routes(self) -> None: assert is_url_pipeline("memory://x|demo:") @@ -203,6 +355,8 @@ async def test_wrapper_adapter_chain(self, tmp_path: Path) -> None: assert isinstance(result.store, TracingStore) assert result.path == "inner/path" assert result.store.context.preceding_url == str(tmp_path) + assert result.store.segment.scheme == "wrap" + assert result.store.segment.body == "inner/path" async def test_nested_wrappers_preserve_residual_paths(self, tmp_path: Path) -> None: # each wrapper joins the preceding residual path with its own, so @@ -274,6 +428,30 @@ async def open_pipeline_segment( with pytest.raises(URLPipelineError, match="does not support read-only conversion"): await resolve_pipeline("memory://base|stubborn:", mode="r") + async def test_read_only_enforcement_failure_closes_store(self) -> None: + # the adapter's store must not leak when it cannot be made read-only + closed: list[bool] = [] + + class StubbornStore(MemoryStore): + def with_read_only(self, read_only: bool = False) -> MemoryStore: + raise NotImplementedError + + def close(self) -> None: + closed.append(True) + super().close() + + class StubbornAdapter(URLPipelineAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + return AdapterResolution(store=await StubbornStore.open(read_only=False)) + + register_url_adapter("stubborn", StubbornAdapter) + with pytest.raises(URLPipelineError): + await resolve_pipeline("memory://base|stubborn:", mode="r") + assert closed == [True] + async def test_resolve_preceding_mode_override(self, tmp_path: Path) -> None: # a wrapper that only reads the preceding resource can open it # read-only regardless of the caller's mode @@ -290,6 +468,34 @@ async def open_pipeline_segment( result = await resolve_pipeline(f"{tmp_path}|rowrap:x", mode="w") assert result.path == "x" + async def test_wrapper_on_local_file_root_read_only(self, tmp_path: Path) -> None: + # a wrapper that opens the preceding local *file* read-only gets a + # store rooted at that file (the mechanism a read-only zip: adapter + # can build on) + target = tmp_path / "data.bin" + target.write_bytes(b"payload") + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{target}|wrap:", mode="r") + assert isinstance(result.store, TracingStore) + assert result.store.read_only + + @pytest.mark.parametrize("mode", [None, "a", "w", "r+", "w-"]) + @pytest.mark.xfail( + strict=True, + raises=FileExistsError, + reason=( + "known limitation: there is no file-resource primitive yet, so resolving a " + "local file root in a writable mode hits LocalStore's create-on-open mkdir" + ), + ) + async def test_wrapper_on_local_file_root_writable_modes( + self, tmp_path: Path, mode: AccessModeLiteral | None + ) -> None: + target = tmp_path / "data.bin" + target.write_bytes(b"payload") + register_url_adapter("wrap", WrapperAdapter) + await resolve_pipeline(f"{target}|wrap:", mode=mode) + async def test_resolve_preceding_storage_options_override(self, tmp_path: Path) -> None: # an adapter that consumed its namespaced keys strips them before # the root store ever sees them @@ -453,6 +659,26 @@ async def test_file_root_rejects_query(self, tmp_path: Path) -> None: with pytest.raises(URLPipelineError, match="do not accept a query"): await resolve_pipeline(f"file:{tmp_path}?v=1|wrap:") + @pytest.mark.skipif(not _has_fsspec, reason="plain memory:// routes to fsspec only with fsspec") + async def test_memory_root_is_distinct_from_fsspec_memory_url(self) -> None: + # documented divergence: a plain memory:// URL is fsspec's in-memory + # filesystem when fsspec is installed, while a pipeline's memory: root + # is zarr's ManagedMemoryStore; the two do not share data + register_url_adapter("wrap", WrapperAdapter) + plain = await make_store("memory://distinct-store") + piped = await make_store("memory://distinct-store|wrap:") + assert not isinstance(plain, ManagedMemoryStore) + assert isinstance(piped, TracingStore) + assert isinstance(piped._store, ManagedMemoryStore) + + async def test_file_root_does_not_percent_decode(self, tmp_path: Path) -> None: + # documented divergence from RFC 8089: consistent with LocalStore, no + # percent-escape is decoded, so %20 names a literal directory + register_url_adapter("wrap", WrapperAdapter) + await resolve_pipeline(f"file:{tmp_path.as_posix()}/a%20b|wrap:") + assert (tmp_path / "a%20b").is_dir() + assert not (tmp_path / "a b").exists() + class TestMakeStoreIntegration: async def test_make_store_path_combines_paths(self, tmp_path: Path) -> None: @@ -574,3 +800,72 @@ async def test_matching_explicit_format_ok(self, tmp_path: Path) -> None: register_url_adapter("fmt2", FormatAdapter) group = zarr.open_group(f"{tmp_path}|fmt2:", mode="w", zarr_format=2) assert group.metadata.zarr_format == 2 + + +class TestZarrFormatMergeInCore: + """ + Every make_store_path caller honors the pipeline-selected zarr format, not + only the top-level zarr.api functions. + """ + + @pytest.fixture(autouse=True) + def _fmt2(self) -> None: + register_url_adapter("fmt2", FormatAdapter) + + async def test_create_array(self, tmp_path: Path) -> None: + arr = zarr.create_array(f"{tmp_path}|fmt2:", name="x", shape=(2,), dtype="i4") + assert arr.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + zarr.create_array(f"{tmp_path}|fmt2:", name="y", shape=(2,), dtype="i4", zarr_format=3) + + async def test_create_array_from_data(self, tmp_path: Path) -> None: + # the data path of create_array goes through from_array + arr = zarr.create_array(f"{tmp_path}|fmt2:", name="x", data=np.arange(3)) + assert arr.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + zarr.create_array(f"{tmp_path}|fmt2:", name="y", data=np.arange(3), zarr_format=3) + + async def test_from_array(self, tmp_path: Path) -> None: + arr = zarr.from_array(f"{tmp_path}|fmt2:", name="x", data=np.arange(3)) + assert arr.metadata.zarr_format == 2 + + async def test_group_from_store(self, tmp_path: Path) -> None: + group = zarr.Group.from_store(f"{tmp_path}|fmt2:") + assert group.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + zarr.Group.from_store(f"{tmp_path}/other|fmt2:", zarr_format=3) + + async def test_group_from_store_default_without_pipeline(self, tmp_path: Path) -> None: + # the default is still the configured format (3) for plain stores + group = zarr.Group.from_store(tmp_path) + assert group.metadata.zarr_format == 3 + + async def test_group_open(self, tmp_path: Path) -> None: + zarr.Group.from_store(f"{tmp_path}|fmt2:") + # explicit None defers to the pipeline; the default (3) conflicts + group = zarr.Group.open(f"{tmp_path}|fmt2:", zarr_format=None) + assert group.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + zarr.Group.open(f"{tmp_path}|fmt2:") + + async def test_array_open(self, tmp_path: Path) -> None: + zarr.create_array(f"{tmp_path}|fmt2:", name="x", shape=(2,), dtype="i4") + arr = zarr.Array.open(f"{tmp_path}/x|fmt2:", zarr_format=None) + assert arr.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + zarr.Array.open(f"{tmp_path}/x|fmt2:") + + async def test_open_like(self, tmp_path: Path) -> None: + # open_like inherits v2 filters/compressor from the reference; the + # pipeline supplies the format, which open_array then creates with + ref = zarr.create_array({}, shape=(2,), dtype="i4", zarr_format=2) + arr = zarr.open_like(ref, store=f"{tmp_path}|fmt2:", path="x") + assert arr.metadata.zarr_format == 2 + + async def test_deprecated_async_array_create(self, tmp_path: Path) -> None: + from zarr.core.array import AsyncArray + + arr = await AsyncArray._create(f"{tmp_path}|fmt2:", shape=(2,), dtype="i4", zarr_format=2) + assert arr.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + await AsyncArray._create(f"{tmp_path}/o|fmt2:", shape=(2,), dtype="i4", zarr_format=3) From 60f14a0989b5d44ed5ed3e129a54df7996b373ab Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 17 Sep 2026 14:17:54 -0700 Subject: [PATCH 06/20] docs: fix wrapper example path handling; document review follow-ups - The extending guide's wrapper example dropped the preceding residual path while its comment claimed the opposite; it now joins the two and a tested snippet shows root|a|b resolving to a/b. - Changelog and storage guide state the memory:// routing split with fsspec, the undecoded percent-escapes in file: roots, the local-file-root limitation, the entry-point collision warnings, and the zarr_format default changes. Co-Authored-By: Claude Fable 5.1 --- changes/4192.feature.md | 24 ++++++++++++++++++++++-- docs/user-guide/extending.md | 22 +++++++++++++++++++--- docs/user-guide/storage.md | 6 ++++++ 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/changes/4192.feature.md b/changes/4192.feature.md index 30f11a559d..3ea14b30d6 100644 --- a/changes/4192.feature.md +++ b/changes/4192.feature.md @@ -17,8 +17,28 @@ Behavior notes: registered adapters cannot intercept zarr's native `file:`/`memory:` routing, and fsspec chained URLs (`zip::s3://...`) keep flowing to fsspec. - Inside a pipeline, `memory:` and `file:` roots follow the URL pipeline - spec's semantics (spelling equivalences; `file:` must be absolute, with at - most a `localhost` authority). + spec's spelling rules (`memory:` ≡ `memory:/` ≡ `memory://`; `file:` must + carry an absolute path with at most a `localhost` authority). Percent-escapes + are *not* decoded in `file:` roots, matching `LocalStore`; this is a + documented divergence from RFC 8089. When fsspec is installed, a plain + `memory://name` URL is still routed to fsspec's in-memory filesystem, whereas + a pipeline's `memory:` root is zarr's `ManagedMemoryStore`; the two do not + share data. +- Wrapper adapters can open a local *file* root only in mode `"r"` (via + `PipelineContext.resolve_preceding(mode="r")`); other modes create a + directory at the root and fail. Writing through a container file is not yet + supported. +- A `zarr.url_adapters` entry point whose scheme is already registered (e.g. by + a builtin adapter) is ignored with a `ZarrUserWarning`, as are duplicate + entry-point names; an entry point that fails to import raises + `URLPipelineError` naming it, and stays discoverable for a retry. +- `zarr.create_array` and `Group.from_store` now default `zarr_format` to + `None`, resolving to the configured default (3) as before unless a URL + pipeline selects a format; `Array.open` gains a `zarr_format` parameter + mirroring `Group.open`. `Group.open` and `Array.open` keep their default of + 3, so pass `zarr_format=None` to defer to a pipeline's format. A format + selected by a pipeline that conflicts with an explicit `zarr_format` raises + `ValueError` in every `open*`/`create*`/`save*` entry point. - Mode `"a"` (open-or-create, the `zarr.open` default) on a *read-only* store now serves the "open" half instead of raising upfront, for all stores; unambiguous write modes (`"w"`, `"w-"`, `"r+"`) still raise. diff --git a/docs/user-guide/extending.md b/docs/user-guide/extending.md index db5769d348..d5fe5726ca 100644 --- a/docs/user-guide/extending.md +++ b/docs/user-guide/extending.md @@ -110,11 +110,27 @@ class MyAdapter(URLPipelineAdapter): async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext ) -> AdapterResolution: - # A *wrapper* adapter opens the resource to its left and wraps it. Carry the - # preceding residual path forward and keep unchanged fields via dataclasses.replace. + # A *wrapper* adapter opens the resource to its left and wraps it. Join the + # preceding residual path with this segment's own path, and keep every other + # field (e.g. zarr_format) via dataclasses.replace. preceding = await context.resolve_preceding() store = WrapperStore(preceding.store) - return dataclasses.replace(preceding, store=store, path=segment.body) + path = "/".join(part.strip("/") for part in (preceding.path, segment.body) if part) + return dataclasses.replace(preceding, store=store, path=path) +``` + +Wrapper adapters compose: in `memory://x|mypackage.myscheme:a|mypackage.myscheme:b` the +inner adapter sees the outer one's residual path `a` and the pipeline resolves to `a/b`. + +```python test="true" session="url-adapter" +import asyncio + +from zarr.registry import register_url_adapter +from zarr.storage import resolve_pipeline + +register_url_adapter("mypackage.myscheme", MyAdapter) +resolution = asyncio.run(resolve_pipeline("memory://x|mypackage.myscheme:a|mypackage.myscheme:b")) +assert resolution.path == "a/b" ``` Register the class under the `zarr.url_adapters` entry-point group, using the URL diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index d572c5bc1d..010abc08a4 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -131,6 +131,12 @@ Registered adapters cannot intercept zarr's native `file:` and `memory:` root schemes, and fsspec's chained-URL syntax (`zip::s3://...`) keeps flowing to fsspec. +Inside a pipeline, a `memory:` root is zarr's managed in-memory store (`memory:`, +`memory:/` and `memory://` are equivalent, and `memory:name` selects a named store). +When fsspec is installed, a plain `memory://name` URL *without* a `|` is still routed to +fsspec's in-memory filesystem, which is a different store. A `file:` root must carry an +absolute path; percent-escapes are not decoded, matching the [local store](#local-store). + ## Explicit Store Creation In some cases, it may be helpful to create a store instance directly. Zarr-Python offers From 03ab3b05612591aad8248ddc4de3df7a6c677185 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 3 Oct 2026 13:54:40 -0700 Subject: [PATCH 07/20] test: skip fsspec memory:// divergence test when fsspec lacks AsyncFileSystemWrapper fsspec < 2024.12.0 (the min_deps job) cannot open a plain memory:// URL at all, so the routing comparison only applies with fsspec>=2024.12.0. Co-Authored-By: Claude Opus 5.5 --- tests/test_url_pipeline/test_resolver.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index 66b9246a48..f9941b5101 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -4,11 +4,12 @@ import sys import threading import warnings -from importlib.metadata import EntryPoint +from importlib.metadata import EntryPoint, version from typing import TYPE_CHECKING, ClassVar import numpy as np import pytest +from packaging.version import parse as parse_version import zarr import zarr.registry @@ -30,6 +31,12 @@ from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline from zarr.storage._utils import _join_paths +# fsspec < 2024.12.0 has no AsyncFileSystemWrapper, so a plain memory:// URL +# (a sync filesystem) cannot be opened at all there +_has_async_fsspec_wrapper = _has_fsspec and parse_version(version("fsspec")) >= parse_version( + "2024.12.0" +) + if TYPE_CHECKING: from pathlib import Path @@ -659,7 +666,10 @@ async def test_file_root_rejects_query(self, tmp_path: Path) -> None: with pytest.raises(URLPipelineError, match="do not accept a query"): await resolve_pipeline(f"file:{tmp_path}?v=1|wrap:") - @pytest.mark.skipif(not _has_fsspec, reason="plain memory:// routes to fsspec only with fsspec") + @pytest.mark.skipif( + not _has_async_fsspec_wrapper, + reason="plain memory:// routes to fsspec only with fsspec>=2024.12.0", + ) async def test_memory_root_is_distinct_from_fsspec_memory_url(self) -> None: # documented divergence: a plain memory:// URL is fsspec's in-memory # filesystem when fsspec is installed, while a pipeline's memory: root From 103c87d87bd5afb34c2158f6f73ae9cb3901b60b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 6 Oct 2026 10:40:54 +0200 Subject: [PATCH 08/20] refactor(url-pipeline): resolve pipelines from a URLPipeline object, never from plain strings Strings passed as a StoreLike keep their pre-existing meaning: a local path containing `|` still names a directory, whatever adapters are installed. A pipeline is opened either through the new `zarr.open_url`, or by passing a `zarr.storage.URLPipeline` (built with `from_url`) to any function that accepts a StoreLike. - `URLPipeline` holds every parsed segment; a trailing `zarr2:`/`zarr3:` format segment is read from it (`zarr_format`, `path`, `store_segments`) instead of through pseudo-adapters, so `AdapterResolution.zarr_format` is gone. - `StorePath` is back to its state on main: no `zarr_format` attribute, and mode "a" on a read-only store raises again. The open-or-create downgrade applies only to pipelines that resolve read-only. - The format selected by a pipeline is merged with the caller's `zarr_format` by `resolve_zarr_format(store, zarr_format)` at each call site, before the store is resolved. `create_array` and `Group.from_store` default to 3 again; `Array.open` keeps its new `zarr_format` parameter. - `is_url_pipeline` and the `pathlib.Path` escape hatch are removed; `resolve_pipeline` accepts a string or a `URLPipeline`, and a single URL is a trivial pipeline. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- changes/4192.feature.md | 69 ++-- docs/api/zarr/functions/open_url.md | 5 + docs/user-guide/extending.md | 5 +- docs/user-guide/storage.md | 64 ++-- mkdocs.yml | 1 + src/zarr/__init__.py | 2 + src/zarr/abc/url_pipeline.py | 16 +- src/zarr/api/asynchronous.py | 71 ++++- src/zarr/api/synchronous.py | 57 +++- src/zarr/core/array.py | 12 +- src/zarr/core/group.py | 14 +- src/zarr/storage/__init__.py | 3 +- src/zarr/storage/_common.py | 116 +++---- src/zarr/storage/_url_pipeline.py | 206 +++++++++--- tests/test_api.py | 7 +- .../test_url_pipeline/test_pipeline_object.py | 115 +++++++ tests/test_url_pipeline/test_resolver.py | 299 ++++++++++-------- 17 files changed, 730 insertions(+), 332 deletions(-) create mode 100644 docs/api/zarr/functions/open_url.md create mode 100644 tests/test_url_pipeline/test_pipeline_object.py diff --git a/changes/4192.feature.md b/changes/4192.feature.md index 3ea14b30d6..91a4748417 100644 --- a/changes/4192.feature.md +++ b/changes/4192.feature.md @@ -1,46 +1,35 @@ Added core support for URL pipelines (https://github.com/jbms/url-pipeline): `|`-chained URLs that address zarr data through nested storage layers, e.g. -`s3://bucket/data.zip|zip:|zarr3:`. This includes the parser -(`zarr.storage.parse_pipeline` / `zarr.storage.resolve_pipeline`), the -single-method `zarr.abc.url_pipeline.URLPipelineAdapter` interface, and the -`zarr.url_adapters` entry-point group through which third-party packages -(e.g. Icechunk) register adapters for their own schemes. Adapters for a scheme -are loaded lazily and individually. Builtin adapters (`zip:`, -`zarr2:`/`zarr3:`) will follow separately. +`s3://bucket/data.zip|zip:|zarr3:`. A pipeline is opened with the new `zarr.open_url` +function, or by passing a parsed `zarr.storage.URLPipeline` (built with +`URLPipeline.from_url`) to any function that accepts a `StoreLike`. Adapters for the +sub-URLs after the first are provided by third-party packages (e.g. Icechunk) through +the `zarr.url_adapters` entry-point group and the single-method +`zarr.abc.url_pipeline.URLPipelineAdapter` interface; they are loaded lazily and +individually. A builtin `zip:` adapter will follow separately. Behavior notes: -- The `|` character is now reserved as the pipeline delimiter in every string - store specification, and no percent-escape is decoded; pass a `pathlib.Path` - to address a local file whose name contains `|`. URLs without a `|` (and - without a registered root adapter scheme) are handled exactly as before — - registered adapters cannot intercept zarr's native `file:`/`memory:` - routing, and fsspec chained URLs (`zip::s3://...`) keep flowing to fsspec. -- Inside a pipeline, `memory:` and `file:` roots follow the URL pipeline - spec's spelling rules (`memory:` ≡ `memory:/` ≡ `memory://`; `file:` must - carry an absolute path with at most a `localhost` authority). Percent-escapes - are *not* decoded in `file:` roots, matching `LocalStore`; this is a - documented divergence from RFC 8089. When fsspec is installed, a plain - `memory://name` URL is still routed to fsspec's in-memory filesystem, whereas - a pipeline's `memory:` root is zarr's `ManagedMemoryStore`; the two do not - share data. +- Plain strings are never interpreted as pipelines: `zarr.open("data|x.zarr")` still + addresses a local directory of that name, whatever adapters are installed. Only + `zarr.open_url` and `URLPipeline.from_url` read a string as a pipeline. +- A trailing `zarr2:`/`zarr3:` segment selects the zarr format of the open or create + call, and its body is the path of the node to open. A `zarr_format` argument that + conflicts with it raises `ValueError`; functions whose `zarr_format` defaults to 3 + (`zarr.create_array`, `Group.from_store`, `Group.open`, `Array.open`) need an explicit + `zarr_format=None` to defer to the pipeline. `Array.open` gains a `zarr_format` + parameter mirroring `Group.open`. +- With the default mode `"a"`, a pipeline that resolves to a read-only resource is + opened for reading instead of raising; other stores behave as before. +- Inside a pipeline, `memory:` and `file:` roots follow the URL pipeline spec's spelling + rules (`memory:` ≡ `memory:/` ≡ `memory://`; `file:` must carry an absolute path with + at most a `localhost` authority). Percent-escapes are not decoded, matching + `LocalStore`. A pipeline's `memory:` root is zarr's `ManagedMemoryStore`, which does + not share data with the fsspec in-memory filesystem a plain `memory://name` string + resolves to. - Wrapper adapters can open a local *file* root only in mode `"r"` (via - `PipelineContext.resolve_preceding(mode="r")`); other modes create a - directory at the root and fail. Writing through a container file is not yet - supported. -- A `zarr.url_adapters` entry point whose scheme is already registered (e.g. by - a builtin adapter) is ignored with a `ZarrUserWarning`, as are duplicate - entry-point names; an entry point that fails to import raises - `URLPipelineError` naming it, and stays discoverable for a retry. -- `zarr.create_array` and `Group.from_store` now default `zarr_format` to - `None`, resolving to the configured default (3) as before unless a URL - pipeline selects a format; `Array.open` gains a `zarr_format` parameter - mirroring `Group.open`. `Group.open` and `Array.open` keep their default of - 3, so pass `zarr_format=None` to defer to a pipeline's format. A format - selected by a pipeline that conflicts with an explicit `zarr_format` raises - `ValueError` in every `open*`/`create*`/`save*` entry point. -- Mode `"a"` (open-or-create, the `zarr.open` default) on a *read-only* store - now serves the "open" half instead of raising upfront, for all stores; - unambiguous write modes (`"w"`, `"w-"`, `"r+"`) still raise. -- For root-adapter URLs (e.g. `gh://org/repo`), `storage_options` are handed - to the adapter and are not validated as used by `make_store`. + `PipelineContext.resolve_preceding(mode="r")`); writing through a container file is + not yet supported. +- A `zarr.url_adapters` entry point whose scheme is already registered is ignored with a + `ZarrUserWarning`, as are duplicate entry-point names; an entry point that fails to + import raises `URLPipelineError` naming it, and stays discoverable for a retry. diff --git a/docs/api/zarr/functions/open_url.md b/docs/api/zarr/functions/open_url.md new file mode 100644 index 0000000000..84774a1710 --- /dev/null +++ b/docs/api/zarr/functions/open_url.md @@ -0,0 +1,5 @@ +--- +title: zarr.open_url +--- + +::: zarr.open_url diff --git a/docs/user-guide/extending.md b/docs/user-guide/extending.md index d5fe5726ca..5138ba3a10 100644 --- a/docs/user-guide/extending.md +++ b/docs/user-guide/extending.md @@ -112,7 +112,7 @@ class MyAdapter(URLPipelineAdapter): ) -> AdapterResolution: # A *wrapper* adapter opens the resource to its left and wraps it. Join the # preceding residual path with this segment's own path, and keep every other - # field (e.g. zarr_format) via dataclasses.replace. + # field via dataclasses.replace. preceding = await context.resolve_preceding() store = WrapperStore(preceding.store) path = "/".join(part.strip("/") for part in (preceding.path, segment.body) if part) @@ -144,7 +144,8 @@ scheme as the entry-point name. Nonstandard schemes should be vendor-prefixed Adapters are loaded lazily, only when a pipeline naming their scheme is resolved. An adapter may also be used as a *root* scheme (e.g. `mypackage.myscheme://org/repo`), in -which case `context.preceding` is empty. The returned store must honor +which case `context.preceding` is empty; such a URL reaches the adapter only through +`zarr.open_url` or a `URLPipeline`, never as a plain string store specification. The returned store must honor `context.read_only`, and `open_pipeline_segment` runs on zarr's internal event loop, so it must not block or call zarr's synchronous API. See [`zarr.abc.url_pipeline`][] for the full contract. diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index 010abc08a4..4d4571c7e3 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -100,8 +100,9 @@ print(group) group = zarr.open_group(UPath('s3://noaa-nwm-retro-v2-zarr-pds', anon=True), mode='r') ``` -- a [URL pipeline](#user-guide-url-pipelines) string containing `|`, such as - `s3://bucket/data.zip|zip:`, which is resolved through registered adapters. +- a [`URLPipeline`][zarr.storage.URLPipeline], the parsed form of a + [URL pipeline](#user-guide-url-pipelines) such as `s3://bucket/data.zip|zip:`, which + is resolved through registered adapters. Strings are never interpreted as pipelines. - a [`Store`][zarr.abc.store.Store] or [`StorePath`][zarr.storage.StorePath] - see explicit store creation below. @@ -114,28 +115,49 @@ sub-URL locates a resource with a conventional URL; each subsequent sub-URL name *adapter* that reinterprets everything to its left (e.g. `s3://bucket/data.zip|zip:|zarr3:`). Adapters are provided by packages through the `zarr.url_adapters` entry-point group — see -[`zarr.abc.url_pipeline`][zarr.abc.url_pipeline] for the adapter interface. Builtin -adapters (`zip:`, `zarr2:`/`zarr3:`) are under development and will expand this -section. URLs without a `|` (and without a registered root scheme) are handled -exactly as before. - -`storage_options` passed to `zarr.open` apply to the *root* sub-URL (e.g. fsspec -options for `s3://...`); adapters may consume adapter-specific, namespaced keys. -Non-dict forms of `storage_options` are reserved for future per-segment -configuration. - -The `|` character is reserved as the pipeline delimiter in every string store -specification, and no percent-escape is decoded: to address a local file whose -*name* contains `|` (or `#`), pass a `pathlib.Path` instead of a string. -Registered adapters cannot intercept zarr's native `file:` and `memory:` root -schemes, and fsspec's chained-URL syntax (`zip::s3://...`) keeps flowing to -fsspec. +[`zarr.abc.url_pipeline`][zarr.abc.url_pipeline] for the adapter interface. A builtin +`zip:` adapter is under development and will expand this section. + +A pipeline is opened either with [`zarr.open_url`][], which is `zarr.open` with its +string argument read as a pipeline, or by passing a parsed +[`URLPipeline`][zarr.storage.URLPipeline] to any function that accepts a `StoreLike`: + +```python exec="true" session="storage" source="above" result="ansi" +from zarr.storage import URLPipeline + +pipeline = URLPipeline.from_url("memory://pipeline-demo|zarr2:") +group = zarr.open_group(pipeline, mode="w") +print(group.metadata.zarr_format) +# the same thing, as a one-liner +print(zarr.open_url("memory://pipeline-demo|zarr2:").metadata.zarr_format) +``` + +Only these two entry points read a string as a pipeline. Plain string store +specifications keep their existing meaning everywhere else, so a local path that +happens to contain `|` still names a directory, whatever adapters are installed. + +A trailing `zarr2:` or `zarr3:` segment is a *format segment*. It is not an adapter: +zarr-python reads it from the pipeline itself and uses it as the `zarr_format` of the +open or create call, and its body (`zarr3:path/to/node`) is the path of the node to +open. A `zarr_format` argument that conflicts with the format segment raises +`ValueError`; functions whose `zarr_format` defaults to 3 (such as `zarr.create_array` +and `Group.open`) need an explicit `zarr_format=None` to defer to the pipeline. + +`storage_options` apply to the *root* sub-URL (e.g. fsspec options for `s3://...`); +adapters may consume adapter-specific, namespaced keys. Non-dict forms of +`storage_options` are reserved for future per-segment configuration. No percent-escape +is decoded anywhere in a pipeline. Registered adapters cannot intercept zarr's native +`file:` and `memory:` root schemes. With the default mode `"a"`, a pipeline that +resolves to a read-only resource is opened for reading instead of raising. Inside a pipeline, a `memory:` root is zarr's managed in-memory store (`memory:`, `memory:/` and `memory://` are equivalent, and `memory:name` selects a named store). -When fsspec is installed, a plain `memory://name` URL *without* a `|` is still routed to -fsspec's in-memory filesystem, which is a different store. A `file:` root must carry an -absolute path; percent-escapes are not decoded, matching the [local store](#local-store). +When fsspec is installed, a plain `memory://name` string store specification is still +routed to fsspec's in-memory filesystem, which is a different store. A `file:` root +must carry an absolute path; percent-escapes are not decoded, matching the +[local store](#local-store). As a zarr-python extension, the root may also be a bare +local path (`data/example.zip|zip:`); such pipelines are not portable to other +implementations. ## Explicit Store Creation diff --git a/mkdocs.yml b/mkdocs.yml index 58b5d3344a..c0b7a96fa0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -81,6 +81,7 @@ nav: - ' zarr.open_consolidated': api/zarr/functions/open_consolidated.md - ' zarr.open_group': api/zarr/functions/open_group.md - ' zarr.open_like': api/zarr/functions/open_like.md + - ' zarr.open_url': api/zarr/functions/open_url.md - ' zarr.print_debug_info': api/zarr/functions/print_debug_info.md - ' zarr.registry': api/zarr/registry.md - ' zarr.save': api/zarr/functions/save.md diff --git a/src/zarr/__init__.py b/src/zarr/__init__.py index f305245f4e..b8ac1fb11e 100644 --- a/src/zarr/__init__.py +++ b/src/zarr/__init__.py @@ -27,6 +27,7 @@ open_consolidated, open_group, open_like, + open_url, save, save_array, save_group, @@ -175,6 +176,7 @@ def set_format(log_format: str) -> None: "open_consolidated", "open_group", "open_like", + "open_url", "print_debug_info", "save", "save_array", diff --git a/src/zarr/abc/url_pipeline.py b/src/zarr/abc/url_pipeline.py index 337563685d..e1cf96db41 100644 --- a/src/zarr/abc/url_pipeline.py +++ b/src/zarr/abc/url_pipeline.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: from zarr.abc.store import Store - from zarr.core.common import AccessModeLiteral, ZarrFormat + from zarr.core.common import AccessModeLiteral __all__ = [ "AdapterResolution", @@ -89,17 +89,17 @@ class AdapterResolution: Residual path *within* the store that the pipeline addresses (e.g. `"path/to/node"` for `...|icechunk://tag.v1/path/to/node`). Empty string when the pipeline addresses the store root. - zarr_format : ZarrFormat | None - Zarr format selected by a format segment (`zarr2:`/`zarr3:`), - or None if unspecified. A wrapper adapter that re-wraps a preceding - resolution must carry every field it does not change forward — - prefer `dataclasses.replace(preceding, store=..., path=...)` over - reconstructing, so fields added later are never silently dropped. + + A wrapper adapter that re-wraps a preceding resolution must carry every + field it does not change forward: prefer + `dataclasses.replace(preceding, store=..., path=...)` over + reconstructing, so fields added later are never silently dropped. + Format segments (`zarr2:`/`zarr3:`) are not adapters; they are consumed + by [`URLPipeline`][zarr.storage.URLPipeline] before resolution. """ store: Store path: str = "" - zarr_format: ZarrFormat | None = None @dataclass(frozen=True, eq=False) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 4bb3ce621d..9b110ed63e 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -48,7 +48,7 @@ ZarrUserWarning, ) from zarr.storage import StorePath -from zarr.storage._common import make_store_path +from zarr.storage._common import make_store_path, resolve_zarr_format if TYPE_CHECKING: from collections.abc import Iterable @@ -88,6 +88,7 @@ "open_consolidated", "open_group", "open_like", + "open_url", "save", "save_array", "save_group", @@ -403,8 +404,8 @@ async def open( mode = "r" else: mode = "a" + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, mode=mode, path=path, storage_options=storage_options) - zarr_format = store_path.resolve_zarr_format(zarr_format) # TODO: the mode check below seems wrong! if "shape" not in kwargs and mode in (*_READ_MODES, "w"): @@ -436,6 +437,60 @@ async def open( return await open_group(store=store_path, zarr_format=zarr_format, mode=mode, **kwargs) +async def open_url( + url: str, + *, + mode: AccessModeLiteral | None = None, + zarr_format: ZarrFormat | None = None, + path: str | None = None, + storage_options: dict[str, Any] | None = None, + **kwargs: Any, +) -> AnyAsyncArray | AsyncGroup: + """Open a group or array addressed by a URL pipeline. + + This is [`open`][zarr.api.asynchronous.open] with its `store` argument + interpreted as a [URL pipeline][user-guide-url-pipelines] string, i.e. + `open(URLPipeline.from_url(url), ...)`. It is the only entry point that + reads a string as a pipeline; `open` and the other `StoreLike`-taking + functions treat strings as they always have. + + Parameters + ---------- + url : str + A URL pipeline such as `"s3://bucket/data.zip|zip:|zarr3:"`. A URL + without `|` is a trivial pipeline consisting of its root alone. + mode : {'r', 'r+', 'a', 'w', 'w-'}, optional + Persistence mode, with the same meaning as for `zarr.open`. + zarr_format : {2, 3, None}, optional + The zarr format to use. None (the default) defers to a trailing + `zarr2:`/`zarr3:` segment of the pipeline; a value that conflicts + with such a segment raises `ValueError`. + path : str or None, optional + The path within the store to open, appended to any node path the + pipeline itself addresses. + storage_options : dict + Options for the pipeline's root sub-URL (e.g. fsspec options). + **kwargs + Additional parameters are passed through to `zarr.open_array` or + `zarr.open_group`. + + Returns + ------- + z : array or group + Return type depends on what exists in the given store. + """ + from zarr.storage import URLPipeline + + return await open( + store=URLPipeline.from_url(url), + mode=mode, + zarr_format=zarr_format, + path=path, + storage_options=storage_options, + **kwargs, + ) + + async def open_consolidated( *args: Any, use_consolidated: Literal[True] = True, **kwargs: Any ) -> AsyncGroup: @@ -531,8 +586,8 @@ async def save_array( raise TypeError("arr argument must be numpy or other NDArrayLike array") mode = kwargs.pop("mode", "a") + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - zarr_format = store_path.resolve_zarr_format(zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() if np.isscalar(arr): @@ -600,8 +655,8 @@ async def save_group( for arr in (*args, *kwargs.values()): get_data_type_from_native_dtype(arr.dtype) + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, path=path, mode="w", storage_options=storage_options) - zarr_format = store_path.resolve_zarr_format(zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() # The group replaces whatever is stored under the path, now that the arguments are known @@ -800,8 +855,8 @@ async def create_group( mode: Literal["a"] = "a" + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - zarr_format = store_path.resolve_zarr_format(zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() @@ -893,8 +948,8 @@ async def open_group( } ) + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path) - zarr_format = store_path.resolve_zarr_format(zarr_format) if attributes is None: attributes = {} @@ -1119,8 +1174,8 @@ async def create( if mode is None: mode = "a" overwrite = overwrite or _infer_overwrite(mode) + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - zarr_format = store_path.resolve_zarr_format(zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() @@ -1336,8 +1391,8 @@ async def open_array( """ mode = kwargs.pop("mode", None) + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - zarr_format = store_path.resolve_zarr_format(zarr_format) if "write_empty_chunks" in kwargs: _warn_write_empty_chunks_kwarg() diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 6778f8d907..a599f6b8e1 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -66,6 +66,7 @@ "open_consolidated", "open_group", "open_like", + "open_url", "save", "save_array", "save_group", @@ -246,6 +247,60 @@ def open( return Group(obj) +def open_url( + url: str, + *, + mode: AccessModeLiteral | None = None, + zarr_format: ZarrFormat | None = None, + path: str | None = None, + storage_options: dict[str, Any] | None = None, + **kwargs: Any, +) -> AnyArray | Group: + """Open a group or array addressed by a URL pipeline. + + This is [`open`][zarr.open] with its `store` argument interpreted as a + [URL pipeline][user-guide-url-pipelines] string, i.e. + `zarr.open(URLPipeline.from_url(url), ...)`. It is the only entry point + that reads a string as a pipeline; `zarr.open` and the other + `StoreLike`-taking functions treat strings as they always have. + + Parameters + ---------- + url : str + A URL pipeline such as `"s3://bucket/data.zip|zip:|zarr3:"`. A URL + without `|` is a trivial pipeline consisting of its root alone. + mode : {'r', 'r+', 'a', 'w', 'w-'}, optional + Persistence mode, with the same meaning as for `zarr.open`. + zarr_format : {2, 3, None}, optional + The zarr format to use. None (the default) defers to a trailing + `zarr2:`/`zarr3:` segment of the pipeline; a value that conflicts + with such a segment raises `ValueError`. + path : str or None, optional + The path within the store to open, appended to any node path the + pipeline itself addresses. + storage_options : dict + Options for the pipeline's root sub-URL (e.g. fsspec options). + **kwargs + Additional parameters are passed through to `zarr.open_array` or + `zarr.open_group`. + + Returns + ------- + z : array or group + Return type depends on what exists in the given store. + """ + from zarr.storage import URLPipeline + + return open( + store=URLPipeline.from_url(url), + mode=mode, + zarr_format=zarr_format, + path=path, + storage_options=storage_options, + **kwargs, + ) + + def open_consolidated(*args: Any, use_consolidated: Literal[True] = True, **kwargs: Any) -> Group: """ Alias for [`open_group`][zarr.api.synchronous.open_group] with `use_consolidated=True`. @@ -866,7 +921,7 @@ def create_array( serializer: SerializerLike = "auto", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, - zarr_format: ZarrFormat | None = None, + zarr_format: ZarrFormat | None = 3, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index f32ee7a94b..feedea7528 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -154,7 +154,7 @@ _parse_bytes_bytes_codec, get_pipeline_class, ) -from zarr.storage._common import StorePath, make_store_path +from zarr.storage._common import StorePath, make_store_path, resolve_zarr_format from zarr.storage._utils import _relativize_path if TYPE_CHECKING: @@ -466,8 +466,8 @@ async def _create( Deprecated in favor of [`zarr.api.asynchronous.create_array`][]. """ + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store) - zarr_format = store_path.resolve_zarr_format(zarr_format) or zarr_format dtype_parsed = parse_dtype(dtype, zarr_format=zarr_format) shape = parse_shapelike(shape) @@ -819,8 +819,8 @@ async def example(): # ``` """ + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store) - zarr_format = store_path.resolve_zarr_format(zarr_format) metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format) # TODO: remove this cast when we have better type hints _metadata_dict = cast("ArrayMetadataJSON_V3", metadata_dict) @@ -4387,8 +4387,8 @@ async def from_array( """ mode: Literal["a"] = "a" config_parsed = parse_array_config(config) + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, path=name, mode=mode, storage_options=storage_options) - zarr_format = store_path.resolve_zarr_format(zarr_format) ( chunks, @@ -4714,7 +4714,7 @@ async def create_array( serializer: SerializerLike = "auto", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, - zarr_format: ZarrFormat | None = None, + zarr_format: ZarrFormat | None = 3, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, @@ -4874,10 +4874,10 @@ async def create_array( else: mode: Literal["a"] = "a" + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path( store, path=name, mode=mode, storage_options=storage_options ) - zarr_format = store_path.resolve_zarr_format(zarr_format) return await init_array( store_path=store_path, shape=shape_parsed, diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index b541aff437..01d6e05df9 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -68,7 +68,7 @@ ZarrUserWarning, ) from zarr.storage import StoreLike, StorePath -from zarr.storage._common import make_store_path +from zarr.storage._common import make_store_path, resolve_zarr_format from zarr.storage._utils import _join_paths, _normalize_path_keys, normalize_path if TYPE_CHECKING: @@ -531,10 +531,10 @@ async def from_store( *, attributes: dict[str, Any] | None = None, overwrite: bool = False, - zarr_format: ZarrFormat | None = None, + zarr_format: ZarrFormat | None = 3, ) -> AsyncGroup: + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store) - zarr_format = store_path.resolve_zarr_format(zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() attributes = attributes or {} @@ -578,8 +578,8 @@ async def open( (``.zmetadata`` by default). Specify the custom key as ``use_consolidated`` to load consolidated metadata from a non-default key. """ + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store) - zarr_format = store_path.resolve_zarr_format(zarr_format) if not store_path.store.supports_consolidated_metadata: # Fail if consolidated metadata was requested but the Store doesn't support it if use_consolidated: @@ -1959,7 +1959,7 @@ def from_store( store: StoreLike, *, attributes: dict[str, Any] | None = None, - zarr_format: ZarrFormat | None = None, + zarr_format: ZarrFormat | None = 3, overwrite: bool = False, ) -> Group: """Instantiate a group from an initialized store. @@ -1973,8 +1973,8 @@ def from_store( attributes : dict, optional A dictionary of JSON-serializable values with user-defined attributes. zarr_format : {2, 3, None}, optional - Zarr storage format version. If None, the format selected by a URL - pipeline segment is used, falling back to the configured default. + Zarr storage format version. None defers to the format selected by a + `URLPipeline` store, falling back to the configured default. overwrite : bool, optional If True, do not raise an error if the group already exists. diff --git a/src/zarr/storage/__init__.py b/src/zarr/storage/__init__.py index edc01f1a5b..66328d5a32 100644 --- a/src/zarr/storage/__init__.py +++ b/src/zarr/storage/__init__.py @@ -10,7 +10,7 @@ from zarr.storage._logging import LoggingStore from zarr.storage._memory import GpuMemoryStore, ManagedMemoryStore, MemoryStore from zarr.storage._obstore import ObjectStore -from zarr.storage._url_pipeline import parse_pipeline, resolve_pipeline +from zarr.storage._url_pipeline import URLPipeline, parse_pipeline, resolve_pipeline from zarr.storage._wrapper import WrapperStore from zarr.storage._zip import ZipStore @@ -24,6 +24,7 @@ "ObjectStore", "StoreLike", "StorePath", + "URLPipeline", "WrapperStore", "ZipStore", "parse_pipeline", diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index f649a7eea6..3886e9d0f8 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -2,7 +2,7 @@ import importlib.util from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal, Self +from typing import TYPE_CHECKING, Any, Literal, Self, overload from zarr.abc.store import ( ByteRequest, @@ -29,7 +29,7 @@ ) from zarr.storage._local import LocalStore from zarr.storage._memory import ManagedMemoryStore, MemoryStore -from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline +from zarr.storage._url_pipeline import URLPipeline, resolve_pipeline from zarr.storage._utils import UPath, _join_paths, normalize_path, parse_store_url _has_fsspec = importlib.util.find_spec("fsspec") @@ -53,57 +53,19 @@ class StorePath: The store to use. path : str The path within the store. - - Attributes - ---------- - zarr_format : ZarrFormat | None - Zarr format selected by a `zarr2:`/`zarr3:` URL pipeline segment, - or None. Not part of the StorePath's identity (ignored by `__eq__`). """ store: Store path: str - zarr_format: ZarrFormat | None def __init__(self, store: Store, path: str = "") -> None: self.store = store self.path = normalize_path(path) - self.zarr_format = None @property def read_only(self) -> bool: return self.store.read_only - def resolve_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat | None: - """ - Combine the zarr format selected by a URL pipeline segment (`zarr2:` / - `zarr3:`) with a caller-supplied `zarr_format`. - - Parameters - ---------- - zarr_format : ZarrFormat | None - The format requested by the caller, or None when unspecified. - - Returns - ------- - ZarrFormat | None - The pipeline's format when the caller did not specify one, otherwise the - caller's format. None when neither is set. - - Raises - ------ - ValueError - If the caller's format differs from the one selected by the pipeline. - """ - if self.zarr_format is None: - return zarr_format - if zarr_format is not None and zarr_format != self.zarr_format: - raise ValueError( - f"zarr_format={zarr_format} conflicts with the 'zarr{self.zarr_format}:' " - "segment of the URL pipeline; pass zarr_format=None to use the pipeline's format" - ) - return self.zarr_format - @classmethod async def _create_open_instance(cls, store: Store, path: str) -> Self: """Helper to create and return a StorePath instance.""" @@ -116,10 +78,7 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No Open StorePath based on the provided mode. * If the mode is None, return an opened version of the store with no changes. - * If the mode is 'r+', 'w-', or 'w' and the store is read-only, raise a ValueError. - * If the mode is 'a' (open-or-create) and the store is read-only, serve the - "open" half: the StorePath is opened read-only, and any subsequent write - fails at the store level. + * If the mode is 'r+', 'w-', 'w', or 'a' and the store is read-only, raise a ValueError. * If the mode is 'r' and the store is not read-only, return a copy of the store with read_only set to True. * If the mode is 'w-' and the store is not read-only and the StorePath contains keys, raise a FileExistsError. @@ -145,8 +104,7 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No FileExistsError If the mode is 'w-' and the store path already exists. ValueError - If the mode is "r+", "w-", or "w" and the store is read-only, or if the - mode is not a valid access mode. + If the mode is not "r" and the store is read-only, or """ # fastpath if mode is None @@ -157,9 +115,8 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") if store.read_only: - # mode "a" (open-or-create) on a read-only store means open-only; - # unambiguous write modes are rejected outright. - if mode not in ("r", "a"): + # Don't allow write operations on a read-only store + if mode != "r": raise ValueError( f"Store is read-only but mode is {mode!r}. Create a writable store or use 'r' mode." ) @@ -320,9 +277,7 @@ def delete_sync(self) -> None: def __truediv__(self, other: str) -> StorePath: """Combine this store path with another path""" - result = self.__class__(self.store, _join_paths([self.path, other])) - result.zarr_format = self.zarr_format - return result + return self.__class__(self.store, _join_paths([self.path, other])) def __str__(self) -> str: return _join_paths([str(self.store), self.path]) @@ -350,7 +305,7 @@ def __eq__(self, other: object) -> bool: return False -type StoreLike = Store | StorePath | FSMap | Path | UPath | str | dict[str, Buffer] +type StoreLike = Store | StorePath | URLPipeline | FSMap | Path | UPath | str | dict[str, Buffer] async def make_store( @@ -371,9 +326,9 @@ async def make_store( - `None` = `MemoryStore` object. - `FSMap` = `FsspecStore` object. - `UPath` = `FsspecStore` object, or `LocalStore` for a local `UPath`. - - `str` containing `|` (or whose scheme has a registered root adapter) = the store - resolved by the [URL pipeline][user-guide-url-pipelines]; the pipeline must address - the store root (use `make_store_path` for pipelines that address a path within a store). + - `URLPipeline` = the store resolved by the [URL pipeline][user-guide-url-pipelines]; + the pipeline must address the store root (use `make_store_path` for pipelines that + address a path within a store). Plain strings are never interpreted as pipelines. Parameters ---------- @@ -404,12 +359,12 @@ async def make_store( raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") _read_only = mode == "r" - if isinstance(store_like, str) and is_url_pipeline(store_like): + if isinstance(store_like, URLPipeline): result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options) if normalize_path(result.path): result.store.close() raise URLPipelineError( - f"the URL pipeline {store_like!r} resolves to a path inside a store; " + f"the URL pipeline {str(store_like)!r} resolves to a path inside a store; " "use zarr.open() or make_store_path() instead of make_store()" ) return result.store @@ -521,8 +476,8 @@ async def make_store_path( ValueError If path is provided for a store that does not support it. URLPipelineError - If `store_like` is a [URL pipeline][user-guide-url-pipelines] string that - cannot be resolved. + If `store_like` is a [URL pipeline][user-guide-url-pipelines] that cannot be + resolved. See Also -------- @@ -530,12 +485,15 @@ async def make_store_path( """ path_normalized = normalize_path(path) - if isinstance(store_like, str) and is_url_pipeline(store_like): + if isinstance(store_like, URLPipeline): result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options) combined_path = _join_paths([normalize_path(result.path), path_normalized]) - store_path = await StorePath.open(result.store, path=combined_path, mode=mode) - store_path.zarr_format = result.zarr_format - return store_path + if mode == "a" and result.store.read_only: + # "a" (open-or-create, the zarr.open default) on a pipeline that + # resolves to a read-only resource serves the "open" half; any + # later write fails at the store level. + mode = "r" + return await StorePath.open(result.store, path=combined_path, mode=mode) if isinstance(store_like, StorePath): # Already a StorePath @@ -556,6 +514,36 @@ async def make_store_path( return await StorePath.open(store, path=path_normalized, mode=mode) +@overload +def resolve_zarr_format(store_like: StoreLike | None, zarr_format: ZarrFormat) -> ZarrFormat: ... + + +@overload +def resolve_zarr_format( + store_like: StoreLike | None, zarr_format: ZarrFormat | None +) -> ZarrFormat | None: ... + + +def resolve_zarr_format( + store_like: StoreLike | None, zarr_format: ZarrFormat | None +) -> ZarrFormat | None: + """ + Merge the zarr format selected by a `URLPipeline` store specification with + the format a caller asked for. + + For any other `StoreLike`, `zarr_format` is returned unchanged. For a + pipeline, see [`URLPipeline.resolve_zarr_format`][zarr.storage.URLPipeline]. + + Raises + ------ + ValueError + If `zarr_format` is not None and differs from the pipeline's format. + """ + if isinstance(store_like, URLPipeline): + return store_like.resolve_zarr_format(zarr_format) + return zarr_format + + async def ensure_no_existing_node( store_path: StorePath, zarr_format: ZarrFormat, diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py index 41a2114daf..ec8c9294eb 100644 --- a/src/zarr/storage/_url_pipeline.py +++ b/src/zarr/storage/_url_pipeline.py @@ -5,11 +5,11 @@ (registered through the `zarr.url_adapters` entry-point group) are loaded only when a pipeline URL naming their scheme is actually resolved. -The `|` character is reserved as the pipeline delimiter in every string -store specification: a string containing `|` is always routed through the -pipeline machinery, and no percent-escape is decoded. To address a local -file whose *name* contains `|` (or `#`), pass a `pathlib.Path` instead of a -string. +A string is never interpreted as a pipeline on its own: the user wraps it +in a [`URLPipeline`][zarr.storage.URLPipeline] (or calls `zarr.open_url`), +and that object is what the `StoreLike` machinery resolves. Plain string +store specifications, including local paths that happen to contain `|`, +keep their pre-existing meaning. No percent-escape is decoded. As a zarr-python extension to the specification (which requires the root sub-URL of an absolute pipeline to carry a scheme), the root sub-URL may be @@ -33,13 +33,18 @@ from zarr.errors import URLPipelineError from zarr.registry import get_url_adapter, list_url_adapter_schemes from zarr.storage._memory import ManagedMemoryStore -from zarr.storage._utils import parse_store_url +from zarr.storage._utils import _join_paths, normalize_path, parse_store_url if TYPE_CHECKING: from zarr.abc.store import Store - from zarr.core.common import AccessModeLiteral + from zarr.core.common import AccessModeLiteral, ZarrFormat -__all__ = ["is_url_pipeline", "parse_pipeline", "resolve_pipeline"] +__all__ = ["URLPipeline", "parse_pipeline", "resolve_pipeline"] + +# Format segments are consumed by zarr-python itself rather than by a +# registered adapter: they select the zarr format of the node the pipeline +# addresses, and their body is a path to that node. +_FORMAT_SCHEMES: dict[str, ZarrFormat] = {"zarr2": 2, "zarr3": 3} # Adapter scheme per RFC 3986 plus "." to permit vendor-prefixed # nonstandard schemes (e.g. "earthmover.myscheme"). @@ -137,6 +142,140 @@ def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]: return tuple(segments) +@dataclasses.dataclass(frozen=True) +class URLPipeline: + """ + A parsed URL pipeline, ready to be passed wherever a `StoreLike` is accepted. + + Construct one with [`from_url`][zarr.storage.URLPipeline.from_url]. Because a + `URLPipeline` is an explicit object, no plain string store specification is + ever interpreted as a pipeline: `zarr.open("data|x.zarr")` still addresses a + local directory of that name, while + `zarr.open(URLPipeline.from_url("s3://bucket/data.zip|zip:|zarr3:"))` (or the + equivalent `zarr.open_url(...)`) resolves the pipeline through registered + adapters. + + A pipeline is a value: instances are immutable, hashable, and compare equal + when they hold the same segments. Nothing is resolved until the pipeline is + opened; resolution needs the access mode and storage options of the open + call. + + Attributes + ---------- + segments : tuple[PipelineSegment, ...] + Every `|`-delimited sub-URL of the pipeline, in order: the root, any + adapter segments, and an optional trailing format segment. + + Raises + ------ + URLPipelineError + If there are no segments, if a format segment (`zarr2:`/`zarr3:`) + appears anywhere but last, or if a format segment carries a query. + """ + + segments: tuple[PipelineSegment, ...] + + def __post_init__(self) -> None: + if not self.segments: + raise URLPipelineError("a URL pipeline needs at least one segment") + for segment in self.segments[1:-1]: + if segment.scheme in _FORMAT_SCHEMES: + raise URLPipelineError( + f"the format segment {segment.raw!r} must be the last segment " + f"of the URL pipeline {str(self)!r}" + ) + format_segment = self._format_segment + if format_segment is not None and format_segment.query is not None: + raise URLPipelineError( + f"'{format_segment.scheme}:' segments do not accept a query: {format_segment.raw!r}" + ) + + @classmethod + def from_url(cls, url: str) -> URLPipeline: + """ + Parse a URL pipeline string. + + A URL without a `|` is a trivial pipeline consisting of its root alone. + + Raises + ------ + URLPipelineError + If the string cannot be parsed or the resulting pipeline is invalid + (see the class docstring). + """ + return cls(parse_pipeline(url)) + + @property + def _format_segment(self) -> PipelineSegment | None: + last = self.segments[-1] + if len(self.segments) > 1 and last.scheme in _FORMAT_SCHEMES: + return last + return None + + @property + def store_segments(self) -> tuple[PipelineSegment, ...]: + """ + The segments that address a store: the root and any adapter segments, + i.e. `segments` without a trailing format segment. + """ + if self._format_segment is None: + return self.segments + return self.segments[:-1] + + @property + def zarr_format(self) -> ZarrFormat | None: + """The format selected by a trailing `zarr2:`/`zarr3:` segment, or None.""" + format_segment = self._format_segment + if format_segment is None: + return None + return _FORMAT_SCHEMES[format_segment.scheme] + + @property + def path(self) -> str: + """ + The node path given as the body of a trailing format segment, + normalized. Empty when there is no format segment or its body is empty. + """ + format_segment = self._format_segment + if format_segment is None: + return "" + return normalize_path(format_segment.body) + + def resolve_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat | None: + """ + Combine the format selected by this pipeline with a caller-supplied one. + + Parameters + ---------- + zarr_format : ZarrFormat | None + The format requested by the caller, or None when unspecified. + + Returns + ------- + ZarrFormat | None + The pipeline's format when the caller did not specify one, otherwise + the caller's format. None when neither is set. + + Raises + ------ + ValueError + If the caller's format differs from the one selected by the pipeline. + """ + pipeline_format = self.zarr_format + if pipeline_format is None: + return zarr_format + if zarr_format is not None and zarr_format != pipeline_format: + raise ValueError( + f"zarr_format={zarr_format} conflicts with the 'zarr{pipeline_format}:' " + f"segment of the URL pipeline {str(self)!r}; pass zarr_format=None to use " + "the pipeline's format" + ) + return pipeline_format + + def __str__(self) -> str: + return "|".join(segment.raw for segment in self.segments) + + def _root_routes_to_adapter(scheme: str) -> bool: """ Whether a root sub-URL with this scheme is dispatched to a registered @@ -149,22 +288,8 @@ def _root_routes_to_adapter(scheme: str) -> bool: ) -def is_url_pipeline(url: str) -> bool: - """ - Whether `url` should be routed through the URL pipeline machinery. - - True when the URL contains a `|` separator, or when its scheme has a - registered URL pipeline adapter (a *root adapter* such as `gh:`). - fsspec chained URLs (`zip::file://...`) and zarr's native `file:` / - `memory:` schemes are never routed to a root adapter. - """ - if "|" in url: - return True - return _root_routes_to_adapter(_root_scheme(url)) - - async def resolve_pipeline( - url: str, + url: str | URLPipeline, *, mode: AccessModeLiteral | None = None, storage_options: dict[str, Any] | None = None, @@ -172,10 +297,17 @@ async def resolve_pipeline( """ Resolve a URL pipeline into a store and a residual path. + The residual path joins the path returned by the adapters with the node + path of a trailing format segment (`...|zarr3:path/to/node`). The format + itself is not part of the resolution; read it from + [`URLPipeline.zarr_format`][zarr.storage.URLPipeline]. + Parameters ---------- - url : str - A URL pipeline, e.g. `"s3://bucket/data.zip|zip:|zarr3:"`. + url : str | URLPipeline + A URL pipeline, e.g. `"s3://bucket/data.zip|zip:|zarr3:"`, or its + parsed form. A URL without `|` is a trivial pipeline consisting of + its root alone. mode : AccessModeLiteral | None The caller's access mode. `"r"` requires a read-only store; the resolver enforces this on whatever the final adapter returns. @@ -187,8 +319,7 @@ async def resolve_pipeline( Raises ------ URLPipelineError - If the URL cannot be parsed, is not a pipeline, names a scheme with - no registered adapter, names an adapter entry point that fails to + If the URL cannot be parsed, names a scheme with no registered adapter, names an adapter entry point that fails to import, or has a root sub-URL that cannot be resolved. TypeError If `storage_options` are passed to a root that does not accept them @@ -197,13 +328,13 @@ async def resolve_pipeline( Errors from opening the root resource (e.g. a missing local directory in mode `"r"`) propagate unchanged, as for non-pipeline stores. """ - segments = parse_pipeline(url) - if len(segments) == 1 and not _root_routes_to_adapter(segments[0].scheme): - raise URLPipelineError( - f"{url!r} is not a URL pipeline: it has no '|' separator and no " - f"URL pipeline adapter is registered for scheme {segments[0].scheme!r}" + pipeline = URLPipeline.from_url(url) if isinstance(url, str) else url + resolution = await _resolve(pipeline.store_segments, mode=mode, storage_options=storage_options) + if pipeline.path: + resolution = dataclasses.replace( + resolution, path=_join_paths([normalize_path(resolution.path), pipeline.path]) ) - return await _resolve(segments, mode=mode, storage_options=storage_options) + return resolution async def _resolve( @@ -218,14 +349,7 @@ async def _resolve( ) *preceding, last = segments - try: - adapter_cls = get_url_adapter(last.scheme) - except URLPipelineError as exc: - raise URLPipelineError( - f"{exc} Note: '|' is reserved as the URL pipeline delimiter; to " - "address a local file whose name contains '|', pass a " - "pathlib.Path instead of a string." - ) from None + adapter_cls = get_url_adapter(last.scheme) context = PipelineContext( preceding=tuple(preceding), diff --git a/tests/test_api.py b/tests/test_api.py index 6a2c4f85f8..f874bd45b7 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -311,12 +311,9 @@ async def test_open_like_creates_array_by_default( assert np.all(Array(new_arr)[:] == ref_arr.fill_value) -async def test_open_like_default_mode_on_read_only_store_raises_not_found( +async def test_open_like_default_mode_rejects_read_only_store( zarr_format: ZarrFormat, ) -> None: - # mode "a" (open-or-create) on a read-only store serves the "open" half: - # since the node does not exist and creating is impossible, the open - # fails with a not-found error rather than rejecting the mode upfront. ref_arr = zarr.create_array( store={}, shape=(11, 12), @@ -325,7 +322,7 @@ async def test_open_like_default_mode_on_read_only_store_raises_not_found( zarr_format=zarr_format, ) - with pytest.raises(ValueError, match="No array found"): + with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"): await zarr.api.asynchronous.open_like( ref_arr, path="foo", diff --git a/tests/test_url_pipeline/test_pipeline_object.py b/tests/test_url_pipeline/test_pipeline_object.py new file mode 100644 index 0000000000..c21e166810 --- /dev/null +++ b/tests/test_url_pipeline/test_pipeline_object.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import pytest + +from zarr.errors import URLPipelineError +from zarr.storage import URLPipeline +from zarr.storage._url_pipeline import parse_pipeline + + +@pytest.mark.parametrize( + ("url", "schemes", "store_schemes", "zarr_format", "path"), + [ + ( + "s3://bucket/data.zip|zip:inner|zarr3:", + ("s3", "zip", "zarr3"), + ("s3", "zip"), + 3, + "", + ), + # the trailing colon of a format segment is optional + ("file:///data/node.zarr|zarr2", ("file", "zarr2"), ("file",), 2, ""), + # the body of a format segment is a node path, normalized + ( + "file:///repo|icechunk:|zarr3:/path/to/array/", + ("file", "icechunk", "zarr3"), + ("file", "icechunk"), + 3, + "path/to/array", + ), + # no format segment + ("memory://x|wrap:a", ("memory", "wrap"), ("memory", "wrap"), None, ""), + # a single URL is a trivial pipeline + ("s3://bucket/key", ("s3",), ("s3",), None, ""), + # schemeless local root (zarr-python extension) + ("/local/data.zip|zip:", ("", "zip"), ("", "zip"), None, ""), + ], +) +def test_from_url( + url: str, + schemes: tuple[str, ...], + store_schemes: tuple[str, ...], + zarr_format: int | None, + path: str, +) -> None: + """ + `URLPipeline.from_url` holds every parsed segment of the URL, `str()` + reproduces the URL exactly, and the zarr format, node path and + store-addressing segments are derived from a trailing `zarr2:`/`zarr3:` + segment. + """ + pipeline = URLPipeline.from_url(url) + assert pipeline.segments == parse_pipeline(url) + assert tuple(segment.scheme for segment in pipeline.segments) == schemes + assert tuple(segment.scheme for segment in pipeline.store_segments) == store_schemes + assert pipeline.zarr_format == zarr_format + assert pipeline.path == path + assert str(pipeline) == url + assert URLPipeline(pipeline.segments) == pipeline + + +def test_format_segment_must_be_last() -> None: + """A `zarr2:`/`zarr3:` segment anywhere but the end of the pipeline is rejected.""" + with pytest.raises(URLPipelineError, match="must be the last segment"): + URLPipeline.from_url("file:/data|zarr3:|zip:") + + +def test_format_segment_rejects_query() -> None: + """A `zarr2:`/`zarr3:` segment takes a node path only; a query is an error.""" + with pytest.raises(URLPipelineError, match="do not accept a query"): + URLPipeline.from_url("file:/data|zarr3:x?opt=1") + + +def test_direct_construction_is_validated() -> None: + """Validation lives in the constructor, so a hand-built pipeline is checked too.""" + segments = parse_pipeline("file:/data|zarr3:|zip:") + with pytest.raises(URLPipelineError, match="must be the last segment"): + URLPipeline(segments) + + +def test_empty_pipeline_rejected() -> None: + """A pipeline needs at least a root segment.""" + with pytest.raises(URLPipelineError, match="at least one segment"): + URLPipeline(()) + + +def test_hashable() -> None: + """Pipelines are values: equal pipelines hash equal and can key a dict.""" + a = URLPipeline.from_url("s3://b/d.zip|zip:|zarr3:") + b = URLPipeline.from_url("s3://b/d.zip|zip:|zarr3:") + assert a == b + assert hash(a) == hash(b) + assert {a: 1}[b] == 1 + + +@pytest.mark.parametrize( + ("pipeline_format", "requested", "expected"), + [(None, None, None), (None, 3, 3), (2, None, 2), (2, 2, 2)], +) +def test_resolve_zarr_format( + pipeline_format: int | None, requested: int | None, expected: int | None +) -> None: + """ + `resolve_zarr_format` returns the pipeline's format when the caller did not + specify one, and the caller's format otherwise. + """ + suffix = "" if pipeline_format is None else f"|zarr{pipeline_format}:" + pipeline = URLPipeline.from_url(f"memory://x{suffix}") + assert pipeline.resolve_zarr_format(requested) == expected # type: ignore[arg-type] + + +def test_resolve_zarr_format_conflict_raises() -> None: + """A caller-supplied format that differs from the pipeline's segment is an error.""" + pipeline = URLPipeline.from_url("memory://x|zarr2:") + with pytest.raises(ValueError, match="conflicts with the 'zarr2:' segment"): + pipeline.resolve_zarr_format(3) diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index f9941b5101..d290a12dee 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -1,5 +1,6 @@ from __future__ import annotations +import contextlib import dataclasses import sys import threading @@ -26,9 +27,9 @@ list_url_adapter_schemes, register_url_adapter, ) -from zarr.storage import ManagedMemoryStore, MemoryStore, WrapperStore +from zarr.storage import LocalStore, ManagedMemoryStore, MemoryStore, URLPipeline, WrapperStore from zarr.storage._common import _has_fsspec, make_store, make_store_path -from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline +from zarr.storage._url_pipeline import resolve_pipeline from zarr.storage._utils import _join_paths # fsspec < 2024.12.0 has no AsyncFileSystemWrapper, so a plain memory:// URL @@ -112,17 +113,6 @@ async def open_pipeline_segment( return AdapterResolution(store=await MemoryStore.open(read_only=False)) -class FormatAdapter(URLPipelineAdapter): - """A dummy format-selecting adapter, like the builtin `zarr2:`.""" - - @classmethod - async def open_pipeline_segment( - cls, segment: PipelineSegment, context: PipelineContext - ) -> AdapterResolution: - preceding = await context.resolve_preceding() - return dataclasses.replace(preceding, zarr_format=2) - - class TestRegistry: def test_register_and_get(self) -> None: register_url_adapter("demo", WrapperAdapter) @@ -318,41 +308,46 @@ def test_duplicate_entry_points_warn_and_first_wins( assert not any(e.name == "dup" for e in pending) -class TestIsURLPipeline: - def test_pipe_routes(self) -> None: - assert is_url_pipeline("memory://x|demo:") +class TestStringsAreNeverPipelines: + """ + Only a `URLPipeline` object is resolved as a pipeline. A plain string + keeps the meaning it had before URL pipelines existed, whatever adapters + are registered. + """ + + async def test_local_path_containing_pipe_is_a_local_store(self, tmp_path: Path) -> None: + """A string path with `|` and a registered adapter's name is still a local directory.""" + register_url_adapter("zip", WrapperAdapter) + store_path = await make_store_path(f"{tmp_path}/a|zip:x", mode="w") + assert isinstance(store_path.store, LocalStore) + assert store_path.store.root == tmp_path / "a|zip:x" - def test_registered_root_scheme_routes(self) -> None: - register_url_adapter("rooty", RootAdapter) - assert is_url_pipeline("rooty://org/repo") + async def test_registered_root_scheme_string_is_not_routed(self) -> None: + """A URL whose scheme names a registered root adapter is not handed to that adapter.""" + calls: list[PipelineSegment] = [] - @pytest.mark.parametrize("url", ["s3://bucket/key", "/local/path", "memory://x", "C:.zarr"]) - def test_plain_urls_do_not_route(self, url: str) -> None: - assert not is_url_pipeline(url) + class CountingRootAdapter(RootAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + calls.append(segment) + return await super().open_pipeline_segment(segment, context) - def test_fsspec_chained_urls_do_not_route(self) -> None: - # fsspec's ``scheme::`` chaining is not a URL pipeline: even with a - # same-named root adapter registered, these keep flowing to fsspec. - register_url_adapter("zip", RootAdapter) - assert not is_url_pipeline("zip::file:///tmp/data.zip") - assert not is_url_pipeline("simplecache::s3://bucket/key") + register_url_adapter("rooty", CountingRootAdapter) + # the string falls through to the fsspec route, which may raise for + # an unknown protocol; whatever happens, the adapter is not consulted + with contextlib.suppress(Exception): + await make_store_path("rooty://org/repo") + assert calls == [] def test_native_schemes_do_not_route_to_adapters(self) -> None: - # an installed package must not be able to intercept zarr's own - # local-path and in-memory routing by registering file:/memory: + """An installed package cannot intercept `file:`/`memory:` roots by registering them.""" register_url_adapter("file", RootAdapter) register_url_adapter("memory", RootAdapter) - assert not is_url_pipeline("file:/tmp/x") - assert not is_url_pipeline("memory://x") - - def test_exotic_authority_root_scheme(self) -> None: - # The spec's own root-URL example: urlparse rejects the bracketed - # non-IP authority, so scheme detection must work on the raw string - # rather than raising before routing. - url = "vendor1-2.custom-1+proto.ext://[authority]/path?query/part?x" - assert not is_url_pipeline(url) - register_url_adapter("vendor1-2.custom-1+proto.ext", RootAdapter) - assert is_url_pipeline(url) + pipeline = URLPipeline.from_url("memory://x") + store = zarr.core.sync.sync(make_store(pipeline)) + assert isinstance(store, ManagedMemoryStore) class TestResolve: @@ -415,7 +410,7 @@ async def test_read_only_is_enforced_on_disobedient_adapters(self) -> None: register_url_adapter("bad", DisobedientAdapter) result = await resolve_pipeline("memory://base|bad:", mode="r") assert result.store.read_only - store = await make_store("memory://base|bad:", mode="r") + store = await make_store(URLPipeline.from_url("memory://base|bad:"), mode="r") assert store.read_only async def test_read_only_enforcement_without_conversion_raises(self) -> None: @@ -551,19 +546,29 @@ async def test_wrapper_at_root_position_raises(self) -> None: with pytest.raises(URLPipelineError, match="no preceding sub-URL"): await resolve_pipeline("wrap:whatever") - async def test_not_a_pipeline_raises(self) -> None: - with pytest.raises(URLPipelineError, match="is not a URL pipeline"): - await resolve_pipeline("s3://bucket/plain") + async def test_single_url_is_a_trivial_pipeline(self) -> None: + """A URL without `|` resolves as a pipeline consisting of its root alone.""" + result = await resolve_pipeline("memory://plain") + assert isinstance(result.store, ManagedMemoryStore) + assert result.path == "" + + async def test_accepts_pipeline_object(self, tmp_path: Path) -> None: + """`resolve_pipeline` takes a parsed `URLPipeline` as well as a string.""" + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(URLPipeline.from_url(f"{tmp_path}|wrap:a")) + assert isinstance(result.store, TracingStore) + assert result.path == "a" + + async def test_format_segment_path_joins_residual_path(self, tmp_path: Path) -> None: + """The node path of a trailing format segment extends the adapters' residual path.""" + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{tmp_path}|wrap:a|zarr3:b") + assert result.path == "a/b" async def test_unknown_adapter_scheme_raises(self) -> None: with pytest.raises(URLPipelineError, match="no URL pipeline adapter is registered"): await resolve_pipeline("memory://base|no-such-adapter:") - async def test_unknown_adapter_error_hints_at_reserved_pipe(self, tmp_path: Path) -> None: - # a filename containing '|' routes here; the error points at the fix - with pytest.raises(URLPipelineError, match="pass a\\s+pathlib.Path"): - await resolve_pipeline(f"{tmp_path}/a|b") - async def test_base_case_value_error_is_wrapped(self) -> None: # a root that only the fallback scheme detection accepts must not # leak urlparse's ValueError out of the base case @@ -571,15 +576,6 @@ async def test_base_case_value_error_is_wrapped(self) -> None: with pytest.raises(URLPipelineError, match="could not resolve the pipeline root"): await resolve_pipeline("vendor.x://[authority]/path|wrap:") - async def test_zarr_format_flows_from_adapter(self, tmp_path: Path) -> None: - register_url_adapter("fmt2", FormatAdapter) - result = await resolve_pipeline(f"{tmp_path}|fmt2:") - assert result.zarr_format == 2 - # wrapper adapters carry the format through (dataclasses.replace) - register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"{tmp_path}|fmt2:|wrap:") - assert result.zarr_format == 2 - class TestSpecRoots: """The spec's `memory:` / `file:` root semantics inside pipelines.""" @@ -606,9 +602,9 @@ async def test_memory_root_named_spellings_equivalent(self, root: str) -> None: async def test_memory_root_shares_data_across_spellings(self) -> None: register_url_adapter("wrap", WrapperAdapter) - group = zarr.open_group("memory:pipe-share|wrap:", mode="w") + group = zarr.open_group(URLPipeline.from_url("memory:pipe-share|wrap:"), mode="w") group.create_array("x", shape=(2,), dtype="i4") - reopened = zarr.open_group("memory://pipe-share|wrap:", mode="r") + reopened = zarr.open_group(URLPipeline.from_url("memory://pipe-share|wrap:"), mode="r") assert "x" in reopened async def test_memory_root_rejects_query(self) -> None: @@ -627,10 +623,10 @@ async def test_file_root_localhost_authority(self, tmp_path: Path) -> None: # spell the path URL-style: on Windows C:/... gains a leading slash posix = tmp_path.as_posix() url_path = posix if posix.startswith("/") else f"/{posix}" - zarr.open_group(f"file://localhost{url_path}|wrap:", mode="w") - opened = zarr.open_group(f"file:{tmp_path}|wrap:", mode="r") + zarr.open_group(URLPipeline.from_url(f"file://localhost{url_path}|wrap:"), mode="w") + opened = zarr.open_group(URLPipeline.from_url(f"file:{tmp_path}|wrap:"), mode="r") assert isinstance(opened, zarr.Group) - zarr.open_group(f"file://{url_path}|wrap:", mode="r") + zarr.open_group(URLPipeline.from_url(f"file://{url_path}|wrap:"), mode="r") async def test_file_root_windows_drive_spellings( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -676,7 +672,7 @@ async def test_memory_root_is_distinct_from_fsspec_memory_url(self) -> None: # is zarr's ManagedMemoryStore; the two do not share data register_url_adapter("wrap", WrapperAdapter) plain = await make_store("memory://distinct-store") - piped = await make_store("memory://distinct-store|wrap:") + piped = await make_store(URLPipeline.from_url("memory://distinct-store|wrap:")) assert not isinstance(plain, ManagedMemoryStore) assert isinstance(piped, TracingStore) assert isinstance(piped._store, ManagedMemoryStore) @@ -691,15 +687,26 @@ async def test_file_root_does_not_percent_decode(self, tmp_path: Path) -> None: class TestMakeStoreIntegration: + """`make_store` / `make_store_path` accept a `URLPipeline` like any other StoreLike.""" + async def test_make_store_path_combines_paths(self, tmp_path: Path) -> None: + """Adapter residual path, format-segment node path and the caller's path join in order.""" register_url_adapter("wrap", WrapperAdapter) - store_path = await make_store_path(f"{tmp_path}|wrap:residual", path="user/sub") - assert store_path.path == "residual/user/sub" + pipeline = URLPipeline.from_url(f"{tmp_path}|wrap:residual|zarr3:node") + store_path = await make_store_path(pipeline, path="user/sub") + assert store_path.path == "residual/node/user/sub" + assert isinstance(store_path.store, TracingStore) async def test_make_store_rejects_residual_path(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): - await make_store(f"{tmp_path}|wrap:residual") + await make_store(URLPipeline.from_url(f"{tmp_path}|wrap:residual")) + + async def test_make_store_rejects_format_segment_node_path(self, tmp_path: Path) -> None: + """A node path given on the format segment also addresses a path inside the store.""" + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): + await make_store(URLPipeline.from_url(f"{tmp_path}|wrap:|zarr3:node")) async def test_make_store_rejects_slash_root_path(self, tmp_path: Path) -> None: # an adapter returning path="/" addresses the store root; the raw @@ -713,7 +720,7 @@ async def open_pipeline_segment( return dataclasses.replace(preceding, path="/") register_url_adapter("slashy", SlashRootAdapter) - store = await make_store(f"{tmp_path}|slashy:") + store = await make_store(URLPipeline.from_url(f"{tmp_path}|slashy:")) assert store is not None async def test_make_store_closes_store_on_residual_path_error(self) -> None: @@ -733,12 +740,12 @@ async def open_pipeline_segment( register_url_adapter("leaky", LeakProbe) with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): - await make_store("memory://base|leaky:") + await make_store(URLPipeline.from_url("memory://base|leaky:")) assert closed == [True] async def test_make_store_no_residual_path(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - store = await make_store(f"{tmp_path}|wrap:") + store = await make_store(URLPipeline.from_url(f"{tmp_path}|wrap:")) assert isinstance(store, TracingStore) async def test_make_store_rejects_invalid_mode_before_resolving(self, tmp_path: Path) -> None: @@ -746,18 +753,18 @@ async def test_make_store_rejects_invalid_mode_before_resolving(self, tmp_path: # rather than being handed to adapters register_url_adapter("wrap", WrapperAdapter) with pytest.raises(ValueError, match="Invalid mode"): - await make_store(f"{tmp_path}|wrap:", mode="invalid") # type: ignore[arg-type] + await make_store(URLPipeline.from_url(f"{tmp_path}|wrap:"), mode="invalid") # type: ignore[arg-type] async def test_zarr_open_end_to_end(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - group = zarr.open_group(f"{tmp_path}|wrap:", mode="w") + group = zarr.open_group(URLPipeline.from_url(f"{tmp_path}|wrap:"), mode="w") array = group.create_array("x", shape=(4,), dtype="i4") array[:] = [1, 2, 3, 4] - assert zarr.open_array(f"{tmp_path}|wrap:x")[2] == 3 + assert zarr.open_array(URLPipeline.from_url(f"{tmp_path}|wrap:x"))[2] == 3 async def test_pipeline_store_read_only_mode(self) -> None: register_url_adapter("native", NativeAdapter) - store_path = await make_store_path("memory://base|native:", mode="r") + store_path = await make_store_path(URLPipeline.from_url("memory://base|native:"), mode="r") assert store_path.read_only async def test_mode_a_downgrades_to_read_only_open(self) -> None: @@ -765,20 +772,19 @@ async def test_mode_a_downgrades_to_read_only_open(self) -> None: # that resolves to a read-only store serves the "open" half instead # of failing outright. register_url_adapter("rooty", RootAdapter) - store_path = await make_store_path("rooty://org/repo", mode="a") + store_path = await make_store_path(URLPipeline.from_url("rooty://org/repo"), mode="a") assert store_path.read_only - async def test_mode_a_downgrade_applies_to_plain_stores_too(self) -> None: - # the open-or-create downgrade lives in StorePath.open, so it is - # uniform across pipeline and non-pipeline stores + async def test_mode_a_on_plain_read_only_store_still_raises(self) -> None: + """The open-or-create downgrade is specific to pipelines; plain stores behave as before.""" store = await MemoryStore.open(read_only=True) - store_path = await make_store_path(store, mode="a") - assert store_path.read_only + with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"): + await make_store_path(store, mode="a") async def test_explicit_write_mode_reaches_adapter(self) -> None: register_url_adapter("rooty", RootAdapter) with pytest.raises(ValueError, match="read-only scheme"): - await make_store_path("rooty://org/repo", mode="w") + await make_store_path(URLPipeline.from_url("rooty://org/repo"), mode="w") async def test_storage_options_forwarded_to_root(self) -> None: # storage_options reach the root sub-URL via resolve_preceding -> @@ -786,96 +792,133 @@ async def test_storage_options_forwarded_to_root(self) -> None: # forwarding them must raise the same TypeError as a non-pipeline open. register_url_adapter("wrap", WrapperAdapter) with pytest.raises(TypeError, match="'storage_options' was provided but unused"): - await make_store("/tmp/some/path|wrap:", storage_options={"anon": True}) + await make_store( + URLPipeline.from_url("/tmp/some/path|wrap:"), storage_options={"anon": True} + ) - async def test_store_path_truediv_keeps_format(self, tmp_path: Path) -> None: - # StorePath.__truediv__ constructs a new instance; the pipeline- - # selected format must survive the division - register_url_adapter("fmt2", FormatAdapter) - store_path = await make_store_path(f"{tmp_path}|fmt2:") - assert store_path.zarr_format == 2 - assert (store_path / "sub/node").zarr_format == 2 + async def test_store_path_has_no_format(self, tmp_path: Path) -> None: + """The pipeline's format is consumed by the open call; it does not linger on the StorePath.""" + store_path = await make_store_path(URLPipeline.from_url(f"{tmp_path}|zarr2:")) + assert not hasattr(store_path, "zarr_format") async def test_zarr_format_merges_into_open(self, tmp_path: Path) -> None: - register_url_adapter("fmt2", FormatAdapter) - group = zarr.open_group(f"{tmp_path}|fmt2:", mode="w") + group = zarr.open_group(URLPipeline.from_url(f"{tmp_path}|zarr2:"), mode="w") assert group.metadata.zarr_format == 2 async def test_conflicting_explicit_format_raises(self, tmp_path: Path) -> None: - register_url_adapter("fmt2", FormatAdapter) with pytest.raises(ValueError, match="conflicts with"): - zarr.open_group(f"{tmp_path}|fmt2:", mode="w", zarr_format=3) + zarr.open_group(URLPipeline.from_url(f"{tmp_path}|zarr2:"), mode="w", zarr_format=3) async def test_matching_explicit_format_ok(self, tmp_path: Path) -> None: - register_url_adapter("fmt2", FormatAdapter) - group = zarr.open_group(f"{tmp_path}|fmt2:", mode="w", zarr_format=2) + group = zarr.open_group(URLPipeline.from_url(f"{tmp_path}|zarr2:"), mode="w", zarr_format=2) + assert group.metadata.zarr_format == 2 + + +class TestOpenURL: + """`zarr.open_url` interprets its string argument as a URL pipeline, and nothing else does.""" + + async def test_open_url_resolves_pipeline_and_format(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + group = zarr.open_url(f"{tmp_path}|wrap:|zarr2:", mode="w") + assert isinstance(group, zarr.Group) + assert group.metadata.zarr_format == 2 + assert isinstance(group.store, TracingStore) + + async def test_open_url_path_and_kwargs_pass_through(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + array = zarr.open_url( + f"{tmp_path}|wrap:|zarr3:sub", path="x", mode="w", shape=(3,), dtype="i4" + ) + assert isinstance(array, zarr.Array) + assert array.metadata.zarr_format == 3 + assert array.store_path.path == "sub/x" + + async def test_open_url_conflicting_format_raises(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="conflicts with"): + zarr.open_url(f"{tmp_path}|zarr2:", mode="w", zarr_format=3) + + async def test_async_open_url(self, tmp_path: Path) -> None: + group = await zarr.api.asynchronous.open_url(f"{tmp_path}|zarr2:", mode="w") assert group.metadata.zarr_format == 2 class TestZarrFormatMergeInCore: """ - Every make_store_path caller honors the pipeline-selected zarr format, not - only the top-level zarr.api functions. + Every function taking a StoreLike honors the format selected by a + `URLPipeline`, not only the top-level zarr.api functions. Functions whose + `zarr_format` defaults to 3 need an explicit `zarr_format=None` to defer to + the pipeline, and raise when the default conflicts with it. """ - @pytest.fixture(autouse=True) - def _fmt2(self) -> None: - register_url_adapter("fmt2", FormatAdapter) + @pytest.fixture + def v2_pipeline(self, tmp_path: Path) -> URLPipeline: + return URLPipeline.from_url(f"{tmp_path}|zarr2:") - async def test_create_array(self, tmp_path: Path) -> None: - arr = zarr.create_array(f"{tmp_path}|fmt2:", name="x", shape=(2,), dtype="i4") + async def test_create_array(self, v2_pipeline: URLPipeline) -> None: + arr = zarr.create_array(v2_pipeline, name="x", shape=(2,), dtype="i4", zarr_format=None) assert arr.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): - zarr.create_array(f"{tmp_path}|fmt2:", name="y", shape=(2,), dtype="i4", zarr_format=3) + zarr.create_array(v2_pipeline, name="y", shape=(2,), dtype="i4") - async def test_create_array_from_data(self, tmp_path: Path) -> None: + async def test_create_array_from_data(self, v2_pipeline: URLPipeline) -> None: # the data path of create_array goes through from_array - arr = zarr.create_array(f"{tmp_path}|fmt2:", name="x", data=np.arange(3)) + arr = zarr.create_array(v2_pipeline, name="x", data=np.arange(3), zarr_format=None) assert arr.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): - zarr.create_array(f"{tmp_path}|fmt2:", name="y", data=np.arange(3), zarr_format=3) + zarr.create_array(v2_pipeline, name="y", data=np.arange(3)) - async def test_from_array(self, tmp_path: Path) -> None: - arr = zarr.from_array(f"{tmp_path}|fmt2:", name="x", data=np.arange(3)) + async def test_create_array_default_without_pipeline(self, tmp_path: Path) -> None: + """The default format for plain stores is still 3, independent of the config default.""" + with zarr.config.set({"default_zarr_format": 2}): + arr = zarr.create_array(tmp_path, shape=(2,), dtype="i4") + assert arr.metadata.zarr_format == 3 + + async def test_from_array(self, v2_pipeline: URLPipeline) -> None: + arr = zarr.from_array(v2_pipeline, name="x", data=np.arange(3)) assert arr.metadata.zarr_format == 2 - async def test_group_from_store(self, tmp_path: Path) -> None: - group = zarr.Group.from_store(f"{tmp_path}|fmt2:") + async def test_group_from_store(self, v2_pipeline: URLPipeline, tmp_path: Path) -> None: + group = zarr.Group.from_store(v2_pipeline, zarr_format=None) assert group.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): - zarr.Group.from_store(f"{tmp_path}/other|fmt2:", zarr_format=3) + zarr.Group.from_store(URLPipeline.from_url(f"{tmp_path}/other|zarr2:")) async def test_group_from_store_default_without_pipeline(self, tmp_path: Path) -> None: - # the default is still the configured format (3) for plain stores - group = zarr.Group.from_store(tmp_path) + """The default format for plain stores is still 3, independent of the config default.""" + with zarr.config.set({"default_zarr_format": 2}): + group = zarr.Group.from_store(tmp_path) assert group.metadata.zarr_format == 3 - async def test_group_open(self, tmp_path: Path) -> None: - zarr.Group.from_store(f"{tmp_path}|fmt2:") - # explicit None defers to the pipeline; the default (3) conflicts - group = zarr.Group.open(f"{tmp_path}|fmt2:", zarr_format=None) + async def test_group_open(self, v2_pipeline: URLPipeline) -> None: + zarr.Group.from_store(v2_pipeline, zarr_format=None) + group = zarr.Group.open(v2_pipeline, zarr_format=None) assert group.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): - zarr.Group.open(f"{tmp_path}|fmt2:") + zarr.Group.open(v2_pipeline) - async def test_array_open(self, tmp_path: Path) -> None: - zarr.create_array(f"{tmp_path}|fmt2:", name="x", shape=(2,), dtype="i4") - arr = zarr.Array.open(f"{tmp_path}/x|fmt2:", zarr_format=None) + async def test_array_open(self, v2_pipeline: URLPipeline, tmp_path: Path) -> None: + zarr.create_array(v2_pipeline, name="x", shape=(2,), dtype="i4", zarr_format=None) + array_pipeline = URLPipeline.from_url(f"{tmp_path}|zarr2:x") + arr = zarr.Array.open(array_pipeline, zarr_format=None) assert arr.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): - zarr.Array.open(f"{tmp_path}/x|fmt2:") + zarr.Array.open(array_pipeline) - async def test_open_like(self, tmp_path: Path) -> None: + async def test_open_like(self, v2_pipeline: URLPipeline) -> None: # open_like inherits v2 filters/compressor from the reference; the # pipeline supplies the format, which open_array then creates with ref = zarr.create_array({}, shape=(2,), dtype="i4", zarr_format=2) - arr = zarr.open_like(ref, store=f"{tmp_path}|fmt2:", path="x") + arr = zarr.open_like(ref, store=v2_pipeline, path="x") assert arr.metadata.zarr_format == 2 - async def test_deprecated_async_array_create(self, tmp_path: Path) -> None: + async def test_deprecated_async_array_create( + self, v2_pipeline: URLPipeline, tmp_path: Path + ) -> None: from zarr.core.array import AsyncArray - arr = await AsyncArray._create(f"{tmp_path}|fmt2:", shape=(2,), dtype="i4", zarr_format=2) + arr = await AsyncArray._create(v2_pipeline, shape=(2,), dtype="i4", zarr_format=2) assert arr.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): - await AsyncArray._create(f"{tmp_path}/o|fmt2:", shape=(2,), dtype="i4", zarr_format=3) + await AsyncArray._create( + URLPipeline.from_url(f"{tmp_path}/o|zarr2:"), shape=(2,), dtype="i4", zarr_format=3 + ) From 72682644b1edd93935edfdf678090bb50f1ead2e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 6 Oct 2026 11:06:55 +0200 Subject: [PATCH 09/20] refactor(url-pipeline): make open_url open existing nodes only `zarr.open_url(url, *, mode="r" | "r+", storage_options=None)` no longer mirrors `zarr.open`: it never creates or overwrites, takes no `path` (the node is addressed by the URL, via the format segment's body), no `zarr_format` (taken from the pipeline or inferred), and no `**kwargs`. Creating a node at a pipeline-addressed location is done by passing a `URLPipeline` to `create_array` / `create_group`. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- changes/4192.feature.md | 7 +-- docs/user-guide/storage.md | 9 ++-- src/zarr/api/asynchronous.py | 55 ++++++++++++------------ src/zarr/api/synchronous.py | 54 +++++++++++------------ tests/test_url_pipeline/test_resolver.py | 45 ++++++++++++++----- 5 files changed, 97 insertions(+), 73 deletions(-) diff --git a/changes/4192.feature.md b/changes/4192.feature.md index 91a4748417..dc3f224045 100644 --- a/changes/4192.feature.md +++ b/changes/4192.feature.md @@ -1,8 +1,9 @@ Added core support for URL pipelines (https://github.com/jbms/url-pipeline): `|`-chained URLs that address zarr data through nested storage layers, e.g. -`s3://bucket/data.zip|zip:|zarr3:`. A pipeline is opened with the new `zarr.open_url` -function, or by passing a parsed `zarr.storage.URLPipeline` (built with -`URLPipeline.from_url`) to any function that accepts a `StoreLike`. Adapters for the +`s3://bucket/data.zip|zip:|zarr3:`. A parsed `zarr.storage.URLPipeline` (built with +`URLPipeline.from_url`) can be passed to any function that accepts a `StoreLike`, and +the new `zarr.open_url(url, mode="r" | "r+")` opens the existing node a pipeline string +addresses without ever creating or overwriting anything. Adapters for the sub-URLs after the first are provided by third-party packages (e.g. Icechunk) through the `zarr.url_adapters` entry-point group and the single-method `zarr.abc.url_pipeline.URLPipelineAdapter` interface; they are loaded lazily and diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index 4d4571c7e3..95a55743d0 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -118,9 +118,9 @@ sub-URL locates a resource with a conventional URL; each subsequent sub-URL name [`zarr.abc.url_pipeline`][zarr.abc.url_pipeline] for the adapter interface. A builtin `zip:` adapter is under development and will expand this section. -A pipeline is opened either with [`zarr.open_url`][], which is `zarr.open` with its -string argument read as a pipeline, or by passing a parsed -[`URLPipeline`][zarr.storage.URLPipeline] to any function that accepts a `StoreLike`: +A parsed [`URLPipeline`][zarr.storage.URLPipeline] can be passed to any function that +accepts a `StoreLike`, and [`zarr.open_url`][] opens the *existing* node a pipeline +string addresses, read-only unless `mode="r+"` is given: ```python exec="true" session="storage" source="above" result="ansi" from zarr.storage import URLPipeline @@ -135,6 +135,9 @@ print(zarr.open_url("memory://pipeline-demo|zarr2:").metadata.zarr_format) Only these two entry points read a string as a pipeline. Plain string store specifications keep their existing meaning everywhere else, so a local path that happens to contain `|` still names a directory, whatever adapters are installed. +Unlike `zarr.open`, `zarr.open_url` never creates or overwrites: to create a node at a +pipeline-addressed location, pass the `URLPipeline` to `zarr.create_array` or +`zarr.create_group`. A trailing `zarr2:` or `zarr3:` segment is a *format segment*. It is not an adapter: zarr-python reads it from the pipeline itself and uses it as the `zarr_format` of the diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 9b110ed63e..4fc0abd8da 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -440,54 +440,53 @@ async def open( async def open_url( url: str, *, - mode: AccessModeLiteral | None = None, - zarr_format: ZarrFormat | None = None, - path: str | None = None, + mode: Literal["r", "r+"] = "r", storage_options: dict[str, Any] | None = None, - **kwargs: Any, ) -> AnyAsyncArray | AsyncGroup: - """Open a group or array addressed by a URL pipeline. + """Open the existing group or array addressed by a URL pipeline. + + This is the only entry point that reads a string as a + [URL pipeline][user-guide-url-pipelines]; [`open`][zarr.api.asynchronous.open] and the other + `StoreLike`-taking functions treat strings as they always have. Unlike + `open`, `open_url` never creates or overwrites anything: to create a node at + a pipeline-addressed location, pass a `URLPipeline` to + [`create_array`][zarr.api.asynchronous.create_array] or + [`create_group`][zarr.api.asynchronous.create_group]. - This is [`open`][zarr.api.asynchronous.open] with its `store` argument - interpreted as a [URL pipeline][user-guide-url-pipelines] string, i.e. - `open(URLPipeline.from_url(url), ...)`. It is the only entry point that - reads a string as a pipeline; `open` and the other `StoreLike`-taking - functions treat strings as they always have. + The zarr format is taken from a trailing `zarr2:`/`zarr3:` segment when + present, and inferred from the stored metadata otherwise. Parameters ---------- url : str A URL pipeline such as `"s3://bucket/data.zip|zip:|zarr3:"`. A URL - without `|` is a trivial pipeline consisting of its root alone. - mode : {'r', 'r+', 'a', 'w', 'w-'}, optional - Persistence mode, with the same meaning as for `zarr.open`. - zarr_format : {2, 3, None}, optional - The zarr format to use. None (the default) defers to a trailing - `zarr2:`/`zarr3:` segment of the pipeline; a value that conflicts - with such a segment raises `ValueError`. - path : str or None, optional - The path within the store to open, appended to any node path the - pipeline itself addresses. - storage_options : dict + without `|` is a trivial pipeline consisting of its root alone. The + node to open is addressed by the URL itself: the body of a trailing + `zarr2:`/`zarr3:` segment is the path of the node within the store. + mode : {'r', 'r+'}, optional + `'r'` (the default) opens read-only; `'r+'` opens for reading and + writing. The node must exist in either case. + storage_options : dict, optional Options for the pipeline's root sub-URL (e.g. fsspec options). - **kwargs - Additional parameters are passed through to `zarr.open_array` or - `zarr.open_group`. Returns ------- z : array or group - Return type depends on what exists in the given store. + Return type depends on what exists at the URL. For a statically typed + result, pass a `URLPipeline` to `open_array` or `open_group` instead. """ from zarr.storage import URLPipeline + if mode not in ("r", "r+"): + raise ValueError( + f"Invalid mode: {mode!r}. open_url opens existing nodes only and accepts " + "'r' or 'r+'; to create a node at a URL pipeline, pass " + "URLPipeline.from_url(url) to create_array or create_group." + ) return await open( store=URLPipeline.from_url(url), mode=mode, - zarr_format=zarr_format, - path=path, storage_options=storage_options, - **kwargs, ) diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index a599f6b8e1..5959316686 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -250,54 +250,52 @@ def open( def open_url( url: str, *, - mode: AccessModeLiteral | None = None, - zarr_format: ZarrFormat | None = None, - path: str | None = None, + mode: Literal["r", "r+"] = "r", storage_options: dict[str, Any] | None = None, - **kwargs: Any, ) -> AnyArray | Group: - """Open a group or array addressed by a URL pipeline. + """Open the existing group or array addressed by a URL pipeline. - This is [`open`][zarr.open] with its `store` argument interpreted as a - [URL pipeline][user-guide-url-pipelines] string, i.e. - `zarr.open(URLPipeline.from_url(url), ...)`. It is the only entry point - that reads a string as a pipeline; `zarr.open` and the other - `StoreLike`-taking functions treat strings as they always have. + This is the only entry point that reads a string as a + [URL pipeline][user-guide-url-pipelines]; [`open`][zarr.open] and the other + `StoreLike`-taking functions treat strings as they always have. Unlike + `open`, `open_url` never creates or overwrites anything: to create a node at + a pipeline-addressed location, pass a `URLPipeline` to + [`create_array`][zarr.create_array] or [`create_group`][zarr.create_group]. + + The zarr format is taken from a trailing `zarr2:`/`zarr3:` segment when + present, and inferred from the stored metadata otherwise. Parameters ---------- url : str A URL pipeline such as `"s3://bucket/data.zip|zip:|zarr3:"`. A URL - without `|` is a trivial pipeline consisting of its root alone. - mode : {'r', 'r+', 'a', 'w', 'w-'}, optional - Persistence mode, with the same meaning as for `zarr.open`. - zarr_format : {2, 3, None}, optional - The zarr format to use. None (the default) defers to a trailing - `zarr2:`/`zarr3:` segment of the pipeline; a value that conflicts - with such a segment raises `ValueError`. - path : str or None, optional - The path within the store to open, appended to any node path the - pipeline itself addresses. - storage_options : dict + without `|` is a trivial pipeline consisting of its root alone. The + node to open is addressed by the URL itself: the body of a trailing + `zarr2:`/`zarr3:` segment is the path of the node within the store. + mode : {'r', 'r+'}, optional + `'r'` (the default) opens read-only; `'r+'` opens for reading and + writing. The node must exist in either case. + storage_options : dict, optional Options for the pipeline's root sub-URL (e.g. fsspec options). - **kwargs - Additional parameters are passed through to `zarr.open_array` or - `zarr.open_group`. Returns ------- z : array or group - Return type depends on what exists in the given store. + Return type depends on what exists at the URL. For a statically typed + result, pass a `URLPipeline` to `open_array` or `open_group` instead. """ from zarr.storage import URLPipeline + if mode not in ("r", "r+"): + raise ValueError( + f"Invalid mode: {mode!r}. open_url opens existing nodes only and accepts " + "'r' or 'r+'; to create a node at a URL pipeline, pass " + "URLPipeline.from_url(url) to create_array or create_group." + ) return open( store=URLPipeline.from_url(url), mode=mode, - zarr_format=zarr_format, - path=path, storage_options=storage_options, - **kwargs, ) diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index d290a12dee..a642402ae2 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -815,30 +815,53 @@ async def test_matching_explicit_format_ok(self, tmp_path: Path) -> None: class TestOpenURL: - """`zarr.open_url` interprets its string argument as a URL pipeline, and nothing else does.""" + """ + `zarr.open_url` interprets its string argument as a URL pipeline and opens + the existing node it addresses. It never creates: creation goes through + `create_array` / `create_group` with a `URLPipeline`. + """ - async def test_open_url_resolves_pipeline_and_format(self, tmp_path: Path) -> None: + async def test_open_url_opens_existing_node_with_pipeline_format(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - group = zarr.open_url(f"{tmp_path}|wrap:|zarr2:", mode="w") + zarr.create_group(URLPipeline.from_url(f"{tmp_path}|wrap:|zarr2:")) + group = zarr.open_url(f"{tmp_path}|wrap:|zarr2:") assert isinstance(group, zarr.Group) assert group.metadata.zarr_format == 2 assert isinstance(group.store, TracingStore) + assert group.read_only - async def test_open_url_path_and_kwargs_pass_through(self, tmp_path: Path) -> None: + async def test_open_url_format_segment_body_addresses_node(self, tmp_path: Path) -> None: + """The node path lives in the URL, as the body of the format segment.""" register_url_adapter("wrap", WrapperAdapter) - array = zarr.open_url( - f"{tmp_path}|wrap:|zarr3:sub", path="x", mode="w", shape=(3,), dtype="i4" + zarr.create_array( + URLPipeline.from_url(f"{tmp_path}|wrap:"), name="sub/x", shape=(3,), dtype="i4" ) + array = zarr.open_url(f"{tmp_path}|wrap:|zarr3:sub/x") assert isinstance(array, zarr.Array) - assert array.metadata.zarr_format == 3 assert array.store_path.path == "sub/x" - async def test_open_url_conflicting_format_raises(self, tmp_path: Path) -> None: - with pytest.raises(ValueError, match="conflicts with"): - zarr.open_url(f"{tmp_path}|zarr2:", mode="w", zarr_format=3) + async def test_open_url_read_write(self, tmp_path: Path) -> None: + zarr.create_group(URLPipeline.from_url(f"{tmp_path}|zarr3:")) + group = zarr.open_url(f"{tmp_path}|zarr3:", mode="r+") + assert not group.read_only + group.attrs["answer"] = 42 + assert zarr.open_url(f"{tmp_path}|zarr3:").attrs["answer"] == 42 + + async def test_open_url_missing_node_raises(self, tmp_path: Path) -> None: + """Nothing is created on open: a URL to a missing node is an error in every mode.""" + with pytest.raises(FileNotFoundError): + zarr.open_url(f"{tmp_path}/missing|zarr3:") + with pytest.raises(FileNotFoundError): + zarr.open_url(f"{tmp_path}/missing|zarr3:", mode="r+") + + async def test_open_url_rejects_create_modes(self, tmp_path: Path) -> None: + """The creating modes of `zarr.open` are not accepted.""" + with pytest.raises(ValueError, match="opens existing nodes only"): + zarr.open_url(f"{tmp_path}|zarr3:", mode="a") # type: ignore[arg-type] async def test_async_open_url(self, tmp_path: Path) -> None: - group = await zarr.api.asynchronous.open_url(f"{tmp_path}|zarr2:", mode="w") + zarr.create_group(URLPipeline.from_url(f"{tmp_path}|zarr2:")) + group = await zarr.api.asynchronous.open_url(f"{tmp_path}|zarr2:") assert group.metadata.zarr_format == 2 From 2ec816cc21050f88b19aea7b386a6591deb3783a Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 6 Oct 2026 14:15:59 +0200 Subject: [PATCH 10/20] refactor(url-pipeline): drop the string-routing leftovers from the resolver With strings never routed as pipelines, four pieces of the resolver only served the old design: - The schemeless-root extension (`data.zip|zip:`). The root of a pipeline now carries a URL scheme, as the specification requires; a bare path is rejected with a message pointing at `file:` URLs and `%7C`. This also removes the one place where `?` and `#` were treated as filename characters. - The negative lookahead that kept fsspec's `zip::...` chained URLs out of root-scheme detection; it protected `zarr.open` strings that no longer reach the resolver. - The mode "a" downgrade for read-only pipelines in `make_store_path`; `open_url` defaults to "r", and a read-only pipeline now raises on "a" exactly like a plain read-only store. - The "non-dict `storage_options` reserved for per-segment configuration" reservation in docstrings and the user guide. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- changes/4192.feature.md | 2 - docs/user-guide/storage.md | 13 +- src/zarr/abc/url_pipeline.py | 7 +- src/zarr/storage/_common.py | 5 - src/zarr/storage/_url_pipeline.py | 56 ++++----- tests/test_url_pipeline/test_parser.py | 41 +------ .../test_url_pipeline/test_pipeline_object.py | 2 - tests/test_url_pipeline/test_resolver.py | 111 ++++++++++-------- 8 files changed, 98 insertions(+), 139 deletions(-) diff --git a/changes/4192.feature.md b/changes/4192.feature.md index dc3f224045..21e87736a3 100644 --- a/changes/4192.feature.md +++ b/changes/4192.feature.md @@ -20,8 +20,6 @@ Behavior notes: (`zarr.create_array`, `Group.from_store`, `Group.open`, `Array.open`) need an explicit `zarr_format=None` to defer to the pipeline. `Array.open` gains a `zarr_format` parameter mirroring `Group.open`. -- With the default mode `"a"`, a pipeline that resolves to a read-only resource is - opened for reading instead of raising; other stores behave as before. - Inside a pipeline, `memory:` and `file:` roots follow the URL pipeline spec's spelling rules (`memory:` ≡ `memory:/` ≡ `memory://`; `file:` must carry an absolute path with at most a `localhost` authority). Percent-escapes are not decoded, matching diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index 95a55743d0..bb294353aa 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -147,20 +147,17 @@ open. A `zarr_format` argument that conflicts with the format segment raises and `Group.open`) need an explicit `zarr_format=None` to defer to the pipeline. `storage_options` apply to the *root* sub-URL (e.g. fsspec options for `s3://...`); -adapters may consume adapter-specific, namespaced keys. Non-dict forms of -`storage_options` are reserved for future per-segment configuration. No percent-escape -is decoded anywhere in a pipeline. Registered adapters cannot intercept zarr's native -`file:` and `memory:` root schemes. With the default mode `"a"`, a pipeline that -resolves to a read-only resource is opened for reading instead of raising. +adapters may consume adapter-specific, namespaced keys. No percent-escape is decoded +anywhere in a pipeline. Registered adapters cannot intercept zarr's native `file:` and +`memory:` root schemes. Inside a pipeline, a `memory:` root is zarr's managed in-memory store (`memory:`, `memory:/` and `memory://` are equivalent, and `memory:name` selects a named store). When fsspec is installed, a plain `memory://name` string store specification is still routed to fsspec's in-memory filesystem, which is a different store. A `file:` root must carry an absolute path; percent-escapes are not decoded, matching the -[local store](#local-store). As a zarr-python extension, the root may also be a bare -local path (`data/example.zip|zip:`); such pipelines are not portable to other -implementations. +[local store](#local-store). The root always carries a scheme: a local path is spelled +as a `file:` URL, never as a bare path. ## Explicit Store Creation diff --git a/src/zarr/abc/url_pipeline.py b/src/zarr/abc/url_pipeline.py index e1cf96db41..f0f58140c8 100644 --- a/src/zarr/abc/url_pipeline.py +++ b/src/zarr/abc/url_pipeline.py @@ -50,8 +50,7 @@ class PipelineSegment: Attributes ---------- scheme : str - The lowercased URL scheme. Empty string only for a schemeless root - (a bare local path), which is treated as opaque text. + The lowercased URL scheme. body : str The text after `scheme:` and before any `?`. Interpretation is scheme-defined; it is **not** URL-normalized, so case-significant @@ -130,10 +129,6 @@ class PipelineContext: backends. An adapter that consumes keys should strip them before resolving the rest of the pipeline, by passing the reduced mapping to [`resolve_preceding`][zarr.abc.url_pipeline.PipelineContext.resolve_preceding]. - Non-dict forms of the caller-facing `storage_options` argument are - reserved for future per-segment configuration (one mapping per - pipeline segment); this attribute will remain a single mapping — - the one addressed to this adapter's segment. """ preceding: tuple[PipelineSegment, ...] diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index 3886e9d0f8..24d0dd4122 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -488,11 +488,6 @@ async def make_store_path( if isinstance(store_like, URLPipeline): result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options) combined_path = _join_paths([normalize_path(result.path), path_normalized]) - if mode == "a" and result.store.read_only: - # "a" (open-or-create, the zarr.open default) on a pipeline that - # resolves to a read-only resource serves the "open" half; any - # later write fails at the store level. - mode = "r" return await StorePath.open(result.store, path=combined_path, mode=mode) if isinstance(store_like, StorePath): diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py index ec8c9294eb..a040c61441 100644 --- a/src/zarr/storage/_url_pipeline.py +++ b/src/zarr/storage/_url_pipeline.py @@ -11,12 +11,8 @@ store specifications, including local paths that happen to contain `|`, keep their pre-existing meaning. No percent-escape is decoded. -As a zarr-python extension to the specification (which requires the root -sub-URL of an absolute pipeline to carry a scheme), the root sub-URL may be -a schemeless local filesystem path, e.g. `data/example.zip|zip:`. Schemeless -roots are treated as opaque text — no query or fragment splitting is applied -to them. Such pipelines are not portable to other URL pipeline -implementations; portable pipelines should spell the root as a `file:` URL. +As the specification requires, the root sub-URL carries a URL scheme: a +local path is spelled as an absolute `file:` URL. """ from __future__ import annotations @@ -50,10 +46,8 @@ # nonstandard schemes (e.g. "earthmover.myscheme"). _SCHEME_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.\-]*$") -# Root scheme at the very start of the sub-URL. The negative lookahead -# excludes fsspec's chained-URL syntax (``zip::file://...``), which is not a -# URL pipeline and keeps flowing through the fsspec machinery. -_ROOT_SCHEME_RE = re.compile(r"([a-zA-Z][a-zA-Z0-9+.\-]*):(?!:)") +# Root scheme at the very start of the sub-URL. +_ROOT_SCHEME_RE = re.compile(r"([a-zA-Z][a-zA-Z0-9+.\-]*):") # Schemes zarr resolves natively at the pipeline root. These are never # dispatched to a registered root adapter, so an installed package cannot @@ -67,12 +61,12 @@ def _root_scheme(root_sub_url: str) -> str: """ - Detect the scheme of the root sub-URL, or `""` for an opaque root. + Detect the scheme of the root sub-URL, or `""` when it has none. - Scheme extraction happens on the raw string (so nothing `urlparse` - would strip or reject — whitespace, exotic authorities — can desync the - detected scheme from the body). Single-letter candidates are delegated - to `parse_store_url`, which knows Windows drive letters are not schemes. + Scheme extraction happens on the raw string, so nothing `urlparse` + would strip or reject can desync the detected scheme from the body. + Single-letter candidates are delegated to `parse_store_url`, which + knows Windows drive letters are not schemes. """ match = _ROOT_SCHEME_RE.match(root_sub_url) if match is None: @@ -107,10 +101,9 @@ def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]: `scheme:body` where the trailing colon is optional when the body is empty (`zip` is equivalent to `zip:`). - A schemeless root (a bare local path) is accepted as a zarr-python - extension to the specification and treated as opaque text: no query or - fragment splitting applies, since `?` and `#` are ordinary filename - characters there. See the module docstring. + The root must carry a scheme; a bare local path is rejected, since the + specification has no schemeless sub-URLs and `|` could not be escaped + in one. Segment text is preserved verbatim (no case or percent-encoding normalization) except that schemes are lowercased. @@ -124,9 +117,11 @@ def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]: if index == 0: scheme = _root_scheme(part) if not scheme: - # opaque root (bare local path, fsspec chained URL, ...) - segments.append(PipelineSegment(scheme="", body=part, query=None, raw=part)) - continue + raise URLPipelineError( + f"the root sub-URL {part!r} of the URL pipeline {url!r} has no URL scheme. " + "Spell a local path as an absolute 'file:' URL, and a literal '|' in " + "it as '%7C'." + ) body_and_scheme, query = _split_query(part) body = body_and_scheme[len(scheme) + 1 :] segments.append(PipelineSegment(scheme=scheme, body=body, query=query, raw=part)) @@ -279,13 +274,11 @@ def __str__(self) -> str: def _root_routes_to_adapter(scheme: str) -> bool: """ Whether a root sub-URL with this scheme is dispatched to a registered - root adapter. Schemes zarr resolves natively (`file:`, `memory:`) and - opaque roots are excluded; the registry check inspects entry-point - names only — no adapter code is imported here. + root adapter. Schemes zarr resolves natively (`file:`, `memory:`) are + excluded; the registry check inspects entry-point names only — no + adapter code is imported here. """ - return ( - bool(scheme) and scheme not in _NATIVE_ROOT_SCHEMES and scheme in list_url_adapter_schemes() - ) + return scheme not in _NATIVE_ROOT_SCHEMES and scheme in list_url_adapter_schemes() async def resolve_pipeline( @@ -313,8 +306,7 @@ async def resolve_pipeline( resolver enforces this on whatever the final adapter returns. storage_options : dict | None Options forwarded to the root sub-URL's store (and visible to - adapters via the context). Non-dict forms are reserved for future - per-segment configuration (one mapping per pipeline segment). + adapters via the context). Raises ------ @@ -388,8 +380,8 @@ async def _resolve_root( `memory:` and `file:` roots are resolved here with the URL pipeline spec's semantics (spelling equivalences, mandatory absolute `file:` - paths); everything else — bare local paths, fsspec URLs — delegates to - the existing `StoreLike` machinery unchanged. + paths); every other scheme (e.g. fsspec URLs) delegates to the existing + `StoreLike` machinery unchanged. """ if segment.scheme == "memory": return _resolve_memory_root(segment, mode=mode, storage_options=storage_options) diff --git a/tests/test_url_pipeline/test_parser.py b/tests/test_url_pipeline/test_parser.py index 13d7846e85..3dc2480ca8 100644 --- a/tests/test_url_pipeline/test_parser.py +++ b/tests/test_url_pipeline/test_parser.py @@ -19,11 +19,11 @@ def test_single_root_url() -> None: assert str(segment) == "s3://bucket/key" -def test_schemeless_root() -> None: - (segment,) = parse_pipeline("/local/path") - assert segment.scheme == "" - assert segment.body == "/local/path" - assert segment.raw == "/local/path" +@pytest.mark.parametrize("url", ["/local/path", "/local/path|zip:", "\tfile:/tmp/x|zip:"]) +def test_schemeless_root_rejected(url: str) -> None: + """The root sub-URL must carry a URL scheme; a bare path (or anything urlparse would not read as a scheme) is an error.""" + with pytest.raises(URLPipelineError, match="URL scheme"): + parse_pipeline(url) def test_adapter_chain() -> None: @@ -111,32 +111,6 @@ def test_single_letter_scheme_with_exotic_authority() -> None: assert segments[0].raw == "x://[authority]/path" -def test_fsspec_chained_root_is_opaque() -> None: - # fsspec's ``scheme::`` chaining is not pipeline syntax; such roots are - # schemeless/opaque and flow through the ordinary store machinery - (segment,) = parse_pipeline("zip::file:///tmp/data.zip") - assert segment.scheme == "" - assert segment.body == "zip::file:///tmp/data.zip" - - -def test_whitespace_prefixed_root_is_opaque() -> None: - # urlparse strips leading whitespace when sniffing a scheme; the parser - # must not, or the detected scheme desyncs from the body slice - segments = parse_pipeline("\tfile:/tmp/x|zip:") - assert segments[0].scheme == "" - assert segments[0].body == "\tfile:/tmp/x" - - -def test_schemeless_root_keeps_query_and_fragment_chars() -> None: - # ? and # are ordinary filename characters in a bare local path - segments = parse_pipeline("/tmp/d?v=1|zip:") - assert segments[0].scheme == "" - assert segments[0].body == "/tmp/d?v=1" - assert segments[0].query is None - segments = parse_pipeline("/tmp/d#frag|zip:") - assert segments[0].body == "/tmp/d#frag" - - def test_round_trip() -> None: url = "s3://bucket/a.zip?v=2|zip:b/inner.zip|zip:c|zarr3:" segments = parse_pipeline(url) @@ -160,9 +134,6 @@ def test_round_trip() -> None: # the root scheme as a local path on one platform but not another _ROOT_SCHEME = st.from_regex(r"[a-zA-Z]{2}[a-zA-Z0-9+.\-]{0,14}", fullmatch=True) _BODY = st.text(st.characters(min_codepoint=32, max_codepoint=126, exclude_characters="|#?")) -# a root body starting with ":" would spell fsspec's chained-URL syntax -# ("scheme::..."), which the parser deliberately treats as an opaque root -_ROOT_BODY = _BODY.filter(lambda body: not body.startswith(":")) _QUERY = st.text(st.characters(min_codepoint=32, max_codepoint=126, exclude_characters="|#")) @@ -176,7 +147,7 @@ def pipelines( parts = [] for index in range(depth): scheme = draw(_ROOT_SCHEME if index == 0 else _ADAPTER_SCHEME) - body = draw(_ROOT_BODY if index == 0 else _BODY) + body = draw(_BODY) query = draw(st.none() | _QUERY) raw = f"{scheme}:{body}" + (f"?{query}" if query is not None else "") parts.append(raw) diff --git a/tests/test_url_pipeline/test_pipeline_object.py b/tests/test_url_pipeline/test_pipeline_object.py index c21e166810..4c0d2779a5 100644 --- a/tests/test_url_pipeline/test_pipeline_object.py +++ b/tests/test_url_pipeline/test_pipeline_object.py @@ -31,8 +31,6 @@ ("memory://x|wrap:a", ("memory", "wrap"), ("memory", "wrap"), None, ""), # a single URL is a trivial pipeline ("s3://bucket/key", ("s3",), ("s3",), None, ""), - # schemeless local root (zarr-python extension) - ("/local/data.zip|zip:", ("", "zip"), ("", "zip"), None, ""), ], ) def test_from_url( diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index a642402ae2..e00c29cb90 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -353,10 +353,10 @@ def test_native_schemes_do_not_route_to_adapters(self) -> None: class TestResolve: async def test_wrapper_adapter_chain(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"{tmp_path}|wrap:inner/path") + result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:inner/path") assert isinstance(result.store, TracingStore) assert result.path == "inner/path" - assert result.store.context.preceding_url == str(tmp_path) + assert result.store.context.preceding_url == f"file:{tmp_path.as_posix()}" assert result.store.segment.scheme == "wrap" assert result.store.segment.body == "inner/path" @@ -364,7 +364,7 @@ async def test_nested_wrappers_preserve_residual_paths(self, tmp_path: Path) -> # each wrapper joins the preceding residual path with its own, so # no segment's path is lost in root|wrap:a|wrap:b register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"{tmp_path}|wrap:a|wrap:b") + result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:a|wrap:b") assert result.path == "a/b" async def test_native_adapter_gets_preceding_url(self) -> None: @@ -395,11 +395,11 @@ async def test_root_adapter_composes_with_chain(self) -> None: async def test_read_only_flag(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"{tmp_path}|wrap:", mode="r") + result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:", mode="r") assert isinstance(result.store, TracingStore) assert result.store.context.read_only assert result.store.context.mode == "r" - result = await resolve_pipeline(f"{tmp_path}|wrap:") + result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:") assert isinstance(result.store, TracingStore) assert not result.store.context.read_only assert result.store.context.mode is None @@ -467,7 +467,7 @@ async def open_pipeline_segment( return dataclasses.replace(preceding, path=segment.body) register_url_adapter("rowrap", ReadOnlyRootWrapper) - result = await resolve_pipeline(f"{tmp_path}|rowrap:x", mode="w") + result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|rowrap:x", mode="w") assert result.path == "x" async def test_wrapper_on_local_file_root_read_only(self, tmp_path: Path) -> None: @@ -477,7 +477,7 @@ async def test_wrapper_on_local_file_root_read_only(self, tmp_path: Path) -> Non target = tmp_path / "data.bin" target.write_bytes(b"payload") register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"{target}|wrap:", mode="r") + result = await resolve_pipeline(f"file:{target.as_posix()}|wrap:", mode="r") assert isinstance(result.store, TracingStore) assert result.store.read_only @@ -496,7 +496,7 @@ async def test_wrapper_on_local_file_root_writable_modes( target = tmp_path / "data.bin" target.write_bytes(b"payload") register_url_adapter("wrap", WrapperAdapter) - await resolve_pipeline(f"{target}|wrap:", mode=mode) + await resolve_pipeline(f"file:{target.as_posix()}|wrap:", mode=mode) async def test_resolve_preceding_storage_options_override(self, tmp_path: Path) -> None: # an adapter that consumed its namespaced keys strips them before @@ -517,7 +517,8 @@ async def open_pipeline_segment( # the local-path root rejects any surviving storage_options, so this # passing proves the adapter's keys were stripped before resolution result = await resolve_pipeline( - f"{tmp_path}|consuming:x", storage_options={"consuming_secret": "s3cr3t"} + f"file:{tmp_path.as_posix()}|consuming:x", + storage_options={"consuming_secret": "s3cr3t"}, ) assert result.path == "x" assert ConsumingWrapper.seen == "s3cr3t" @@ -555,14 +556,14 @@ async def test_single_url_is_a_trivial_pipeline(self) -> None: async def test_accepts_pipeline_object(self, tmp_path: Path) -> None: """`resolve_pipeline` takes a parsed `URLPipeline` as well as a string.""" register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(URLPipeline.from_url(f"{tmp_path}|wrap:a")) + result = await resolve_pipeline(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:a")) assert isinstance(result.store, TracingStore) assert result.path == "a" async def test_format_segment_path_joins_residual_path(self, tmp_path: Path) -> None: """The node path of a trailing format segment extends the adapters' residual path.""" register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"{tmp_path}|wrap:a|zarr3:b") + result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:a|zarr3:b") assert result.path == "a/b" async def test_unknown_adapter_scheme_raises(self) -> None: @@ -692,7 +693,7 @@ class TestMakeStoreIntegration: async def test_make_store_path_combines_paths(self, tmp_path: Path) -> None: """Adapter residual path, format-segment node path and the caller's path join in order.""" register_url_adapter("wrap", WrapperAdapter) - pipeline = URLPipeline.from_url(f"{tmp_path}|wrap:residual|zarr3:node") + pipeline = URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:residual|zarr3:node") store_path = await make_store_path(pipeline, path="user/sub") assert store_path.path == "residual/node/user/sub" assert isinstance(store_path.store, TracingStore) @@ -700,13 +701,13 @@ async def test_make_store_path_combines_paths(self, tmp_path: Path) -> None: async def test_make_store_rejects_residual_path(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): - await make_store(URLPipeline.from_url(f"{tmp_path}|wrap:residual")) + await make_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:residual")) async def test_make_store_rejects_format_segment_node_path(self, tmp_path: Path) -> None: """A node path given on the format segment also addresses a path inside the store.""" register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): - await make_store(URLPipeline.from_url(f"{tmp_path}|wrap:|zarr3:node")) + await make_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:|zarr3:node")) async def test_make_store_rejects_slash_root_path(self, tmp_path: Path) -> None: # an adapter returning path="/" addresses the store root; the raw @@ -720,7 +721,7 @@ async def open_pipeline_segment( return dataclasses.replace(preceding, path="/") register_url_adapter("slashy", SlashRootAdapter) - store = await make_store(URLPipeline.from_url(f"{tmp_path}|slashy:")) + store = await make_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|slashy:")) assert store is not None async def test_make_store_closes_store_on_residual_path_error(self) -> None: @@ -745,7 +746,7 @@ async def open_pipeline_segment( async def test_make_store_no_residual_path(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - store = await make_store(URLPipeline.from_url(f"{tmp_path}|wrap:")) + store = await make_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:")) assert isinstance(store, TracingStore) async def test_make_store_rejects_invalid_mode_before_resolving(self, tmp_path: Path) -> None: @@ -753,30 +754,28 @@ async def test_make_store_rejects_invalid_mode_before_resolving(self, tmp_path: # rather than being handed to adapters register_url_adapter("wrap", WrapperAdapter) with pytest.raises(ValueError, match="Invalid mode"): - await make_store(URLPipeline.from_url(f"{tmp_path}|wrap:"), mode="invalid") # type: ignore[arg-type] + await make_store( + URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:"), + mode="invalid", # type: ignore[arg-type] + ) async def test_zarr_open_end_to_end(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - group = zarr.open_group(URLPipeline.from_url(f"{tmp_path}|wrap:"), mode="w") + group = zarr.open_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:"), mode="w") array = group.create_array("x", shape=(4,), dtype="i4") array[:] = [1, 2, 3, 4] - assert zarr.open_array(URLPipeline.from_url(f"{tmp_path}|wrap:x"))[2] == 3 + assert zarr.open_array(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:x"))[2] == 3 async def test_pipeline_store_read_only_mode(self) -> None: register_url_adapter("native", NativeAdapter) store_path = await make_store_path(URLPipeline.from_url("memory://base|native:"), mode="r") assert store_path.read_only - async def test_mode_a_downgrades_to_read_only_open(self) -> None: - # mode "a" (the zarr.open default) is open-or-create: a pipeline - # that resolves to a read-only store serves the "open" half instead - # of failing outright. + async def test_mode_a_on_read_only_store_raises(self) -> None: + """Mode "a" on a read-only store raises, whether the store comes from a pipeline or not.""" register_url_adapter("rooty", RootAdapter) - store_path = await make_store_path(URLPipeline.from_url("rooty://org/repo"), mode="a") - assert store_path.read_only - - async def test_mode_a_on_plain_read_only_store_still_raises(self) -> None: - """The open-or-create downgrade is specific to pipelines; plain stores behave as before.""" + with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"): + await make_store_path(URLPipeline.from_url("rooty://org/repo"), mode="a") store = await MemoryStore.open(read_only=True) with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"): await make_store_path(store, mode="a") @@ -793,24 +792,32 @@ async def test_storage_options_forwarded_to_root(self) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(TypeError, match="'storage_options' was provided but unused"): await make_store( - URLPipeline.from_url("/tmp/some/path|wrap:"), storage_options={"anon": True} + URLPipeline.from_url("file:/tmp/some/path|wrap:"), storage_options={"anon": True} ) async def test_store_path_has_no_format(self, tmp_path: Path) -> None: """The pipeline's format is consumed by the open call; it does not linger on the StorePath.""" - store_path = await make_store_path(URLPipeline.from_url(f"{tmp_path}|zarr2:")) + store_path = await make_store_path( + URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:") + ) assert not hasattr(store_path, "zarr_format") async def test_zarr_format_merges_into_open(self, tmp_path: Path) -> None: - group = zarr.open_group(URLPipeline.from_url(f"{tmp_path}|zarr2:"), mode="w") + group = zarr.open_group( + URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:"), mode="w" + ) assert group.metadata.zarr_format == 2 async def test_conflicting_explicit_format_raises(self, tmp_path: Path) -> None: with pytest.raises(ValueError, match="conflicts with"): - zarr.open_group(URLPipeline.from_url(f"{tmp_path}|zarr2:"), mode="w", zarr_format=3) + zarr.open_group( + URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:"), mode="w", zarr_format=3 + ) async def test_matching_explicit_format_ok(self, tmp_path: Path) -> None: - group = zarr.open_group(URLPipeline.from_url(f"{tmp_path}|zarr2:"), mode="w", zarr_format=2) + group = zarr.open_group( + URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:"), mode="w", zarr_format=2 + ) assert group.metadata.zarr_format == 2 @@ -823,8 +830,8 @@ class TestOpenURL: async def test_open_url_opens_existing_node_with_pipeline_format(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - zarr.create_group(URLPipeline.from_url(f"{tmp_path}|wrap:|zarr2:")) - group = zarr.open_url(f"{tmp_path}|wrap:|zarr2:") + zarr.create_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:|zarr2:")) + group = zarr.open_url(f"file:{tmp_path.as_posix()}|wrap:|zarr2:") assert isinstance(group, zarr.Group) assert group.metadata.zarr_format == 2 assert isinstance(group.store, TracingStore) @@ -834,34 +841,37 @@ async def test_open_url_format_segment_body_addresses_node(self, tmp_path: Path) """The node path lives in the URL, as the body of the format segment.""" register_url_adapter("wrap", WrapperAdapter) zarr.create_array( - URLPipeline.from_url(f"{tmp_path}|wrap:"), name="sub/x", shape=(3,), dtype="i4" + URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:"), + name="sub/x", + shape=(3,), + dtype="i4", ) - array = zarr.open_url(f"{tmp_path}|wrap:|zarr3:sub/x") + array = zarr.open_url(f"file:{tmp_path.as_posix()}|wrap:|zarr3:sub/x") assert isinstance(array, zarr.Array) assert array.store_path.path == "sub/x" async def test_open_url_read_write(self, tmp_path: Path) -> None: - zarr.create_group(URLPipeline.from_url(f"{tmp_path}|zarr3:")) - group = zarr.open_url(f"{tmp_path}|zarr3:", mode="r+") + zarr.create_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr3:")) + group = zarr.open_url(f"file:{tmp_path.as_posix()}|zarr3:", mode="r+") assert not group.read_only group.attrs["answer"] = 42 - assert zarr.open_url(f"{tmp_path}|zarr3:").attrs["answer"] == 42 + assert zarr.open_url(f"file:{tmp_path.as_posix()}|zarr3:").attrs["answer"] == 42 async def test_open_url_missing_node_raises(self, tmp_path: Path) -> None: """Nothing is created on open: a URL to a missing node is an error in every mode.""" with pytest.raises(FileNotFoundError): - zarr.open_url(f"{tmp_path}/missing|zarr3:") + zarr.open_url(f"file:{tmp_path.as_posix()}/missing|zarr3:") with pytest.raises(FileNotFoundError): - zarr.open_url(f"{tmp_path}/missing|zarr3:", mode="r+") + zarr.open_url(f"file:{tmp_path.as_posix()}/missing|zarr3:", mode="r+") async def test_open_url_rejects_create_modes(self, tmp_path: Path) -> None: """The creating modes of `zarr.open` are not accepted.""" with pytest.raises(ValueError, match="opens existing nodes only"): - zarr.open_url(f"{tmp_path}|zarr3:", mode="a") # type: ignore[arg-type] + zarr.open_url(f"file:{tmp_path.as_posix()}|zarr3:", mode="a") # type: ignore[arg-type] async def test_async_open_url(self, tmp_path: Path) -> None: - zarr.create_group(URLPipeline.from_url(f"{tmp_path}|zarr2:")) - group = await zarr.api.asynchronous.open_url(f"{tmp_path}|zarr2:") + zarr.create_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:")) + group = await zarr.api.asynchronous.open_url(f"file:{tmp_path.as_posix()}|zarr2:") assert group.metadata.zarr_format == 2 @@ -875,7 +885,7 @@ class TestZarrFormatMergeInCore: @pytest.fixture def v2_pipeline(self, tmp_path: Path) -> URLPipeline: - return URLPipeline.from_url(f"{tmp_path}|zarr2:") + return URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:") async def test_create_array(self, v2_pipeline: URLPipeline) -> None: arr = zarr.create_array(v2_pipeline, name="x", shape=(2,), dtype="i4", zarr_format=None) @@ -904,7 +914,7 @@ async def test_group_from_store(self, v2_pipeline: URLPipeline, tmp_path: Path) group = zarr.Group.from_store(v2_pipeline, zarr_format=None) assert group.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): - zarr.Group.from_store(URLPipeline.from_url(f"{tmp_path}/other|zarr2:")) + zarr.Group.from_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}/other|zarr2:")) async def test_group_from_store_default_without_pipeline(self, tmp_path: Path) -> None: """The default format for plain stores is still 3, independent of the config default.""" @@ -921,7 +931,7 @@ async def test_group_open(self, v2_pipeline: URLPipeline) -> None: async def test_array_open(self, v2_pipeline: URLPipeline, tmp_path: Path) -> None: zarr.create_array(v2_pipeline, name="x", shape=(2,), dtype="i4", zarr_format=None) - array_pipeline = URLPipeline.from_url(f"{tmp_path}|zarr2:x") + array_pipeline = URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:x") arr = zarr.Array.open(array_pipeline, zarr_format=None) assert arr.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): @@ -943,5 +953,8 @@ async def test_deprecated_async_array_create( assert arr.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): await AsyncArray._create( - URLPipeline.from_url(f"{tmp_path}/o|zarr2:"), shape=(2,), dtype="i4", zarr_format=3 + URLPipeline.from_url(f"file:{tmp_path.as_posix()}/o|zarr2:"), + shape=(2,), + dtype="i4", + zarr_format=3, ) From 7cee87ba79f0ccd68c6d4e37bf34bc66e4e75658 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 6 Oct 2026 14:18:35 +0200 Subject: [PATCH 11/20] refactor(url-pipeline): resolve through URLPipeline.resolve; simplify root scheme detection `URLPipeline.resolve(mode=..., storage_options=...)` replaces the module-level `resolve_pipeline`, and `parse_pipeline` leaves the public `zarr.storage` surface: the object is the one entry point for parsing and resolving a pipeline. Root scheme detection no longer consults `parse_store_url` for single-letter candidates. A letter followed by `:` is a URL scheme on every platform, so parsing is deterministic across operating systems; a Windows drive path is spelled `file:/C:/...` in a pipeline, as the root must carry a scheme anyway. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- docs/user-guide/extending.md | 5 +- src/zarr/storage/__init__.py | 4 +- src/zarr/storage/_common.py | 6 +- src/zarr/storage/_url_pipeline.py | 118 +++++++++++------------ tests/test_url_pipeline/test_parser.py | 19 ++-- tests/test_url_pipeline/test_resolver.py | 76 ++++++++------- 6 files changed, 106 insertions(+), 122 deletions(-) diff --git a/docs/user-guide/extending.md b/docs/user-guide/extending.md index 5138ba3a10..34128ce52a 100644 --- a/docs/user-guide/extending.md +++ b/docs/user-guide/extending.md @@ -126,10 +126,11 @@ inner adapter sees the outer one's residual path `a` and the pipeline resolves t import asyncio from zarr.registry import register_url_adapter -from zarr.storage import resolve_pipeline +from zarr.storage import URLPipeline register_url_adapter("mypackage.myscheme", MyAdapter) -resolution = asyncio.run(resolve_pipeline("memory://x|mypackage.myscheme:a|mypackage.myscheme:b")) +pipeline = URLPipeline.from_url("memory://x|mypackage.myscheme:a|mypackage.myscheme:b") +resolution = asyncio.run(pipeline.resolve()) assert resolution.path == "a/b" ``` diff --git a/src/zarr/storage/__init__.py b/src/zarr/storage/__init__.py index 66328d5a32..c4d8d534ac 100644 --- a/src/zarr/storage/__init__.py +++ b/src/zarr/storage/__init__.py @@ -10,7 +10,7 @@ from zarr.storage._logging import LoggingStore from zarr.storage._memory import GpuMemoryStore, ManagedMemoryStore, MemoryStore from zarr.storage._obstore import ObjectStore -from zarr.storage._url_pipeline import URLPipeline, parse_pipeline, resolve_pipeline +from zarr.storage._url_pipeline import URLPipeline from zarr.storage._wrapper import WrapperStore from zarr.storage._zip import ZipStore @@ -27,8 +27,6 @@ "URLPipeline", "WrapperStore", "ZipStore", - "parse_pipeline", - "resolve_pipeline", ] diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index 24d0dd4122..00d8f5b4f5 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -29,7 +29,7 @@ ) from zarr.storage._local import LocalStore from zarr.storage._memory import ManagedMemoryStore, MemoryStore -from zarr.storage._url_pipeline import URLPipeline, resolve_pipeline +from zarr.storage._url_pipeline import URLPipeline from zarr.storage._utils import UPath, _join_paths, normalize_path, parse_store_url _has_fsspec = importlib.util.find_spec("fsspec") @@ -360,7 +360,7 @@ async def make_store( _read_only = mode == "r" if isinstance(store_like, URLPipeline): - result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options) + result = await store_like.resolve(mode=mode, storage_options=storage_options) if normalize_path(result.path): result.store.close() raise URLPipelineError( @@ -486,7 +486,7 @@ async def make_store_path( path_normalized = normalize_path(path) if isinstance(store_like, URLPipeline): - result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options) + result = await store_like.resolve(mode=mode, storage_options=storage_options) combined_path = _join_paths([normalize_path(result.path), path_normalized]) return await StorePath.open(result.store, path=combined_path, mode=mode) diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py index a040c61441..e53fc4f04d 100644 --- a/src/zarr/storage/_url_pipeline.py +++ b/src/zarr/storage/_url_pipeline.py @@ -29,13 +29,13 @@ from zarr.errors import URLPipelineError from zarr.registry import get_url_adapter, list_url_adapter_schemes from zarr.storage._memory import ManagedMemoryStore -from zarr.storage._utils import _join_paths, normalize_path, parse_store_url +from zarr.storage._utils import _join_paths, normalize_path if TYPE_CHECKING: from zarr.abc.store import Store from zarr.core.common import AccessModeLiteral, ZarrFormat -__all__ = ["URLPipeline", "parse_pipeline", "resolve_pipeline"] +__all__ = ["URLPipeline"] # Format segments are consumed by zarr-python itself rather than by a # registered adapter: they select the zarr format of the node the pipeline @@ -64,20 +64,14 @@ def _root_scheme(root_sub_url: str) -> str: Detect the scheme of the root sub-URL, or `""` when it has none. Scheme extraction happens on the raw string, so nothing `urlparse` - would strip or reject can desync the detected scheme from the body. - Single-letter candidates are delegated to `parse_store_url`, which - knows Windows drive letters are not schemes. + would strip or reject can desync the detected scheme from the body. A + single letter followed by `:` is a scheme on every platform: a Windows + drive path is spelled `file:/C:/...` in a pipeline, never bare. """ match = _ROOT_SCHEME_RE.match(root_sub_url) if match is None: return "" - scheme = match.group(1) - if len(scheme) == 1: - try: - return parse_store_url(root_sub_url).scheme.lower() - except ValueError: - return scheme.lower() - return scheme.lower() + return match.group(1).lower() def _split_query(sub_url: str) -> tuple[str, str | None]: @@ -95,9 +89,7 @@ def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]: """ Parse a URL pipeline into its `|`-delimited segments. - The first segment is the *root* sub-URL; its scheme is detected with the - same rules as ordinary store URLs (Windows drive letters are not - schemes). Subsequent segments are *adapter* sub-URLs of the form + The first segment is the *root* sub-URL. Subsequent segments are *adapter* sub-URLs of the form `scheme:body` where the trailing colon is optional when the body is empty (`zip` is equivalent to `zip:`). @@ -142,7 +134,9 @@ class URLPipeline: """ A parsed URL pipeline, ready to be passed wherever a `StoreLike` is accepted. - Construct one with [`from_url`][zarr.storage.URLPipeline.from_url]. Because a + Construct one with [`from_url`][zarr.storage.URLPipeline.from_url] and open it + through any `StoreLike`-taking function, or resolve it directly with + [`resolve`][zarr.storage.URLPipeline.resolve]. Because a `URLPipeline` is an explicit object, no plain string store specification is ever interpreted as a pipeline: `zarr.open("data|x.zarr")` still addresses a local directory of that name, while @@ -267,6 +261,50 @@ def resolve_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat | No ) return pipeline_format + async def resolve( + self, + *, + mode: AccessModeLiteral | None = None, + storage_options: dict[str, Any] | None = None, + ) -> AdapterResolution: + """ + Resolve the pipeline into a store and a residual path. + + The residual path joins the path returned by the adapters with the + node path of a trailing format segment (`...|zarr3:path/to/node`). + The format itself is not part of the resolution; read it from + `zarr_format`. + + Parameters + ---------- + mode : AccessModeLiteral | None + The caller's access mode. `"r"` requires a read-only store; the + resolver enforces this on whatever the final adapter returns. + storage_options : dict | None + Options forwarded to the root sub-URL's store (and visible to + adapters via the context). + + Raises + ------ + URLPipelineError + If a segment names a scheme with no registered adapter, names an + adapter entry point that fails to import, or the root sub-URL + cannot be resolved. + TypeError + If `storage_options` are passed to a root that does not accept + them (`file:` and `memory:`), as for non-pipeline stores. + OSError + Errors from opening the root resource (e.g. a missing local + directory in mode `"r"`) propagate unchanged, as for non-pipeline + stores. + """ + resolution = await _resolve(self.store_segments, mode=mode, storage_options=storage_options) + if self.path: + resolution = dataclasses.replace( + resolution, path=_join_paths([normalize_path(resolution.path), self.path]) + ) + return resolution + def __str__(self) -> str: return "|".join(segment.raw for segment in self.segments) @@ -281,54 +319,6 @@ def _root_routes_to_adapter(scheme: str) -> bool: return scheme not in _NATIVE_ROOT_SCHEMES and scheme in list_url_adapter_schemes() -async def resolve_pipeline( - url: str | URLPipeline, - *, - mode: AccessModeLiteral | None = None, - storage_options: dict[str, Any] | None = None, -) -> AdapterResolution: - """ - Resolve a URL pipeline into a store and a residual path. - - The residual path joins the path returned by the adapters with the node - path of a trailing format segment (`...|zarr3:path/to/node`). The format - itself is not part of the resolution; read it from - [`URLPipeline.zarr_format`][zarr.storage.URLPipeline]. - - Parameters - ---------- - url : str | URLPipeline - A URL pipeline, e.g. `"s3://bucket/data.zip|zip:|zarr3:"`, or its - parsed form. A URL without `|` is a trivial pipeline consisting of - its root alone. - mode : AccessModeLiteral | None - The caller's access mode. `"r"` requires a read-only store; the - resolver enforces this on whatever the final adapter returns. - storage_options : dict | None - Options forwarded to the root sub-URL's store (and visible to - adapters via the context). - - Raises - ------ - URLPipelineError - If the URL cannot be parsed, names a scheme with no registered adapter, names an adapter entry point that fails to - import, or has a root sub-URL that cannot be resolved. - TypeError - If `storage_options` are passed to a root that does not accept them - (local paths and `memory:`), as for non-pipeline stores. - OSError - Errors from opening the root resource (e.g. a missing local directory - in mode `"r"`) propagate unchanged, as for non-pipeline stores. - """ - pipeline = URLPipeline.from_url(url) if isinstance(url, str) else url - resolution = await _resolve(pipeline.store_segments, mode=mode, storage_options=storage_options) - if pipeline.path: - resolution = dataclasses.replace( - resolution, path=_join_paths([normalize_path(resolution.path), pipeline.path]) - ) - return resolution - - async def _resolve( segments: tuple[PipelineSegment, ...], *, diff --git a/tests/test_url_pipeline/test_parser.py b/tests/test_url_pipeline/test_parser.py index 3dc2480ca8..bcc2dd3283 100644 --- a/tests/test_url_pipeline/test_parser.py +++ b/tests/test_url_pipeline/test_parser.py @@ -1,7 +1,5 @@ from __future__ import annotations -import sys - import pytest from hypothesis import given from hypothesis import strategies as st @@ -74,16 +72,12 @@ def test_empty_query() -> None: assert segment.query == "" -def test_windows_drive_path_is_not_a_scheme() -> None: - # On Windows parse_store_url treats C:\... as a local path; elsewhere the - # single-letter scheme is preserved but must not crash the parser. +def test_single_letter_scheme_is_a_scheme_everywhere() -> None: + """A drive-letter-like prefix is a URL scheme on every platform; bare Windows paths are not pipelines.""" (segment,) = parse_pipeline(r"C:\data\store") + assert segment.scheme == "c" + assert segment.body == r"\data\store" assert segment.raw == r"C:\data\store" - if sys.platform == "win32": - assert segment.scheme == "" - assert segment.body == r"C:\data\store" - else: - assert segment.scheme == "c" @pytest.mark.parametrize("url", ["a||b:", "|zip:", "file:/tmp|", ""]) @@ -104,10 +98,9 @@ def test_invalid_adapter_scheme_rejected(segment: str) -> None: def test_single_letter_scheme_with_exotic_authority() -> None: - # single-letter candidates are delegated to parse_store_url, which may - # reject an exotic authority; the raw scheme is used as the fallback + """Scheme detection works on the raw text, so an authority urlparse would reject is no obstacle.""" segments = parse_pipeline("x://[authority]/path|zip:") - assert segments[0].scheme in ("", "x") + assert segments[0].scheme == "x" assert segments[0].raw == "x://[authority]/path" diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index e00c29cb90..ecfa37a37f 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -29,7 +29,6 @@ ) from zarr.storage import LocalStore, ManagedMemoryStore, MemoryStore, URLPipeline, WrapperStore from zarr.storage._common import _has_fsspec, make_store, make_store_path -from zarr.storage._url_pipeline import resolve_pipeline from zarr.storage._utils import _join_paths # fsspec < 2024.12.0 has no AsyncFileSystemWrapper, so a plain memory:// URL @@ -140,7 +139,9 @@ def test_entrypoint_discovery(self) -> None: @pytest.mark.usefixtures("set_path") async def test_entrypoint_end_to_end(self) -> None: - result = await resolve_pipeline("memory://src|example-pkg.entrypoint-scheme:sub/path") + result = await URLPipeline.from_url( + "memory://src|example-pkg.entrypoint-scheme:sub/path" + ).resolve() assert result.path == "sub/path" @pytest.mark.usefixtures("set_path") @@ -353,7 +354,7 @@ def test_native_schemes_do_not_route_to_adapters(self) -> None: class TestResolve: async def test_wrapper_adapter_chain(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:inner/path") + result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:inner/path").resolve() assert isinstance(result.store, TracingStore) assert result.path == "inner/path" assert result.store.context.preceding_url == f"file:{tmp_path.as_posix()}" @@ -364,25 +365,25 @@ async def test_nested_wrappers_preserve_residual_paths(self, tmp_path: Path) -> # each wrapper joins the preceding residual path with its own, so # no segment's path is lost in root|wrap:a|wrap:b register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:a|wrap:b") + result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:a|wrap:b").resolve() assert result.path == "a/b" async def test_native_adapter_gets_preceding_url(self) -> None: register_url_adapter("native", NativeAdapter) NativeAdapter.seen_urls.clear() - await resolve_pipeline("s3://bucket/repo|native:") + await URLPipeline.from_url("s3://bucket/repo|native:").resolve() assert NativeAdapter.seen_urls == ["s3://bucket/repo"] async def test_multi_segment_preceding_url(self) -> None: register_url_adapter("wrap", WrapperAdapter) register_url_adapter("native", NativeAdapter) NativeAdapter.seen_urls.clear() - await resolve_pipeline("memory://base|wrap:a|native:x") + await URLPipeline.from_url("memory://base|wrap:a|native:x").resolve() assert NativeAdapter.seen_urls == ["memory://base|wrap:a"] async def test_root_adapter(self) -> None: register_url_adapter("rooty", RootAdapter) - result = await resolve_pipeline("rooty://org/repo") + result = await URLPipeline.from_url("rooty://org/repo").resolve() assert result.path == "org/repo" assert result.store.read_only @@ -390,16 +391,16 @@ async def test_root_adapter_composes_with_chain(self) -> None: register_url_adapter("rooty", RootAdapter) register_url_adapter("native", NativeAdapter) NativeAdapter.seen_urls.clear() - await resolve_pipeline("rooty://org/repo|native:x") + await URLPipeline.from_url("rooty://org/repo|native:x").resolve() assert NativeAdapter.seen_urls == ["rooty://org/repo"] async def test_read_only_flag(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:", mode="r") + result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:").resolve(mode="r") assert isinstance(result.store, TracingStore) assert result.store.context.read_only assert result.store.context.mode == "r" - result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:") + result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:").resolve() assert isinstance(result.store, TracingStore) assert not result.store.context.read_only assert result.store.context.mode is None @@ -408,7 +409,7 @@ async def test_read_only_is_enforced_on_disobedient_adapters(self) -> None: # the resolver downgrades a writable store returned under mode "r" # instead of trusting the adapter to have honored context.read_only register_url_adapter("bad", DisobedientAdapter) - result = await resolve_pipeline("memory://base|bad:", mode="r") + result = await URLPipeline.from_url("memory://base|bad:").resolve(mode="r") assert result.store.read_only store = await make_store(URLPipeline.from_url("memory://base|bad:"), mode="r") assert store.read_only @@ -428,7 +429,7 @@ async def open_pipeline_segment( register_url_adapter("stubborn", StubbornAdapter) with pytest.raises(URLPipelineError, match="does not support read-only conversion"): - await resolve_pipeline("memory://base|stubborn:", mode="r") + await URLPipeline.from_url("memory://base|stubborn:").resolve(mode="r") async def test_read_only_enforcement_failure_closes_store(self) -> None: # the adapter's store must not leak when it cannot be made read-only @@ -451,7 +452,7 @@ async def open_pipeline_segment( register_url_adapter("stubborn", StubbornAdapter) with pytest.raises(URLPipelineError): - await resolve_pipeline("memory://base|stubborn:", mode="r") + await URLPipeline.from_url("memory://base|stubborn:").resolve(mode="r") assert closed == [True] async def test_resolve_preceding_mode_override(self, tmp_path: Path) -> None: @@ -467,7 +468,9 @@ async def open_pipeline_segment( return dataclasses.replace(preceding, path=segment.body) register_url_adapter("rowrap", ReadOnlyRootWrapper) - result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|rowrap:x", mode="w") + result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|rowrap:x").resolve( + mode="w" + ) assert result.path == "x" async def test_wrapper_on_local_file_root_read_only(self, tmp_path: Path) -> None: @@ -477,7 +480,7 @@ async def test_wrapper_on_local_file_root_read_only(self, tmp_path: Path) -> Non target = tmp_path / "data.bin" target.write_bytes(b"payload") register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"file:{target.as_posix()}|wrap:", mode="r") + result = await URLPipeline.from_url(f"file:{target.as_posix()}|wrap:").resolve(mode="r") assert isinstance(result.store, TracingStore) assert result.store.read_only @@ -496,7 +499,7 @@ async def test_wrapper_on_local_file_root_writable_modes( target = tmp_path / "data.bin" target.write_bytes(b"payload") register_url_adapter("wrap", WrapperAdapter) - await resolve_pipeline(f"file:{target.as_posix()}|wrap:", mode=mode) + await URLPipeline.from_url(f"file:{target.as_posix()}|wrap:").resolve(mode=mode) async def test_resolve_preceding_storage_options_override(self, tmp_path: Path) -> None: # an adapter that consumed its namespaced keys strips them before @@ -516,8 +519,7 @@ async def open_pipeline_segment( register_url_adapter("consuming", ConsumingWrapper) # the local-path root rejects any surviving storage_options, so this # passing proves the adapter's keys were stripped before resolution - result = await resolve_pipeline( - f"file:{tmp_path.as_posix()}|consuming:x", + result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|consuming:x").resolve( storage_options={"consuming_secret": "s3cr3t"}, ) assert result.path == "x" @@ -538,44 +540,44 @@ async def open_pipeline_segment( register_url_adapter("probe", OptionsProbe) opts = {"anon": True} - await resolve_pipeline("s3://bucket/x|probe:", storage_options=opts) + await URLPipeline.from_url("s3://bucket/x|probe:").resolve(storage_options=opts) assert OptionsProbe.seen_options == opts async def test_wrapper_at_root_position_raises(self) -> None: # a wrapper adapter used as the pipeline root has nothing to wrap register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="no preceding sub-URL"): - await resolve_pipeline("wrap:whatever") + await URLPipeline.from_url("wrap:whatever").resolve() async def test_single_url_is_a_trivial_pipeline(self) -> None: """A URL without `|` resolves as a pipeline consisting of its root alone.""" - result = await resolve_pipeline("memory://plain") + result = await URLPipeline.from_url("memory://plain").resolve() assert isinstance(result.store, ManagedMemoryStore) assert result.path == "" - async def test_accepts_pipeline_object(self, tmp_path: Path) -> None: - """`resolve_pipeline` takes a parsed `URLPipeline` as well as a string.""" + async def test_resolve_method(self, tmp_path: Path) -> None: + """`URLPipeline.resolve` yields the adapter's store and residual path.""" register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:a")) + result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:a").resolve() assert isinstance(result.store, TracingStore) assert result.path == "a" async def test_format_segment_path_joins_residual_path(self, tmp_path: Path) -> None: """The node path of a trailing format segment extends the adapters' residual path.""" register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:a|zarr3:b") + result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:a|zarr3:b").resolve() assert result.path == "a/b" async def test_unknown_adapter_scheme_raises(self) -> None: with pytest.raises(URLPipelineError, match="no URL pipeline adapter is registered"): - await resolve_pipeline("memory://base|no-such-adapter:") + await URLPipeline.from_url("memory://base|no-such-adapter:").resolve() async def test_base_case_value_error_is_wrapped(self) -> None: # a root that only the fallback scheme detection accepts must not # leak urlparse's ValueError out of the base case register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="could not resolve the pipeline root"): - await resolve_pipeline("vendor.x://[authority]/path|wrap:") + await URLPipeline.from_url("vendor.x://[authority]/path|wrap:").resolve() class TestSpecRoots: @@ -584,7 +586,7 @@ class TestSpecRoots: @pytest.mark.parametrize("root", ["memory:", "memory:/", "memory://"]) async def test_memory_root_spellings_equivalent(self, root: str) -> None: register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"{root}|wrap:") + result = await URLPipeline.from_url(f"{root}|wrap:").resolve() assert isinstance(result.store, TracingStore) inner = result.store._store assert isinstance(inner, ManagedMemoryStore) @@ -594,7 +596,7 @@ async def test_memory_root_spellings_equivalent(self, root: str) -> None: @pytest.mark.parametrize("root", ["memory:a/b", "memory:/a/b", "memory://a/b"]) async def test_memory_root_named_spellings_equivalent(self, root: str) -> None: register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"{root}|wrap:") + result = await URLPipeline.from_url(f"{root}|wrap:").resolve() assert isinstance(result.store, TracingStore) inner = result.store._store assert isinstance(inner, ManagedMemoryStore) @@ -611,12 +613,12 @@ async def test_memory_root_shares_data_across_spellings(self) -> None: async def test_memory_root_rejects_query(self) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="do not accept a query"): - await resolve_pipeline("memory:a?opt=1|wrap:") + await URLPipeline.from_url("memory:a?opt=1|wrap:").resolve() async def test_memory_root_rejects_storage_options(self) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(TypeError, match="'storage_options' was provided but unused"): - await resolve_pipeline("memory:a|wrap:", storage_options={"anon": True}) + await URLPipeline.from_url("memory:a|wrap:").resolve(storage_options={"anon": True}) async def test_file_root_localhost_authority(self, tmp_path: Path) -> None: # file://localhost/p and file:///p are equivalent to file:/p @@ -642,26 +644,26 @@ async def test_file_root_windows_drive_spellings( # chdir into tmp so it is created there monkeypatch.chdir(tmp_path) drive_path = "C:/drive/data" - r1 = await resolve_pipeline(f"file:{drive_path}|wrap:") - r2 = await resolve_pipeline(f"file:///{drive_path}|wrap:") + r1 = await URLPipeline.from_url(f"file:{drive_path}|wrap:").resolve() + r2 = await URLPipeline.from_url(f"file:///{drive_path}|wrap:").resolve() assert isinstance(r1.store, TracingStore) assert isinstance(r2.store, TracingStore) async def test_file_root_rejects_other_authority(self) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="unsupported authority"): - await resolve_pipeline("file://example.com/tmp/x|wrap:") + await URLPipeline.from_url("file://example.com/tmp/x|wrap:").resolve() @pytest.mark.parametrize("root", ["file:relative/path", "file://localhost"]) async def test_file_root_rejects_relative_paths(self, root: str) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="absolute path"): - await resolve_pipeline(f"{root}|wrap:") + await URLPipeline.from_url(f"{root}|wrap:").resolve() async def test_file_root_rejects_query(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="do not accept a query"): - await resolve_pipeline(f"file:{tmp_path}?v=1|wrap:") + await URLPipeline.from_url(f"file:{tmp_path}?v=1|wrap:").resolve() @pytest.mark.skipif( not _has_async_fsspec_wrapper, @@ -682,7 +684,7 @@ async def test_file_root_does_not_percent_decode(self, tmp_path: Path) -> None: # documented divergence from RFC 8089: consistent with LocalStore, no # percent-escape is decoded, so %20 names a literal directory register_url_adapter("wrap", WrapperAdapter) - await resolve_pipeline(f"file:{tmp_path.as_posix()}/a%20b|wrap:") + await URLPipeline.from_url(f"file:{tmp_path.as_posix()}/a%20b|wrap:").resolve() assert (tmp_path / "a%20b").is_dir() assert not (tmp_path / "a b").exists() From 61abcbd5b60a5e62f05ec5f7bc1e15476aa9ebb6 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 17 Oct 2026 20:26:25 -0700 Subject: [PATCH 12/20] test: skip the '|'-in-local-path test on Windows Windows forbids '|' in file names, so the directory cannot be created there. Also use a test-only scheme name rather than 'zip', which a builtin adapter will claim. Co-Authored-By: Claude Opus 5.5 --- tests/test_url_pipeline/test_resolver.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index ecfa37a37f..2b19e22a46 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -316,12 +316,13 @@ class TestStringsAreNeverPipelines: are registered. """ + @pytest.mark.skipif(sys.platform == "win32", reason="'|' is not allowed in Windows file names") async def test_local_path_containing_pipe_is_a_local_store(self, tmp_path: Path) -> None: """A string path with `|` and a registered adapter's name is still a local directory.""" - register_url_adapter("zip", WrapperAdapter) - store_path = await make_store_path(f"{tmp_path}/a|zip:x", mode="w") + register_url_adapter("wrap", WrapperAdapter) + store_path = await make_store_path(f"{tmp_path}/a|wrap:x", mode="w") assert isinstance(store_path.store, LocalStore) - assert store_path.store.root == tmp_path / "a|zip:x" + assert store_path.store.root == tmp_path / "a|wrap:x" async def test_registered_root_scheme_string_is_not_routed(self) -> None: """A URL whose scheme names a registered root adapter is not handed to that adapter.""" From 3ca3e6103cf7e121e6f0834c94109ceeb723c664 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 19 Oct 2026 19:18:30 +0100 Subject: [PATCH 13/20] refactor(url-pipeline): shrink the adapter API to an ABC plus StorePath Following review on #4192: - Adapters return a StorePath instead of AdapterResolution, which had become a StorePath without the methods. AdapterResolution is removed. - zarr.abc.url_pipeline now holds only the URLPipelineAdapter ABC. PipelineSegment and PipelineContext are plain data classes and move next to the resolver; they are exported from zarr.storage. This also removes the lazy import of the resolver from zarr.abc. - PipelineContext.resolve_preceding() takes no arguments. An adapter that needs another mode or options resolves through a derived context, dataclasses.replace(context, mode="r").resolve_preceding(), so the _Unset sentinel is no longer needed. - Less public surface: URLPipeline keeps from_url and segments; its resolve(), store_segments, zarr_format, path and resolve_zarr_format become private, as do zarr.registry.get_url_adapter and list_url_adapter_schemes. register_url_adapter stays public. Co-Authored-By: Claude Opus 5.5 --- changes/4192.feature.md | 11 +- docs/user-guide/extending.md | 40 ++-- src/zarr/abc/url_pipeline.py | 213 ++--------------- src/zarr/registry.py | 8 +- src/zarr/storage/__init__.py | 4 +- src/zarr/storage/_common.py | 11 +- src/zarr/storage/_url_pipeline.py | 198 +++++++++++++--- tests/package_with_entrypoint/__init__.py | 10 +- .../test_url_pipeline/test_pipeline_object.py | 10 +- tests/test_url_pipeline/test_resolver.py | 218 ++++++++++-------- 10 files changed, 344 insertions(+), 379 deletions(-) diff --git a/changes/4192.feature.md b/changes/4192.feature.md index 21e87736a3..98b3d64996 100644 --- a/changes/4192.feature.md +++ b/changes/4192.feature.md @@ -6,8 +6,11 @@ the new `zarr.open_url(url, mode="r" | "r+")` opens the existing node a pipeline addresses without ever creating or overwriting anything. Adapters for the sub-URLs after the first are provided by third-party packages (e.g. Icechunk) through the `zarr.url_adapters` entry-point group and the single-method -`zarr.abc.url_pipeline.URLPipelineAdapter` interface; they are loaded lazily and -individually. A builtin `zip:` adapter will follow separately. +`zarr.abc.url_pipeline.URLPipelineAdapter` interface, which receives a +`zarr.storage.PipelineSegment` and `zarr.storage.PipelineContext` and returns a +`zarr.storage.StorePath`; adapters are loaded lazily and individually, and +`zarr.registry.register_url_adapter` registers one without an entry point. A builtin +`zip:` adapter will follow separately. Behavior notes: @@ -27,8 +30,8 @@ Behavior notes: not share data with the fsspec in-memory filesystem a plain `memory://name` string resolves to. - Wrapper adapters can open a local *file* root only in mode `"r"` (via - `PipelineContext.resolve_preceding(mode="r")`); writing through a container file is - not yet supported. + `dataclasses.replace(context, mode="r").resolve_preceding()`); writing through a + container file is not yet supported. - A `zarr.url_adapters` entry point whose scheme is already registered is ignored with a `ZarrUserWarning`, as are duplicate entry-point names; an entry point that fails to import raises `URLPipelineError` naming it, and stays discoverable for a retry. diff --git a/docs/user-guide/extending.md b/docs/user-guide/extending.md index 34128ce52a..5dc5d51358 100644 --- a/docs/user-guide/extending.md +++ b/docs/user-guide/extending.md @@ -91,47 +91,44 @@ See [developing custom stores](storage.md#developing-custom-stores) for more inf `s3://bucket/data.zip|zip:|zarr3:`. Each sub-URL after the first names an *adapter* that reinterprets everything to its left. A package provides an adapter for a scheme by subclassing [`zarr.abc.url_pipeline.URLPipelineAdapter`][] and implementing its single -classmethod, which resolves the adapter's segment into an open store and a residual path: +classmethod, which resolves the adapter's segment into an open store and a residual path, +returned as a [`StorePath`][zarr.storage.StorePath]: ```python test="true" session="url-adapter" -import dataclasses - -from zarr.abc.url_pipeline import ( - AdapterResolution, - PipelineContext, - PipelineSegment, - URLPipelineAdapter, -) -from zarr.storage import WrapperStore +from zarr.abc.url_pipeline import URLPipelineAdapter +from zarr.storage import PipelineContext, PipelineSegment, StorePath, WrapperStore class MyAdapter(URLPipelineAdapter): @classmethod async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext - ) -> AdapterResolution: - # A *wrapper* adapter opens the resource to its left and wraps it. Join the - # preceding residual path with this segment's own path, and keep every other - # field via dataclasses.replace. + ) -> StorePath: + # A *wrapper* adapter opens the resource to its left and wraps it, joining the + # preceding residual path with this segment's own path. preceding = await context.resolve_preceding() store = WrapperStore(preceding.store) path = "/".join(part.strip("/") for part in (preceding.path, segment.body) if part) - return dataclasses.replace(preceding, store=store, path=path) + return StorePath(store, path) ``` +To resolve the preceding pipeline with a different mode or options (for example +read-only, or without the adapter's own `storage_options` keys), resolve through a derived +context: `await dataclasses.replace(context, mode="r").resolve_preceding()`. + Wrapper adapters compose: in `memory://x|mypackage.myscheme:a|mypackage.myscheme:b` the inner adapter sees the outer one's residual path `a` and the pipeline resolves to `a/b`. ```python test="true" session="url-adapter" -import asyncio - +import zarr from zarr.registry import register_url_adapter from zarr.storage import URLPipeline register_url_adapter("mypackage.myscheme", MyAdapter) -pipeline = URLPipeline.from_url("memory://x|mypackage.myscheme:a|mypackage.myscheme:b") -resolution = asyncio.run(pipeline.resolve()) -assert resolution.path == "a/b" +url = "memory://x|mypackage.myscheme:a|mypackage.myscheme:b" +created = zarr.create_group(URLPipeline.from_url(url)) +group = zarr.open_url(url) +assert group.path == "a/b" ``` Register the class under the `zarr.url_adapters` entry-point group, using the URL @@ -149,7 +146,8 @@ which case `context.preceding` is empty; such a URL reaches the adapter only thr `zarr.open_url` or a `URLPipeline`, never as a plain string store specification. The returned store must honor `context.read_only`, and `open_pipeline_segment` runs on zarr's internal event loop, so it must not block or call zarr's synchronous API. See -[`zarr.abc.url_pipeline`][] for the full contract. +[`zarr.abc.url_pipeline`][] and [`PipelineContext`][zarr.storage.PipelineContext] for the +full contract. ## Custom array buffers diff --git a/src/zarr/abc/url_pipeline.py b/src/zarr/abc/url_pipeline.py index f0f58140c8..8280e1cca6 100644 --- a/src/zarr/abc/url_pipeline.py +++ b/src/zarr/abc/url_pipeline.py @@ -1,5 +1,5 @@ """ -Abstract base class and data model for URL pipeline adapters. +Abstract base class for URL pipeline adapters. A URL pipeline is a `|`-separated chain of sub-URLs, read outer-to-inner, as specified by https://github.com/jbms/url-pipeline. The first sub-URL (the @@ -11,208 +11,21 @@ Third-party packages provide adapters by subclassing [`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] and registering the class under the `zarr.url_adapters` entry-point group, -using the URL scheme as the entry-point name. +using the URL scheme as the entry-point name. The values an adapter receives +and returns, [`PipelineSegment`][zarr.storage.PipelineSegment], +[`PipelineContext`][zarr.storage.PipelineContext] and +[`StorePath`][zarr.storage.StorePath], live in `zarr.storage`. """ from __future__ import annotations -import enum from abc import ABC, abstractmethod -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any - -from zarr.errors import URLPipelineError +from typing import TYPE_CHECKING if TYPE_CHECKING: - from zarr.abc.store import Store - from zarr.core.common import AccessModeLiteral - -__all__ = [ - "AdapterResolution", - "PipelineContext", - "PipelineSegment", - "URLPipelineAdapter", -] - - -class _Unset(enum.Enum): - token = 0 - - -_UNSET = _Unset.token - - -@dataclass(frozen=True) -class PipelineSegment: - """ - One `|`-delimited sub-URL of a URL pipeline. - - Attributes - ---------- - scheme : str - The lowercased URL scheme. - body : str - The text after `scheme:` and before any `?`. Interpretation is - scheme-defined; it is **not** URL-normalized, so case-significant - content (e.g. icechunk snapshot IDs) is preserved. - query : str | None - The raw query string after `?`, or None. Interpretation is - scheme-defined. - raw : str - The exact original sub-URL text, preserved for lossless - reconstruction of the pipeline. - """ - - scheme: str - body: str - query: str | None - raw: str - - def __str__(self) -> str: - return self.raw - - -@dataclass(frozen=True, eq=False) -class AdapterResolution: - """ - The result of resolving a URL pipeline (or a prefix of one). - - Instances are immutable but compare and hash by identity, since a - [`Store`][zarr.abc.store.Store] is not hashable. - - Attributes - ---------- - store : Store - The resolved store. - path : str - Residual path *within* the store that the pipeline addresses - (e.g. `"path/to/node"` for `...|icechunk://tag.v1/path/to/node`). - Empty string when the pipeline addresses the store root. - - A wrapper adapter that re-wraps a preceding resolution must carry every - field it does not change forward: prefer - `dataclasses.replace(preceding, store=..., path=...)` over - reconstructing, so fields added later are never silently dropped. - Format segments (`zarr2:`/`zarr3:`) are not adapters; they are consumed - by [`URLPipeline`][zarr.storage.URLPipeline] before resolution. - """ - - store: Store - path: str = "" - - -@dataclass(frozen=True, eq=False) -class PipelineContext: - """ - Context handed to a [`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] - describing the pipeline to the left of its segment. - - Instances are immutable but compare and hash by identity. - - Attributes - ---------- - preceding : tuple[PipelineSegment, ...] - The parsed sub-URLs to the left of the adapter's segment, outer to - inner. Empty when the adapter's segment is the pipeline root. - mode : AccessModeLiteral | None - The access mode requested by the caller (e.g. `zarr.open(mode=...)`), - or None when unspecified. Adapters for read-only resources should - raise for unambiguous write modes (`"w"`, `"w-"`, `"r+"`) and - open read-only otherwise. `"a"` (the `zarr.open` default) means - open-or-create: read-only adapters serve the "open" half, and any - subsequent write fails at the store level. - storage_options : dict[str, Any] | None - Options passed by the caller. By convention these configure the - *root* sub-URL (e.g. fsspec options); adapters may consume - adapter-specific keys, and should namespace them (e.g. - `myscheme_credentials`) to avoid collisions with other segments' - backends. An adapter that consumes keys should strip them before - resolving the rest of the pipeline, by passing the reduced mapping - to [`resolve_preceding`][zarr.abc.url_pipeline.PipelineContext.resolve_preceding]. - """ - - preceding: tuple[PipelineSegment, ...] - mode: AccessModeLiteral | None - storage_options: dict[str, Any] | None - - @property - def read_only(self) -> bool: - """ - True when the caller requires a read-only store (`mode == "r"`). - - Adapters must construct their store read-only when this is set - (the resolver enforces it afterwards); when it is False, they may - construct a writable store if the underlying resource supports - writing. - """ - return self.mode == "r" - - @property - def preceding_url(self) -> str: - """ - The pipeline to the left of this segment, reconstructed exactly. - - An adapter that consumes this string instead of calling - [`resolve_preceding`][zarr.abc.url_pipeline.PipelineContext.resolve_preceding] - takes ownership of the *entire* preceding pipeline: it must - validate every preceding segment itself and raise - [`URLPipelineError`][zarr.errors.URLPipelineError] for segments it - does not understand, so that no segment is ever silently ignored. - """ - return "|".join(segment.raw for segment in self.preceding) - - async def resolve_preceding( - self, - *, - mode: AccessModeLiteral | _Unset | None = _UNSET, - storage_options: dict[str, Any] | _Unset | None = _UNSET, - ) -> AdapterResolution: - """ - Resolve the preceding pipeline into a store. - - This is the entry point for *wrapper* adapters (e.g. `zip:`) that - operate on the resource produced by the segments to their left. It - composes with any preceding adapters, because each segment is - resolved by its own adapter. Adapters backed by their own I/O - machinery (e.g. `icechunk:`) may instead consume - [`preceding_url`][zarr.abc.url_pipeline.PipelineContext.preceding_url] - and never materialize the intermediate store — subject to the - ownership contract documented there. - - Parameters - ---------- - mode : AccessModeLiteral | None, optional - Override the mode used to resolve the preceding pipeline. - Wrapper adapters that only read the preceding resource should - pass `mode="r"` so the root is opened read-only and without - create-on-open side effects, regardless of the caller's mode. - When omitted, the caller's mode is used. Note that a local - *file* root (e.g. `data.zip|zip:`) can currently only be - resolved in mode `"r"`: every other mode creates a directory at - the root and fails with `FileExistsError`, because there is no - file-resource primitive yet. Writing *through* a container file - is therefore not supported by wrapper adapters at this time. - storage_options : dict | None, optional - Override the options forwarded to the preceding pipeline. An - adapter that consumed adapter-specific keys should pass the - remaining mapping here (or None when nothing remains), so the - root store never sees keys that were not addressed to it. - When omitted, the caller's options are forwarded unchanged. - """ - from zarr.storage._url_pipeline import _resolve + from zarr.storage import PipelineContext, PipelineSegment, StorePath - if not self.preceding: - raise URLPipelineError( - "this adapter segment is at the pipeline root; " - "there is no preceding sub-URL to resolve" - ) - return await _resolve( - self.preceding, - mode=self.mode if isinstance(mode, _Unset) else mode, - storage_options=( - self.storage_options if isinstance(storage_options, _Unset) else storage_options - ), - ) +__all__ = ["URLPipelineAdapter"] class URLPipelineAdapter(ABC): @@ -241,21 +54,23 @@ class URLPipelineAdapter(ABC): @abstractmethod async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext - ) -> AdapterResolution: + ) -> StorePath: """ Resolve `segment` (in the context of the pipeline to its left) - into a store and an optional residual path within that store. + into a store and a residual path within that store. The returned store must already be open and must honor `context.read_only` (the resolver additionally enforces it by downgrading — or rejecting — a writable store when the caller - required read-only). + required read-only). The path of the returned + [`StorePath`][zarr.storage.StorePath] is the location the pipeline + addresses inside the store; it is empty for the store root. This coroutine runs on zarr's internal I/O event loop. It must not block (do I/O through async APIs or a thread executor) and must not call zarr's synchronous API (`zarr.open`, `Group.open`, or anything else that uses `zarr.core.sync.sync`) — doing so raises `SyncError`. To open the preceding pipeline, use - [`resolve_preceding`][zarr.abc.url_pipeline.PipelineContext.resolve_preceding]. + [`resolve_preceding`][zarr.storage.PipelineContext.resolve_preceding]. """ ... diff --git a/src/zarr/registry.py b/src/zarr/registry.py index 990ee51572..892e728da0 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -35,8 +35,6 @@ "get_codec_class", "get_ndbuffer_class", "get_pipeline_class", - "get_url_adapter", - "list_url_adapter_schemes", "register_buffer", "register_chunk_key_encoding", "register_codec", @@ -533,7 +531,7 @@ class for a URL scheme. _url_adapter_registry.register(cls, key) -def list_url_adapter_schemes() -> set[str]: +def _list_url_adapter_schemes() -> set[str]: """ The set of URL schemes with a registered URL pipeline adapter. @@ -549,7 +547,7 @@ def list_url_adapter_schemes() -> set[str]: _url_adapter_lock = threading.Lock() -def get_url_adapter(scheme: str) -> type[URLPipelineAdapter]: +def _get_url_adapter(scheme: str) -> type[URLPipelineAdapter]: """ Get the URL pipeline adapter class registered for `scheme`. @@ -612,7 +610,7 @@ def get_url_adapter(scheme: str) -> type[URLPipelineAdapter]: try: return _url_adapter_registry[key] except KeyError: - registered_schemes = sorted(list_url_adapter_schemes()) + registered_schemes = sorted(_list_url_adapter_schemes()) raise URLPipelineError( f"no URL pipeline adapter is registered for scheme {scheme!r}. " f"Registered schemes: {registered_schemes}. Adapters are provided by " diff --git a/src/zarr/storage/__init__.py b/src/zarr/storage/__init__.py index c4d8d534ac..11839ad9b9 100644 --- a/src/zarr/storage/__init__.py +++ b/src/zarr/storage/__init__.py @@ -10,7 +10,7 @@ from zarr.storage._logging import LoggingStore from zarr.storage._memory import GpuMemoryStore, ManagedMemoryStore, MemoryStore from zarr.storage._obstore import ObjectStore -from zarr.storage._url_pipeline import URLPipeline +from zarr.storage._url_pipeline import PipelineContext, PipelineSegment, URLPipeline from zarr.storage._wrapper import WrapperStore from zarr.storage._zip import ZipStore @@ -22,6 +22,8 @@ "ManagedMemoryStore", "MemoryStore", "ObjectStore", + "PipelineContext", + "PipelineSegment", "StoreLike", "StorePath", "URLPipeline", diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index 00d8f5b4f5..be84dd075a 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -360,7 +360,7 @@ async def make_store( _read_only = mode == "r" if isinstance(store_like, URLPipeline): - result = await store_like.resolve(mode=mode, storage_options=storage_options) + result = await store_like._resolve_store_path(mode=mode, storage_options=storage_options) if normalize_path(result.path): result.store.close() raise URLPipelineError( @@ -486,7 +486,7 @@ async def make_store_path( path_normalized = normalize_path(path) if isinstance(store_like, URLPipeline): - result = await store_like.resolve(mode=mode, storage_options=storage_options) + result = await store_like._resolve_store_path(mode=mode, storage_options=storage_options) combined_path = _join_paths([normalize_path(result.path), path_normalized]) return await StorePath.open(result.store, path=combined_path, mode=mode) @@ -526,8 +526,9 @@ def resolve_zarr_format( Merge the zarr format selected by a `URLPipeline` store specification with the format a caller asked for. - For any other `StoreLike`, `zarr_format` is returned unchanged. For a - pipeline, see [`URLPipeline.resolve_zarr_format`][zarr.storage.URLPipeline]. + For any other `StoreLike`, `zarr_format` is returned unchanged. A pipeline + with a trailing `zarr2:`/`zarr3:` segment selects that format when the + caller passes None. Raises ------ @@ -535,7 +536,7 @@ def resolve_zarr_format( If `zarr_format` is not None and differs from the pipeline's format. """ if isinstance(store_like, URLPipeline): - return store_like.resolve_zarr_format(zarr_format) + return store_like._resolve_zarr_format(zarr_format) return zarr_format diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py index e53fc4f04d..123efaeb63 100644 --- a/src/zarr/storage/_url_pipeline.py +++ b/src/zarr/storage/_url_pipeline.py @@ -21,21 +21,144 @@ import re from typing import TYPE_CHECKING, Any -from zarr.abc.url_pipeline import ( - AdapterResolution, - PipelineContext, - PipelineSegment, -) from zarr.errors import URLPipelineError -from zarr.registry import get_url_adapter, list_url_adapter_schemes +from zarr.registry import _get_url_adapter, _list_url_adapter_schemes from zarr.storage._memory import ManagedMemoryStore from zarr.storage._utils import _join_paths, normalize_path if TYPE_CHECKING: from zarr.abc.store import Store from zarr.core.common import AccessModeLiteral, ZarrFormat + from zarr.storage._common import StorePath + +__all__ = ["PipelineContext", "PipelineSegment", "URLPipeline"] + + +@dataclasses.dataclass(frozen=True) +class PipelineSegment: + """ + One `|`-delimited sub-URL of a URL pipeline. + + Attributes + ---------- + scheme : str + The lowercased URL scheme. + body : str + The text after `scheme:` and before any `?`. Interpretation is + scheme-defined; it is **not** URL-normalized, so case-significant + content (e.g. icechunk snapshot IDs) is preserved. + query : str | None + The raw query string after `?`, or None. Interpretation is + scheme-defined. + raw : str + The exact original sub-URL text, preserved for lossless + reconstruction of the pipeline. + """ + + scheme: str + body: str + query: str | None + raw: str + + def __str__(self) -> str: + return self.raw + + +@dataclasses.dataclass(frozen=True, eq=False) +class PipelineContext: + """ + Context handed to a [`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] + describing the pipeline to the left of its segment. + + Instances are immutable but compare and hash by identity. An adapter that + needs to resolve the preceding pipeline with a different mode or options + derives a new context with `dataclasses.replace` (see + [`resolve_preceding`][zarr.storage.PipelineContext.resolve_preceding]). + + Attributes + ---------- + preceding : tuple[PipelineSegment, ...] + The parsed sub-URLs to the left of the adapter's segment, outer to + inner. Empty when the adapter's segment is the pipeline root. + mode : AccessModeLiteral | None + The access mode requested by the caller (e.g. + `zarr.open_url(mode=...)`), or None when unspecified. Adapters for + read-only resources should raise for every mode that may write, and + open read-only for `"r"` and None. + storage_options : dict[str, Any] | None + Options passed by the caller. By convention these configure the + *root* sub-URL (e.g. fsspec options); adapters may consume + adapter-specific keys, and should namespace them (e.g. + `myscheme_credentials`) to avoid collisions with other segments' + backends. An adapter that consumes keys should strip them before + resolving the rest of the pipeline. + """ + + preceding: tuple[PipelineSegment, ...] + mode: AccessModeLiteral | None + storage_options: dict[str, Any] | None + + @property + def read_only(self) -> bool: + """ + True when the caller requires a read-only store (`mode == "r"`). + + Adapters must construct their store read-only when this is set + (the resolver enforces it afterwards); when it is False, they may + construct a writable store if the underlying resource supports + writing. + """ + return self.mode == "r" + + @property + def preceding_url(self) -> str: + """ + The pipeline to the left of this segment, reconstructed exactly. + + An adapter that consumes this string instead of calling + [`resolve_preceding`][zarr.storage.PipelineContext.resolve_preceding] + takes ownership of the *entire* preceding pipeline: it must + validate every preceding segment itself and raise + [`URLPipelineError`][zarr.errors.URLPipelineError] for segments it + does not understand, so that no segment is ever silently ignored. + """ + return "|".join(segment.raw for segment in self.preceding) + + async def resolve_preceding(self) -> StorePath: + """ + Resolve the preceding pipeline into a store and a residual path. + + This is the entry point for *wrapper* adapters (e.g. `zip:`) that + operate on the resource produced by the segments to their left. It + composes with any preceding adapters, because each segment is + resolved by its own adapter. Adapters backed by their own I/O + machinery (e.g. `icechunk:`) may instead consume + [`preceding_url`][zarr.storage.PipelineContext.preceding_url] + and never materialize the intermediate store — subject to the + ownership contract documented there. + + The preceding pipeline is resolved with this context's `mode` and + `storage_options`. To use others, resolve through a derived + context: + + preceding = await dataclasses.replace( + context, mode="r", storage_options=remaining_options + ).resolve_preceding() + + Wrapper adapters that only read the preceding resource should pass + `mode="r"`, so the root is opened read-only and without create-on-open + side effects. Note that a local *file* root (e.g. `file:/data.zip|zip:`) + can currently only be resolved in mode `"r"`: every other mode creates + a directory at the root and fails with `FileExistsError`, because + there is no file-resource primitive yet. + """ + if not self.preceding: + raise URLPipelineError( + "this adapter segment is at the pipeline root; " + "there is no preceding sub-URL to resolve" + ) + return await _resolve(self.preceding, mode=self.mode, storage_options=self.storage_options) -__all__ = ["URLPipeline"] # Format segments are consumed by zarr-python itself rather than by a # registered adapter: they select the zarr format of the node the pipeline @@ -135,8 +258,7 @@ class URLPipeline: A parsed URL pipeline, ready to be passed wherever a `StoreLike` is accepted. Construct one with [`from_url`][zarr.storage.URLPipeline.from_url] and open it - through any `StoreLike`-taking function, or resolve it directly with - [`resolve`][zarr.storage.URLPipeline.resolve]. Because a + through any `StoreLike`-taking function. Because a `URLPipeline` is an explicit object, no plain string store specification is ever interpreted as a pipeline: `zarr.open("data|x.zarr")` still addresses a local directory of that name, while @@ -202,7 +324,7 @@ def _format_segment(self) -> PipelineSegment | None: return None @property - def store_segments(self) -> tuple[PipelineSegment, ...]: + def _store_segments(self) -> tuple[PipelineSegment, ...]: """ The segments that address a store: the root and any adapter segments, i.e. `segments` without a trailing format segment. @@ -212,7 +334,7 @@ def store_segments(self) -> tuple[PipelineSegment, ...]: return self.segments[:-1] @property - def zarr_format(self) -> ZarrFormat | None: + def _zarr_format(self) -> ZarrFormat | None: """The format selected by a trailing `zarr2:`/`zarr3:` segment, or None.""" format_segment = self._format_segment if format_segment is None: @@ -220,7 +342,7 @@ def zarr_format(self) -> ZarrFormat | None: return _FORMAT_SCHEMES[format_segment.scheme] @property - def path(self) -> str: + def _node_path(self) -> str: """ The node path given as the body of a trailing format segment, normalized. Empty when there is no format segment or its body is empty. @@ -230,7 +352,7 @@ def path(self) -> str: return "" return normalize_path(format_segment.body) - def resolve_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat | None: + def _resolve_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat | None: """ Combine the format selected by this pipeline with a caller-supplied one. @@ -250,7 +372,7 @@ def resolve_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat | No ValueError If the caller's format differs from the one selected by the pipeline. """ - pipeline_format = self.zarr_format + pipeline_format = self._zarr_format if pipeline_format is None: return zarr_format if zarr_format is not None and zarr_format != pipeline_format: @@ -261,19 +383,19 @@ def resolve_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat | No ) return pipeline_format - async def resolve( + async def _resolve_store_path( self, *, mode: AccessModeLiteral | None = None, storage_options: dict[str, Any] | None = None, - ) -> AdapterResolution: + ) -> StorePath: """ Resolve the pipeline into a store and a residual path. The residual path joins the path returned by the adapters with the node path of a trailing format segment (`...|zarr3:path/to/node`). - The format itself is not part of the resolution; read it from - `zarr_format`. + The format itself is not part of the result; read it from + `_zarr_format`. Parameters ---------- @@ -298,12 +420,14 @@ async def resolve( directory in mode `"r"`) propagate unchanged, as for non-pipeline stores. """ - resolution = await _resolve(self.store_segments, mode=mode, storage_options=storage_options) - if self.path: - resolution = dataclasses.replace( - resolution, path=_join_paths([normalize_path(resolution.path), self.path]) - ) - return resolution + from zarr.storage._common import StorePath # circular import + + resolved = await _resolve(self._store_segments, mode=mode, storage_options=storage_options) + if not self._node_path: + return resolved + return StorePath( + resolved.store, _join_paths([normalize_path(resolved.path), self._node_path]) + ) def __str__(self) -> str: return "|".join(segment.raw for segment in self.segments) @@ -316,7 +440,7 @@ def _root_routes_to_adapter(scheme: str) -> bool: excluded; the registry check inspects entry-point names only — no adapter code is imported here. """ - return scheme not in _NATIVE_ROOT_SCHEMES and scheme in list_url_adapter_schemes() + return scheme not in _NATIVE_ROOT_SCHEMES and scheme in _list_url_adapter_schemes() async def _resolve( @@ -324,39 +448,41 @@ async def _resolve( *, mode: AccessModeLiteral | None, storage_options: dict[str, Any] | None, -) -> AdapterResolution: +) -> StorePath: + from zarr.storage._common import StorePath # circular import + if len(segments) == 1 and not _root_routes_to_adapter(segments[0].scheme): - return AdapterResolution( - store=await _resolve_root(segments[0], mode=mode, storage_options=storage_options) + return StorePath( + await _resolve_root(segments[0], mode=mode, storage_options=storage_options) ) *preceding, last = segments - adapter_cls = get_url_adapter(last.scheme) + adapter_cls = _get_url_adapter(last.scheme) context = PipelineContext( preceding=tuple(preceding), mode=mode, storage_options=storage_options, ) - resolution = await adapter_cls.open_pipeline_segment(last, context) - if mode == "r" and not resolution.store.read_only: + resolved = await adapter_cls.open_pipeline_segment(last, context) + if mode == "r" and not resolved.store.read_only: # The caller required read-only; enforce it rather than trusting # the adapter to have honored context.read_only. try: - read_only_store = resolution.store.with_read_only(True) + read_only_store = resolved.store.with_read_only(True) await read_only_store._ensure_open() except NotImplementedError as exc: - resolution.store.close() + resolved.store.close() raise URLPipelineError( f"adapter {last.scheme!r} returned a writable store for mode 'r', " "and the store does not support read-only conversion via " ".with_read_only()" ) from exc except Exception: - resolution.store.close() + resolved.store.close() raise - resolution = dataclasses.replace(resolution, store=read_only_store) - return resolution + resolved = StorePath(read_only_store, resolved.path) + return resolved async def _resolve_root( diff --git a/tests/package_with_entrypoint/__init__.py b/tests/package_with_entrypoint/__init__.py index cad73d061d..521cb8a968 100644 --- a/tests/package_with_entrypoint/__init__.py +++ b/tests/package_with_entrypoint/__init__.py @@ -7,7 +7,7 @@ import zarr.core.buffer from zarr.abc.codec import ArrayBytesCodec, CodecInput, CodecPipeline -from zarr.abc.url_pipeline import AdapterResolution, URLPipelineAdapter +from zarr.abc.url_pipeline import URLPipelineAdapter from zarr.codecs import BytesCodec from zarr.core.buffer import Buffer, NDBuffer from zarr.core.dtype.npy.bool import Bool @@ -17,10 +17,10 @@ from collections.abc import Iterable from typing import Any, ClassVar, Literal, Self - from zarr.abc.url_pipeline import PipelineContext, PipelineSegment from zarr.core.array_spec import ArraySpec from zarr.core.common import ZarrFormat from zarr.core.dtype.common import DTypeJSON, DTypeSpec_V2 + from zarr.storage import PipelineContext, PipelineSegment, StorePath class TestEntrypointCodec(ArrayBytesCodec): @@ -110,8 +110,8 @@ class TestEntrypointURLAdapter(URLPipelineAdapter): @classmethod async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext - ) -> AdapterResolution: - from zarr.storage import MemoryStore + ) -> StorePath: + from zarr.storage import MemoryStore, StorePath store = await MemoryStore.open(read_only=context.read_only) - return AdapterResolution(store=store, path=segment.body) + return StorePath(store, path=segment.body) diff --git a/tests/test_url_pipeline/test_pipeline_object.py b/tests/test_url_pipeline/test_pipeline_object.py index 4c0d2779a5..78c50ec655 100644 --- a/tests/test_url_pipeline/test_pipeline_object.py +++ b/tests/test_url_pipeline/test_pipeline_object.py @@ -49,9 +49,9 @@ def test_from_url( pipeline = URLPipeline.from_url(url) assert pipeline.segments == parse_pipeline(url) assert tuple(segment.scheme for segment in pipeline.segments) == schemes - assert tuple(segment.scheme for segment in pipeline.store_segments) == store_schemes - assert pipeline.zarr_format == zarr_format - assert pipeline.path == path + assert tuple(segment.scheme for segment in pipeline._store_segments) == store_schemes + assert pipeline._zarr_format == zarr_format + assert pipeline._node_path == path assert str(pipeline) == url assert URLPipeline(pipeline.segments) == pipeline @@ -103,11 +103,11 @@ def test_resolve_zarr_format( """ suffix = "" if pipeline_format is None else f"|zarr{pipeline_format}:" pipeline = URLPipeline.from_url(f"memory://x{suffix}") - assert pipeline.resolve_zarr_format(requested) == expected # type: ignore[arg-type] + assert pipeline._resolve_zarr_format(requested) == expected # type: ignore[arg-type] def test_resolve_zarr_format_conflict_raises() -> None: """A caller-supplied format that differs from the pipeline's segment is an error.""" pipeline = URLPipeline.from_url("memory://x|zarr2:") with pytest.raises(ValueError, match="conflicts with the 'zarr2:' segment"): - pipeline.resolve_zarr_format(3) + pipeline._resolve_zarr_format(3) diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index 2b19e22a46..903b176f32 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -15,19 +15,23 @@ import zarr import zarr.registry from zarr.abc.store import Store -from zarr.abc.url_pipeline import ( - AdapterResolution, - PipelineContext, - PipelineSegment, - URLPipelineAdapter, -) +from zarr.abc.url_pipeline import URLPipelineAdapter from zarr.errors import URLPipelineError, ZarrUserWarning from zarr.registry import ( - get_url_adapter, - list_url_adapter_schemes, + _get_url_adapter, + _list_url_adapter_schemes, register_url_adapter, ) -from zarr.storage import LocalStore, ManagedMemoryStore, MemoryStore, URLPipeline, WrapperStore +from zarr.storage import ( + LocalStore, + ManagedMemoryStore, + MemoryStore, + PipelineContext, + PipelineSegment, + StorePath, + URLPipeline, + WrapperStore, +) from zarr.storage._common import _has_fsspec, make_store, make_store_path from zarr.storage._utils import _join_paths @@ -56,22 +60,19 @@ class WrapperAdapter(URLPipelineAdapter): """ A wrapper-style adapter: resolves the preceding pipeline into a store. - Follows the wrapper contract: the preceding resolution's residual path - is carried forward (joined with this segment's own path), and unchanged - fields survive via `dataclasses.replace`. + Follows the wrapper contract: the preceding residual path is carried + forward, joined with this segment's own path. """ @classmethod async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext - ) -> AdapterResolution: + ) -> StorePath: preceding = await context.resolve_preceding() store = TracingStore(preceding.store) store.context = context store.segment = segment - return dataclasses.replace( - preceding, store=store, path=_join_paths([preceding.path, segment.body]) - ) + return StorePath(store, _join_paths([preceding.path, segment.body])) class NativeAdapter(URLPipelineAdapter): @@ -82,10 +83,10 @@ class NativeAdapter(URLPipelineAdapter): @classmethod async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext - ) -> AdapterResolution: + ) -> StorePath: cls.seen_urls.append(context.preceding_url) store = await MemoryStore.open(read_only=context.read_only) - return AdapterResolution(store=store, path=segment.body) + return StorePath(store, path=segment.body) class RootAdapter(URLPipelineAdapter): @@ -94,12 +95,12 @@ class RootAdapter(URLPipelineAdapter): @classmethod async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext - ) -> AdapterResolution: + ) -> StorePath: assert context.preceding == () if context.mode in ("w", "w-", "r+"): raise ValueError("read-only scheme") store = await MemoryStore.open(read_only=True) - return AdapterResolution(store=store, path=segment.body.lstrip("/")) + return StorePath(store, path=segment.body.lstrip("/")) class DisobedientAdapter(URLPipelineAdapter): @@ -108,46 +109,46 @@ class DisobedientAdapter(URLPipelineAdapter): @classmethod async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext - ) -> AdapterResolution: - return AdapterResolution(store=await MemoryStore.open(read_only=False)) + ) -> StorePath: + return StorePath(await MemoryStore.open(read_only=False)) class TestRegistry: def test_register_and_get(self) -> None: register_url_adapter("demo", WrapperAdapter) - assert get_url_adapter("demo") is WrapperAdapter - assert get_url_adapter("DEMO") is WrapperAdapter - assert "demo" in list_url_adapter_schemes() + assert _get_url_adapter("demo") is WrapperAdapter + assert _get_url_adapter("DEMO") is WrapperAdapter + assert "demo" in _list_url_adapter_schemes() def test_unknown_scheme(self) -> None: with pytest.raises(URLPipelineError, match="no URL pipeline adapter is registered"): - get_url_adapter("nonexistent-scheme") + _get_url_adapter("nonexistent-scheme") def test_reregistering_scheme_warns(self) -> None: register_url_adapter("demo", WrapperAdapter) with pytest.warns(ZarrUserWarning, match="is being replaced"): register_url_adapter("demo", NativeAdapter) - assert get_url_adapter("demo") is NativeAdapter + assert _get_url_adapter("demo") is NativeAdapter # re-registering the same class is not a collision register_url_adapter("demo", NativeAdapter) @pytest.mark.usefixtures("set_path") def test_entrypoint_discovery(self) -> None: - assert "example-pkg.entrypoint-scheme" in list_url_adapter_schemes() - cls = get_url_adapter("example-pkg.entrypoint-scheme") + assert "example-pkg.entrypoint-scheme" in _list_url_adapter_schemes() + cls = _get_url_adapter("example-pkg.entrypoint-scheme") assert cls.__name__ == "TestEntrypointURLAdapter" @pytest.mark.usefixtures("set_path") async def test_entrypoint_end_to_end(self) -> None: result = await URLPipeline.from_url( "memory://src|example-pkg.entrypoint-scheme:sub/path" - ).resolve() + )._resolve_store_path() assert result.path == "sub/path" @pytest.mark.usefixtures("set_path") def test_entrypoint_scheme_lookup_is_case_insensitive(self) -> None: # entry-point names are matched case-insensitively, like schemes - cls = get_url_adapter("Example-PKG.Entrypoint-Scheme") + cls = _get_url_adapter("Example-PKG.Entrypoint-Scheme") assert cls.__name__ == "TestEntrypointURLAdapter" @pytest.mark.usefixtures("set_path") @@ -156,9 +157,9 @@ def test_loading_one_scheme_leaves_others_pending(self) -> None: registry = zarr.registry._url_adapter_registry assert any(e.name == "example-pkg.entrypoint-scheme" for e in registry.lazy_load_list) with pytest.raises(URLPipelineError, match="no URL pipeline adapter"): - get_url_adapter("some-other-scheme") + _get_url_adapter("some-other-scheme") assert any(e.name == "example-pkg.entrypoint-scheme" for e in registry.lazy_load_list) - assert get_url_adapter("example-pkg.entrypoint-scheme").__name__ == ( + assert _get_url_adapter("example-pkg.entrypoint-scheme").__name__ == ( "TestEntrypointURLAdapter" ) @@ -177,14 +178,14 @@ def _fake_entry_point( _ADAPTER_SOURCE = """ -from zarr.abc.url_pipeline import AdapterResolution, URLPipelineAdapter -from zarr.storage import MemoryStore +from zarr.abc.url_pipeline import URLPipelineAdapter +from zarr.storage import MemoryStore, StorePath {prelude} class Adapter(URLPipelineAdapter): @classmethod async def open_pipeline_segment(cls, segment, context): - return AdapterResolution(store=await MemoryStore.open(read_only=context.read_only)) + return StorePath(await MemoryStore.open(read_only=context.read_only)) """ @@ -207,7 +208,7 @@ def test_adapter_import_may_resolve_other_adapters( prelude=( "import zarr.registry\n" "try:\n" - " zarr.registry.get_url_adapter('no-such-scheme-at-import')\n" + " zarr.registry._get_url_adapter('no-such-scheme-at-import')\n" "except Exception:\n" " pass\n" ) @@ -215,10 +216,10 @@ def test_adapter_import_may_resolve_other_adapters( ) ) result: list[object] = [] - thread = threading.Thread(target=lambda: result.append(get_url_adapter("reentrant"))) + thread = threading.Thread(target=lambda: result.append(_get_url_adapter("reentrant"))) thread.start() thread.join(timeout=10) - assert not thread.is_alive(), "get_url_adapter deadlocked on a re-entrant lookup" + assert not thread.is_alive(), "_get_url_adapter deadlocked on a re-entrant lookup" assert getattr(result[0], "__name__", None) == "Adapter" def test_concurrent_loads_keep_other_schemes_pending( @@ -242,16 +243,17 @@ def test_concurrent_loads_keep_other_schemes_pending( ) ) threads = [ - threading.Thread(target=get_url_adapter, args=(scheme,)) for scheme in ("slow", "quick") + threading.Thread(target=_get_url_adapter, args=(scheme,)) + for scheme in ("slow", "quick") ] for thread in threads: thread.start() for thread in threads: thread.join(timeout=10) assert not any(thread.is_alive() for thread in threads) - assert "untouched" in list_url_adapter_schemes() + assert "untouched" in _list_url_adapter_schemes() assert [e.name for e in pending] == ["untouched"] - assert get_url_adapter("untouched").__name__ == "Adapter" + assert _get_url_adapter("untouched").__name__ == "Adapter" def test_import_failure_is_wrapped_and_not_permanent(self) -> None: pending = zarr.registry._url_adapter_registry.lazy_load_list @@ -259,11 +261,11 @@ def test_import_failure_is_wrapped_and_not_permanent(self) -> None: EntryPoint(name="broken", value="no_such_module_xyz:Adapter", group="zarr.url_adapters") ) with pytest.raises(URLPipelineError, match="could not be loaded"): - get_url_adapter("broken") + _get_url_adapter("broken") # still advertised, so a transient failure can be retried - assert "broken" in list_url_adapter_schemes() + assert "broken" in _list_url_adapter_schemes() with pytest.raises(URLPipelineError, match="could not be loaded"): - get_url_adapter("broken") + _get_url_adapter("broken") def test_registered_adapter_shadows_entry_point_with_warning( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -282,12 +284,12 @@ def test_registered_adapter_shadows_entry_point_with_warning( ) register_url_adapter("shadowed", NativeAdapter) with pytest.warns(ZarrUserWarning, match="already registered"): - assert get_url_adapter("shadowed") is NativeAdapter + assert _get_url_adapter("shadowed") is NativeAdapter assert not any(e.name == "shadowed" for e in pending) # a second lookup is silent with warnings.catch_warnings(): warnings.simplefilter("error") - assert get_url_adapter("shadowed") is NativeAdapter + assert _get_url_adapter("shadowed") is NativeAdapter def test_duplicate_entry_points_warn_and_first_wins( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -304,7 +306,7 @@ def test_duplicate_entry_points_warn_and_first_wins( ) ) with pytest.warns(ZarrUserWarning, match="multiple 'zarr.url_adapters' entry points"): - cls = get_url_adapter("dup") + cls = _get_url_adapter("dup") assert getattr(sys.modules[cls.__module__], "ORIGIN", None) == "dup_first" assert not any(e.name == "dup" for e in pending) @@ -332,7 +334,7 @@ class CountingRootAdapter(RootAdapter): @classmethod async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext - ) -> AdapterResolution: + ) -> StorePath: calls.append(segment) return await super().open_pipeline_segment(segment, context) @@ -355,7 +357,9 @@ def test_native_schemes_do_not_route_to_adapters(self) -> None: class TestResolve: async def test_wrapper_adapter_chain(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:inner/path").resolve() + result = await URLPipeline.from_url( + f"file:{tmp_path.as_posix()}|wrap:inner/path" + )._resolve_store_path() assert isinstance(result.store, TracingStore) assert result.path == "inner/path" assert result.store.context.preceding_url == f"file:{tmp_path.as_posix()}" @@ -366,25 +370,27 @@ async def test_nested_wrappers_preserve_residual_paths(self, tmp_path: Path) -> # each wrapper joins the preceding residual path with its own, so # no segment's path is lost in root|wrap:a|wrap:b register_url_adapter("wrap", WrapperAdapter) - result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:a|wrap:b").resolve() + result = await URLPipeline.from_url( + f"file:{tmp_path.as_posix()}|wrap:a|wrap:b" + )._resolve_store_path() assert result.path == "a/b" async def test_native_adapter_gets_preceding_url(self) -> None: register_url_adapter("native", NativeAdapter) NativeAdapter.seen_urls.clear() - await URLPipeline.from_url("s3://bucket/repo|native:").resolve() + await URLPipeline.from_url("s3://bucket/repo|native:")._resolve_store_path() assert NativeAdapter.seen_urls == ["s3://bucket/repo"] async def test_multi_segment_preceding_url(self) -> None: register_url_adapter("wrap", WrapperAdapter) register_url_adapter("native", NativeAdapter) NativeAdapter.seen_urls.clear() - await URLPipeline.from_url("memory://base|wrap:a|native:x").resolve() + await URLPipeline.from_url("memory://base|wrap:a|native:x")._resolve_store_path() assert NativeAdapter.seen_urls == ["memory://base|wrap:a"] async def test_root_adapter(self) -> None: register_url_adapter("rooty", RootAdapter) - result = await URLPipeline.from_url("rooty://org/repo").resolve() + result = await URLPipeline.from_url("rooty://org/repo")._resolve_store_path() assert result.path == "org/repo" assert result.store.read_only @@ -392,16 +398,20 @@ async def test_root_adapter_composes_with_chain(self) -> None: register_url_adapter("rooty", RootAdapter) register_url_adapter("native", NativeAdapter) NativeAdapter.seen_urls.clear() - await URLPipeline.from_url("rooty://org/repo|native:x").resolve() + await URLPipeline.from_url("rooty://org/repo|native:x")._resolve_store_path() assert NativeAdapter.seen_urls == ["rooty://org/repo"] async def test_read_only_flag(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:").resolve(mode="r") + result = await URLPipeline.from_url( + f"file:{tmp_path.as_posix()}|wrap:" + )._resolve_store_path(mode="r") assert isinstance(result.store, TracingStore) assert result.store.context.read_only assert result.store.context.mode == "r" - result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:").resolve() + result = await URLPipeline.from_url( + f"file:{tmp_path.as_posix()}|wrap:" + )._resolve_store_path() assert isinstance(result.store, TracingStore) assert not result.store.context.read_only assert result.store.context.mode is None @@ -410,7 +420,7 @@ async def test_read_only_is_enforced_on_disobedient_adapters(self) -> None: # the resolver downgrades a writable store returned under mode "r" # instead of trusting the adapter to have honored context.read_only register_url_adapter("bad", DisobedientAdapter) - result = await URLPipeline.from_url("memory://base|bad:").resolve(mode="r") + result = await URLPipeline.from_url("memory://base|bad:")._resolve_store_path(mode="r") assert result.store.read_only store = await make_store(URLPipeline.from_url("memory://base|bad:"), mode="r") assert store.read_only @@ -425,12 +435,12 @@ class StubbornAdapter(URLPipelineAdapter): @classmethod async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext - ) -> AdapterResolution: - return AdapterResolution(store=await StubbornStore.open(read_only=False)) + ) -> StorePath: + return StorePath(await StubbornStore.open(read_only=False)) register_url_adapter("stubborn", StubbornAdapter) with pytest.raises(URLPipelineError, match="does not support read-only conversion"): - await URLPipeline.from_url("memory://base|stubborn:").resolve(mode="r") + await URLPipeline.from_url("memory://base|stubborn:")._resolve_store_path(mode="r") async def test_read_only_enforcement_failure_closes_store(self) -> None: # the adapter's store must not leak when it cannot be made read-only @@ -448,12 +458,12 @@ class StubbornAdapter(URLPipelineAdapter): @classmethod async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext - ) -> AdapterResolution: - return AdapterResolution(store=await StubbornStore.open(read_only=False)) + ) -> StorePath: + return StorePath(await StubbornStore.open(read_only=False)) register_url_adapter("stubborn", StubbornAdapter) with pytest.raises(URLPipelineError): - await URLPipeline.from_url("memory://base|stubborn:").resolve(mode="r") + await URLPipeline.from_url("memory://base|stubborn:")._resolve_store_path(mode="r") assert closed == [True] async def test_resolve_preceding_mode_override(self, tmp_path: Path) -> None: @@ -463,15 +473,15 @@ class ReadOnlyRootWrapper(WrapperAdapter): @classmethod async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext - ) -> AdapterResolution: - preceding = await context.resolve_preceding(mode="r") + ) -> StorePath: + preceding = await dataclasses.replace(context, mode="r").resolve_preceding() assert preceding.store.read_only - return dataclasses.replace(preceding, path=segment.body) + return StorePath(preceding.store, segment.body) register_url_adapter("rowrap", ReadOnlyRootWrapper) - result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|rowrap:x").resolve( - mode="w" - ) + result = await URLPipeline.from_url( + f"file:{tmp_path.as_posix()}|rowrap:x" + )._resolve_store_path(mode="w") assert result.path == "x" async def test_wrapper_on_local_file_root_read_only(self, tmp_path: Path) -> None: @@ -481,7 +491,9 @@ async def test_wrapper_on_local_file_root_read_only(self, tmp_path: Path) -> Non target = tmp_path / "data.bin" target.write_bytes(b"payload") register_url_adapter("wrap", WrapperAdapter) - result = await URLPipeline.from_url(f"file:{target.as_posix()}|wrap:").resolve(mode="r") + result = await URLPipeline.from_url(f"file:{target.as_posix()}|wrap:")._resolve_store_path( + mode="r" + ) assert isinstance(result.store, TracingStore) assert result.store.read_only @@ -500,7 +512,7 @@ async def test_wrapper_on_local_file_root_writable_modes( target = tmp_path / "data.bin" target.write_bytes(b"payload") register_url_adapter("wrap", WrapperAdapter) - await URLPipeline.from_url(f"file:{target.as_posix()}|wrap:").resolve(mode=mode) + await URLPipeline.from_url(f"file:{target.as_posix()}|wrap:")._resolve_store_path(mode=mode) async def test_resolve_preceding_storage_options_override(self, tmp_path: Path) -> None: # an adapter that consumed its namespaced keys strips them before @@ -511,16 +523,20 @@ class ConsumingWrapper(WrapperAdapter): @classmethod async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext - ) -> AdapterResolution: + ) -> StorePath: options = dict(context.storage_options or {}) cls.seen = options.pop("consuming_secret", None) - preceding = await context.resolve_preceding(storage_options=options or None) - return dataclasses.replace(preceding, path=segment.body) + preceding = await dataclasses.replace( + context, storage_options=options or None + ).resolve_preceding() + return StorePath(preceding.store, segment.body) register_url_adapter("consuming", ConsumingWrapper) # the local-path root rejects any surviving storage_options, so this # passing proves the adapter's keys were stripped before resolution - result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|consuming:x").resolve( + result = await URLPipeline.from_url( + f"file:{tmp_path.as_posix()}|consuming:x" + )._resolve_store_path( storage_options={"consuming_secret": "s3cr3t"}, ) assert result.path == "x" @@ -535,50 +551,54 @@ class OptionsProbe(NativeAdapter): @classmethod async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext - ) -> AdapterResolution: + ) -> StorePath: cls.seen_options = context.storage_options return await super().open_pipeline_segment(segment, context) register_url_adapter("probe", OptionsProbe) opts = {"anon": True} - await URLPipeline.from_url("s3://bucket/x|probe:").resolve(storage_options=opts) + await URLPipeline.from_url("s3://bucket/x|probe:")._resolve_store_path(storage_options=opts) assert OptionsProbe.seen_options == opts async def test_wrapper_at_root_position_raises(self) -> None: # a wrapper adapter used as the pipeline root has nothing to wrap register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="no preceding sub-URL"): - await URLPipeline.from_url("wrap:whatever").resolve() + await URLPipeline.from_url("wrap:whatever")._resolve_store_path() async def test_single_url_is_a_trivial_pipeline(self) -> None: """A URL without `|` resolves as a pipeline consisting of its root alone.""" - result = await URLPipeline.from_url("memory://plain").resolve() + result = await URLPipeline.from_url("memory://plain")._resolve_store_path() assert isinstance(result.store, ManagedMemoryStore) assert result.path == "" async def test_resolve_method(self, tmp_path: Path) -> None: """`URLPipeline.resolve` yields the adapter's store and residual path.""" register_url_adapter("wrap", WrapperAdapter) - result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:a").resolve() + result = await URLPipeline.from_url( + f"file:{tmp_path.as_posix()}|wrap:a" + )._resolve_store_path() assert isinstance(result.store, TracingStore) assert result.path == "a" async def test_format_segment_path_joins_residual_path(self, tmp_path: Path) -> None: """The node path of a trailing format segment extends the adapters' residual path.""" register_url_adapter("wrap", WrapperAdapter) - result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:a|zarr3:b").resolve() + result = await URLPipeline.from_url( + f"file:{tmp_path.as_posix()}|wrap:a|zarr3:b" + )._resolve_store_path() assert result.path == "a/b" async def test_unknown_adapter_scheme_raises(self) -> None: with pytest.raises(URLPipelineError, match="no URL pipeline adapter is registered"): - await URLPipeline.from_url("memory://base|no-such-adapter:").resolve() + await URLPipeline.from_url("memory://base|no-such-adapter:")._resolve_store_path() async def test_base_case_value_error_is_wrapped(self) -> None: # a root that only the fallback scheme detection accepts must not # leak urlparse's ValueError out of the base case register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="could not resolve the pipeline root"): - await URLPipeline.from_url("vendor.x://[authority]/path|wrap:").resolve() + await URLPipeline.from_url("vendor.x://[authority]/path|wrap:")._resolve_store_path() class TestSpecRoots: @@ -587,7 +607,7 @@ class TestSpecRoots: @pytest.mark.parametrize("root", ["memory:", "memory:/", "memory://"]) async def test_memory_root_spellings_equivalent(self, root: str) -> None: register_url_adapter("wrap", WrapperAdapter) - result = await URLPipeline.from_url(f"{root}|wrap:").resolve() + result = await URLPipeline.from_url(f"{root}|wrap:")._resolve_store_path() assert isinstance(result.store, TracingStore) inner = result.store._store assert isinstance(inner, ManagedMemoryStore) @@ -597,7 +617,7 @@ async def test_memory_root_spellings_equivalent(self, root: str) -> None: @pytest.mark.parametrize("root", ["memory:a/b", "memory:/a/b", "memory://a/b"]) async def test_memory_root_named_spellings_equivalent(self, root: str) -> None: register_url_adapter("wrap", WrapperAdapter) - result = await URLPipeline.from_url(f"{root}|wrap:").resolve() + result = await URLPipeline.from_url(f"{root}|wrap:")._resolve_store_path() assert isinstance(result.store, TracingStore) inner = result.store._store assert isinstance(inner, ManagedMemoryStore) @@ -614,12 +634,14 @@ async def test_memory_root_shares_data_across_spellings(self) -> None: async def test_memory_root_rejects_query(self) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="do not accept a query"): - await URLPipeline.from_url("memory:a?opt=1|wrap:").resolve() + await URLPipeline.from_url("memory:a?opt=1|wrap:")._resolve_store_path() async def test_memory_root_rejects_storage_options(self) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(TypeError, match="'storage_options' was provided but unused"): - await URLPipeline.from_url("memory:a|wrap:").resolve(storage_options={"anon": True}) + await URLPipeline.from_url("memory:a|wrap:")._resolve_store_path( + storage_options={"anon": True} + ) async def test_file_root_localhost_authority(self, tmp_path: Path) -> None: # file://localhost/p and file:///p are equivalent to file:/p @@ -645,26 +667,26 @@ async def test_file_root_windows_drive_spellings( # chdir into tmp so it is created there monkeypatch.chdir(tmp_path) drive_path = "C:/drive/data" - r1 = await URLPipeline.from_url(f"file:{drive_path}|wrap:").resolve() - r2 = await URLPipeline.from_url(f"file:///{drive_path}|wrap:").resolve() + r1 = await URLPipeline.from_url(f"file:{drive_path}|wrap:")._resolve_store_path() + r2 = await URLPipeline.from_url(f"file:///{drive_path}|wrap:")._resolve_store_path() assert isinstance(r1.store, TracingStore) assert isinstance(r2.store, TracingStore) async def test_file_root_rejects_other_authority(self) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="unsupported authority"): - await URLPipeline.from_url("file://example.com/tmp/x|wrap:").resolve() + await URLPipeline.from_url("file://example.com/tmp/x|wrap:")._resolve_store_path() @pytest.mark.parametrize("root", ["file:relative/path", "file://localhost"]) async def test_file_root_rejects_relative_paths(self, root: str) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="absolute path"): - await URLPipeline.from_url(f"{root}|wrap:").resolve() + await URLPipeline.from_url(f"{root}|wrap:")._resolve_store_path() async def test_file_root_rejects_query(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="do not accept a query"): - await URLPipeline.from_url(f"file:{tmp_path}?v=1|wrap:").resolve() + await URLPipeline.from_url(f"file:{tmp_path}?v=1|wrap:")._resolve_store_path() @pytest.mark.skipif( not _has_async_fsspec_wrapper, @@ -685,7 +707,7 @@ async def test_file_root_does_not_percent_decode(self, tmp_path: Path) -> None: # documented divergence from RFC 8089: consistent with LocalStore, no # percent-escape is decoded, so %20 names a literal directory register_url_adapter("wrap", WrapperAdapter) - await URLPipeline.from_url(f"file:{tmp_path.as_posix()}/a%20b|wrap:").resolve() + await URLPipeline.from_url(f"file:{tmp_path.as_posix()}/a%20b|wrap:")._resolve_store_path() assert (tmp_path / "a%20b").is_dir() assert not (tmp_path / "a b").exists() @@ -719,9 +741,9 @@ class SlashRootAdapter(WrapperAdapter): @classmethod async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext - ) -> AdapterResolution: + ) -> StorePath: preceding = await context.resolve_preceding() - return dataclasses.replace(preceding, path="/") + return StorePath(preceding.store, "/") register_url_adapter("slashy", SlashRootAdapter) store = await make_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|slashy:")) @@ -739,8 +761,8 @@ class LeakProbe(URLPipelineAdapter): @classmethod async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext - ) -> AdapterResolution: - return AdapterResolution(store=await ClosingStore.open(), path="residual") + ) -> StorePath: + return StorePath(await ClosingStore.open(), path="residual") register_url_adapter("leaky", LeakProbe) with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): From 3dbd88c53cd12ac67a88c56358f9ad27330af5d4 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 22 Oct 2026 21:39:23 +0200 Subject: [PATCH 14/20] refactor(url-pipeline): drop open_url; open_* accept URLPipeline objects only Scope agreed on #4192: `zarr.open_url` moves to a follow-on PR, and the open functions accept `URLPipeline` objects but never read a plain string as a pipeline. - Remove `zarr.open_url` / `zarr.api.{a,}synchronous.open_url`, its API page and tests; docs and docstrings point at `URLPipeline` instead. - Revert the public signature changes the core does not need: `Array.open` no longer gains a `zarr_format` parameter and `Group.from_store` keeps `zarr_format: ZarrFormat = 3`. A conflicting default raises, as with any other explicit format. - Validate the dtype in `AsyncArray._create` before creating the store, as before this PR. - Test `open`, `open_group`, `open_array` and `open_consolidated` with a `URLPipeline`, and that the same text passed as a string is a path. Co-Authored-By: Claude Opus 5.5 --- changes/4192.feature.md | 16 ++-- docs/api/zarr/functions/open_url.md | 5 -- docs/user-guide/extending.md | 4 +- docs/user-guide/storage.md | 18 ++-- mkdocs.yml | 1 - src/zarr/__init__.py | 2 - src/zarr/api/asynchronous.py | 54 ------------ src/zarr/api/synchronous.py | 53 ----------- src/zarr/core/array.py | 8 +- src/zarr/core/group.py | 12 +-- src/zarr/storage/_url_pipeline.py | 10 +-- tests/test_url_pipeline/test_resolver.py | 107 +++++++++++++---------- 12 files changed, 87 insertions(+), 203 deletions(-) delete mode 100644 docs/api/zarr/functions/open_url.md diff --git a/changes/4192.feature.md b/changes/4192.feature.md index 98b3d64996..bdcd786fe6 100644 --- a/changes/4192.feature.md +++ b/changes/4192.feature.md @@ -1,9 +1,9 @@ Added core support for URL pipelines (https://github.com/jbms/url-pipeline): `|`-chained URLs that address zarr data through nested storage layers, e.g. `s3://bucket/data.zip|zip:|zarr3:`. A parsed `zarr.storage.URLPipeline` (built with -`URLPipeline.from_url`) can be passed to any function that accepts a `StoreLike`, and -the new `zarr.open_url(url, mode="r" | "r+")` opens the existing node a pipeline string -addresses without ever creating or overwriting anything. Adapters for the +`URLPipeline.from_url`) can be passed to any function that accepts a `StoreLike`, +including `zarr.open`, `zarr.open_group`, `zarr.open_array` and +`zarr.open_consolidated`. Adapters for the sub-URLs after the first are provided by third-party packages (e.g. Icechunk) through the `zarr.url_adapters` entry-point group and the single-method `zarr.abc.url_pipeline.URLPipelineAdapter` interface, which receives a @@ -16,13 +16,13 @@ Behavior notes: - Plain strings are never interpreted as pipelines: `zarr.open("data|x.zarr")` still addresses a local directory of that name, whatever adapters are installed. Only - `zarr.open_url` and `URLPipeline.from_url` read a string as a pipeline. + `URLPipeline.from_url` reads a string as a pipeline. - A trailing `zarr2:`/`zarr3:` segment selects the zarr format of the open or create call, and its body is the path of the node to open. A `zarr_format` argument that - conflicts with it raises `ValueError`; functions whose `zarr_format` defaults to 3 - (`zarr.create_array`, `Group.from_store`, `Group.open`, `Array.open`) need an explicit - `zarr_format=None` to defer to the pipeline. `Array.open` gains a `zarr_format` - parameter mirroring `Group.open`. + conflicts with it raises `ValueError`, including the default of 3 in + `zarr.create_array`, `Group.from_store`, `Group.open` and `Array.open`: pass the + matching format (or `zarr_format=None` where it is accepted) to open a `zarr2:` + pipeline with these. - Inside a pipeline, `memory:` and `file:` roots follow the URL pipeline spec's spelling rules (`memory:` ≡ `memory:/` ≡ `memory://`; `file:` must carry an absolute path with at most a `localhost` authority). Percent-escapes are not decoded, matching diff --git a/docs/api/zarr/functions/open_url.md b/docs/api/zarr/functions/open_url.md deleted file mode 100644 index 84774a1710..0000000000 --- a/docs/api/zarr/functions/open_url.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: zarr.open_url ---- - -::: zarr.open_url diff --git a/docs/user-guide/extending.md b/docs/user-guide/extending.md index 5dc5d51358..5ecc90b2b8 100644 --- a/docs/user-guide/extending.md +++ b/docs/user-guide/extending.md @@ -127,7 +127,7 @@ from zarr.storage import URLPipeline register_url_adapter("mypackage.myscheme", MyAdapter) url = "memory://x|mypackage.myscheme:a|mypackage.myscheme:b" created = zarr.create_group(URLPipeline.from_url(url)) -group = zarr.open_url(url) +group = zarr.open_group(URLPipeline.from_url(url), mode="r") assert group.path == "a/b" ``` @@ -143,7 +143,7 @@ scheme as the entry-point name. Nonstandard schemes should be vendor-prefixed Adapters are loaded lazily, only when a pipeline naming their scheme is resolved. An adapter may also be used as a *root* scheme (e.g. `mypackage.myscheme://org/repo`), in which case `context.preceding` is empty; such a URL reaches the adapter only through -`zarr.open_url` or a `URLPipeline`, never as a plain string store specification. The returned store must honor +a `URLPipeline`, never as a plain string store specification. The returned store must honor `context.read_only`, and `open_pipeline_segment` runs on zarr's internal event loop, so it must not block or call zarr's synchronous API. See [`zarr.abc.url_pipeline`][] and [`PipelineContext`][zarr.storage.PipelineContext] for the diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index bb294353aa..fe568e8df3 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -119,8 +119,7 @@ sub-URL locates a resource with a conventional URL; each subsequent sub-URL name `zip:` adapter is under development and will expand this section. A parsed [`URLPipeline`][zarr.storage.URLPipeline] can be passed to any function that -accepts a `StoreLike`, and [`zarr.open_url`][] opens the *existing* node a pipeline -string addresses, read-only unless `mode="r+"` is given: +accepts a `StoreLike`: ```python exec="true" session="storage" source="above" result="ansi" from zarr.storage import URLPipeline @@ -128,23 +127,20 @@ from zarr.storage import URLPipeline pipeline = URLPipeline.from_url("memory://pipeline-demo|zarr2:") group = zarr.open_group(pipeline, mode="w") print(group.metadata.zarr_format) -# the same thing, as a one-liner -print(zarr.open_url("memory://pipeline-demo|zarr2:").metadata.zarr_format) +print(zarr.open_group(pipeline, mode="r").metadata.zarr_format) ``` -Only these two entry points read a string as a pipeline. Plain string store -specifications keep their existing meaning everywhere else, so a local path that -happens to contain `|` still names a directory, whatever adapters are installed. -Unlike `zarr.open`, `zarr.open_url` never creates or overwrites: to create a node at a -pipeline-addressed location, pass the `URLPipeline` to `zarr.create_array` or -`zarr.create_group`. +Only `URLPipeline.from_url` reads a string as a pipeline. Plain string store +specifications keep their existing meaning, so a local path that happens to contain `|` +still names a directory, whatever adapters are installed. A trailing `zarr2:` or `zarr3:` segment is a *format segment*. It is not an adapter: zarr-python reads it from the pipeline itself and uses it as the `zarr_format` of the open or create call, and its body (`zarr3:path/to/node`) is the path of the node to open. A `zarr_format` argument that conflicts with the format segment raises `ValueError`; functions whose `zarr_format` defaults to 3 (such as `zarr.create_array` -and `Group.open`) need an explicit `zarr_format=None` to defer to the pipeline. +and `Group.open`) need an explicit `zarr_format=None` (or the matching format) to defer +to the pipeline. `storage_options` apply to the *root* sub-URL (e.g. fsspec options for `s3://...`); adapters may consume adapter-specific, namespaced keys. No percent-escape is decoded diff --git a/mkdocs.yml b/mkdocs.yml index c0b7a96fa0..58b5d3344a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -81,7 +81,6 @@ nav: - ' zarr.open_consolidated': api/zarr/functions/open_consolidated.md - ' zarr.open_group': api/zarr/functions/open_group.md - ' zarr.open_like': api/zarr/functions/open_like.md - - ' zarr.open_url': api/zarr/functions/open_url.md - ' zarr.print_debug_info': api/zarr/functions/print_debug_info.md - ' zarr.registry': api/zarr/registry.md - ' zarr.save': api/zarr/functions/save.md diff --git a/src/zarr/__init__.py b/src/zarr/__init__.py index b8ac1fb11e..f305245f4e 100644 --- a/src/zarr/__init__.py +++ b/src/zarr/__init__.py @@ -27,7 +27,6 @@ open_consolidated, open_group, open_like, - open_url, save, save_array, save_group, @@ -176,7 +175,6 @@ def set_format(log_format: str) -> None: "open_consolidated", "open_group", "open_like", - "open_url", "print_debug_info", "save", "save_array", diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index b794946277..f873c777d6 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -88,7 +88,6 @@ "open_consolidated", "open_group", "open_like", - "open_url", "save", "save_array", "save_group", @@ -437,59 +436,6 @@ async def open( return await open_group(store=store_path, zarr_format=zarr_format, mode=mode, **kwargs) -async def open_url( - url: str, - *, - mode: Literal["r", "r+"] = "r", - storage_options: dict[str, Any] | None = None, -) -> AnyAsyncArray | AsyncGroup: - """Open the existing group or array addressed by a URL pipeline. - - This is the only entry point that reads a string as a - [URL pipeline][user-guide-url-pipelines]; [`open`][zarr.api.asynchronous.open] and the other - `StoreLike`-taking functions treat strings as they always have. Unlike - `open`, `open_url` never creates or overwrites anything: to create a node at - a pipeline-addressed location, pass a `URLPipeline` to - [`create_array`][zarr.api.asynchronous.create_array] or - [`create_group`][zarr.api.asynchronous.create_group]. - - The zarr format is taken from a trailing `zarr2:`/`zarr3:` segment when - present, and inferred from the stored metadata otherwise. - - Parameters - ---------- - url : str - A URL pipeline such as `"s3://bucket/data.zip|zip:|zarr3:"`. A URL - without `|` is a trivial pipeline consisting of its root alone. The - node to open is addressed by the URL itself: the body of a trailing - `zarr2:`/`zarr3:` segment is the path of the node within the store. - mode : {'r', 'r+'}, optional - `'r'` (the default) opens read-only; `'r+'` opens for reading and - writing. The node must exist in either case. - storage_options : dict, optional - Options for the pipeline's root sub-URL (e.g. fsspec options). - - Returns - ------- - z : array or group - Return type depends on what exists at the URL. For a statically typed - result, pass a `URLPipeline` to `open_array` or `open_group` instead. - """ - from zarr.storage import URLPipeline - - if mode not in ("r", "r+"): - raise ValueError( - f"Invalid mode: {mode!r}. open_url opens existing nodes only and accepts " - "'r' or 'r+'; to create a node at a URL pipeline, pass " - "URLPipeline.from_url(url) to create_array or create_group." - ) - return await open( - store=URLPipeline.from_url(url), - mode=mode, - storage_options=storage_options, - ) - - async def open_consolidated( *args: Any, use_consolidated: Literal[True] = True, **kwargs: Any ) -> AsyncGroup: diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 0980c2316b..b383085bda 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -66,7 +66,6 @@ "open_consolidated", "open_group", "open_like", - "open_url", "save", "save_array", "save_group", @@ -247,58 +246,6 @@ def open( return Group(obj) -def open_url( - url: str, - *, - mode: Literal["r", "r+"] = "r", - storage_options: dict[str, Any] | None = None, -) -> AnyArray | Group: - """Open the existing group or array addressed by a URL pipeline. - - This is the only entry point that reads a string as a - [URL pipeline][user-guide-url-pipelines]; [`open`][zarr.open] and the other - `StoreLike`-taking functions treat strings as they always have. Unlike - `open`, `open_url` never creates or overwrites anything: to create a node at - a pipeline-addressed location, pass a `URLPipeline` to - [`create_array`][zarr.create_array] or [`create_group`][zarr.create_group]. - - The zarr format is taken from a trailing `zarr2:`/`zarr3:` segment when - present, and inferred from the stored metadata otherwise. - - Parameters - ---------- - url : str - A URL pipeline such as `"s3://bucket/data.zip|zip:|zarr3:"`. A URL - without `|` is a trivial pipeline consisting of its root alone. The - node to open is addressed by the URL itself: the body of a trailing - `zarr2:`/`zarr3:` segment is the path of the node within the store. - mode : {'r', 'r+'}, optional - `'r'` (the default) opens read-only; `'r+'` opens for reading and - writing. The node must exist in either case. - storage_options : dict, optional - Options for the pipeline's root sub-URL (e.g. fsspec options). - - Returns - ------- - z : array or group - Return type depends on what exists at the URL. For a statically typed - result, pass a `URLPipeline` to `open_array` or `open_group` instead. - """ - from zarr.storage import URLPipeline - - if mode not in ("r", "r+"): - raise ValueError( - f"Invalid mode: {mode!r}. open_url opens existing nodes only and accepts " - "'r' or 'r+'; to create a node at a URL pipeline, pass " - "URLPipeline.from_url(url) to create_array or create_group." - ) - return open( - store=URLPipeline.from_url(url), - mode=mode, - storage_options=storage_options, - ) - - def open_consolidated(*args: Any, use_consolidated: Literal[True] = True, **kwargs: Any) -> Group: """ Alias for [`open_group`][zarr.api.synchronous.open_group] with `use_consolidated=True`. diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 9816da4d64..46a5d27fc7 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -470,8 +470,8 @@ async def _create( """ zarr_format = resolve_zarr_format(store, zarr_format) - store_path = await make_store_path(store) dtype_parsed = parse_dtype(dtype, zarr_format=zarr_format) + store_path = await make_store_path(store) shape = parse_shapelike(shape) @@ -2058,7 +2058,6 @@ def from_dict( def open( cls, store: StoreLike, - zarr_format: ZarrFormat | None = 3, ) -> Self: """Opens an existing Array from a store. @@ -2068,16 +2067,13 @@ def open( Store containing the Array. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. - zarr_format : {2, 3, None}, optional - The zarr format to expect. If None, the format selected by a URL pipeline - segment is used, otherwise the format is inferred from the store contents. Returns ------- Array Array opened from the store. """ - async_array = sync(AsyncArray.open(store, zarr_format=zarr_format)) + async_array = sync(AsyncArray.open(store)) return cls(async_array) @property diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 01d6e05df9..c888c3bc85 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -43,7 +43,6 @@ NodeType, ShapeLike, ZarrFormat, - _default_zarr_format, parse_shapelike, ) from zarr.core.config import config @@ -531,12 +530,10 @@ async def from_store( *, attributes: dict[str, Any] | None = None, overwrite: bool = False, - zarr_format: ZarrFormat | None = 3, + zarr_format: ZarrFormat = 3, ) -> AsyncGroup: zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store) - if zarr_format is None: - zarr_format = _default_zarr_format() attributes = attributes or {} group = cls( metadata=GroupMetadata(attributes=attributes, zarr_format=zarr_format), @@ -1959,7 +1956,7 @@ def from_store( store: StoreLike, *, attributes: dict[str, Any] | None = None, - zarr_format: ZarrFormat | None = 3, + zarr_format: ZarrFormat = 3, overwrite: bool = False, ) -> Group: """Instantiate a group from an initialized store. @@ -1972,9 +1969,8 @@ def from_store( for a description of all valid StoreLike values. attributes : dict, optional A dictionary of JSON-serializable values with user-defined attributes. - zarr_format : {2, 3, None}, optional - Zarr storage format version. None defers to the format selected by a - `URLPipeline` store, falling back to the configured default. + zarr_format : {2, 3}, optional + Zarr storage format version. overwrite : bool, optional If True, do not raise an error if the group already exists. diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py index 123efaeb63..0dcd907d07 100644 --- a/src/zarr/storage/_url_pipeline.py +++ b/src/zarr/storage/_url_pipeline.py @@ -6,8 +6,7 @@ only when a pipeline URL naming their scheme is actually resolved. A string is never interpreted as a pipeline on its own: the user wraps it -in a [`URLPipeline`][zarr.storage.URLPipeline] (or calls `zarr.open_url`), -and that object is what the `StoreLike` machinery resolves. Plain string +in a [`URLPipeline`][zarr.storage.URLPipeline], and that object is what the `StoreLike` machinery resolves. Plain string store specifications, including local paths that happen to contain `|`, keep their pre-existing meaning. No percent-escape is decoded. @@ -82,7 +81,7 @@ class PipelineContext: inner. Empty when the adapter's segment is the pipeline root. mode : AccessModeLiteral | None The access mode requested by the caller (e.g. - `zarr.open_url(mode=...)`), or None when unspecified. Adapters for + `zarr.open(mode=...)`), or None when unspecified. Adapters for read-only resources should raise for every mode that may write, and open read-only for `"r"` and None. storage_options : dict[str, Any] | None @@ -262,9 +261,8 @@ class URLPipeline: `URLPipeline` is an explicit object, no plain string store specification is ever interpreted as a pipeline: `zarr.open("data|x.zarr")` still addresses a local directory of that name, while - `zarr.open(URLPipeline.from_url("s3://bucket/data.zip|zip:|zarr3:"))` (or the - equivalent `zarr.open_url(...)`) resolves the pipeline through registered - adapters. + `zarr.open(URLPipeline.from_url("s3://bucket/data.zip|zip:|zarr3:"))` resolves + the pipeline through registered adapters. A pipeline is a value: instances are immutable, hashable, and compare equal when they hold the same segments. Nothing is resolved until the pipeline is diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index 903b176f32..3460b4b87e 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -42,6 +42,7 @@ ) if TYPE_CHECKING: + from collections.abc import Callable from pathlib import Path from zarr.core.common import AccessModeLiteral @@ -846,66 +847,72 @@ async def test_matching_explicit_format_ok(self, tmp_path: Path) -> None: assert group.metadata.zarr_format == 2 -class TestOpenURL: +class TestOpenFunctionsAcceptPipelines: """ - `zarr.open_url` interprets its string argument as a URL pipeline and opens - the existing node it addresses. It never creates: creation goes through - `create_array` / `create_group` with a `URLPipeline`. + `zarr.open`, `open_group`, `open_array` and `open_consolidated` accept a + `URLPipeline` and open the node it addresses; the body of a trailing format + segment is the node path. """ - async def test_open_url_opens_existing_node_with_pipeline_format(self, tmp_path: Path) -> None: + @pytest.fixture + def root(self, tmp_path: Path) -> str: register_url_adapter("wrap", WrapperAdapter) - zarr.create_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:|zarr2:")) - group = zarr.open_url(f"file:{tmp_path.as_posix()}|wrap:|zarr2:") + url = f"file:{tmp_path.as_posix()}|wrap:" + group = zarr.create_group(URLPipeline.from_url(url + "|zarr2:"), zarr_format=2) + group.create_array("sub/x", shape=(3,), dtype="i4")[:] = [1, 2, 3] + zarr.consolidate_metadata(URLPipeline.from_url(url + "|zarr2:")) + return url + + async def test_open(self, root: str) -> None: + group = zarr.open(URLPipeline.from_url(root + "|zarr2:"), mode="r") assert isinstance(group, zarr.Group) assert group.metadata.zarr_format == 2 assert isinstance(group.store, TracingStore) + array = zarr.open(URLPipeline.from_url(root + "|zarr2:sub/x"), mode="r") + assert isinstance(array, zarr.Array) + assert array[1] == 2 + + async def test_open_group(self, root: str) -> None: + group = zarr.open_group(URLPipeline.from_url(root + "|zarr2:sub"), mode="r") + assert group.path == "sub" assert group.read_only - async def test_open_url_format_segment_body_addresses_node(self, tmp_path: Path) -> None: - """The node path lives in the URL, as the body of the format segment.""" - register_url_adapter("wrap", WrapperAdapter) - zarr.create_array( - URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:"), - name="sub/x", - shape=(3,), - dtype="i4", - ) - array = zarr.open_url(f"file:{tmp_path.as_posix()}|wrap:|zarr3:sub/x") - assert isinstance(array, zarr.Array) + async def test_open_array(self, root: str) -> None: + array = zarr.open_array(URLPipeline.from_url(root + "sub|zarr2:x"), mode="r") assert array.store_path.path == "sub/x" + assert array[2] == 3 - async def test_open_url_read_write(self, tmp_path: Path) -> None: - zarr.create_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr3:")) - group = zarr.open_url(f"file:{tmp_path.as_posix()}|zarr3:", mode="r+") - assert not group.read_only - group.attrs["answer"] = 42 - assert zarr.open_url(f"file:{tmp_path.as_posix()}|zarr3:").attrs["answer"] == 42 - - async def test_open_url_missing_node_raises(self, tmp_path: Path) -> None: - """Nothing is created on open: a URL to a missing node is an error in every mode.""" - with pytest.raises(FileNotFoundError): - zarr.open_url(f"file:{tmp_path.as_posix()}/missing|zarr3:") - with pytest.raises(FileNotFoundError): - zarr.open_url(f"file:{tmp_path.as_posix()}/missing|zarr3:", mode="r+") + async def test_open_consolidated(self, root: str) -> None: + group = zarr.open_consolidated(URLPipeline.from_url(root + "|zarr2:"), mode="r") + assert group.metadata.consolidated_metadata is not None + assert group.metadata.zarr_format == 2 - async def test_open_url_rejects_create_modes(self, tmp_path: Path) -> None: - """The creating modes of `zarr.open` are not accepted.""" - with pytest.raises(ValueError, match="opens existing nodes only"): - zarr.open_url(f"file:{tmp_path.as_posix()}|zarr3:", mode="a") # type: ignore[arg-type] + async def test_async_open(self, root: str) -> None: + group = await zarr.api.asynchronous.open( + store=URLPipeline.from_url(root + "|zarr2:"), mode="r" + ) + assert isinstance(group, zarr.AsyncGroup) - async def test_async_open_url(self, tmp_path: Path) -> None: - zarr.create_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:")) - group = await zarr.api.asynchronous.open_url(f"file:{tmp_path.as_posix()}|zarr2:") - assert group.metadata.zarr_format == 2 + @pytest.mark.parametrize( + "open_func", + [zarr.open, zarr.open_group, zarr.open_array, zarr.open_consolidated], + ) + @pytest.mark.skipif(sys.platform == "win32", reason="'|' is not allowed in Windows file names") + async def test_strings_are_not_pipelines( + self, open_func: Callable[..., object], root: str, tmp_path: Path + ) -> None: + """The same text passed as a string addresses a local path, not the pipeline.""" + with pytest.raises(FileNotFoundError): + open_func(f"{tmp_path}|wrap:|zarr2:", mode="r") class TestZarrFormatMergeInCore: """ Every function taking a StoreLike honors the format selected by a `URLPipeline`, not only the top-level zarr.api functions. Functions whose - `zarr_format` defaults to 3 need an explicit `zarr_format=None` to defer to - the pipeline, and raise when the default conflicts with it. + `zarr_format` defaults to 3 raise when the default conflicts with the + pipeline; `create_array` and `Group.open` accept `zarr_format=None` to defer + to it. """ @pytest.fixture @@ -936,7 +943,7 @@ async def test_from_array(self, v2_pipeline: URLPipeline) -> None: assert arr.metadata.zarr_format == 2 async def test_group_from_store(self, v2_pipeline: URLPipeline, tmp_path: Path) -> None: - group = zarr.Group.from_store(v2_pipeline, zarr_format=None) + group = zarr.Group.from_store(v2_pipeline, zarr_format=2) assert group.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): zarr.Group.from_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}/other|zarr2:")) @@ -948,7 +955,7 @@ async def test_group_from_store_default_without_pipeline(self, tmp_path: Path) - assert group.metadata.zarr_format == 3 async def test_group_open(self, v2_pipeline: URLPipeline) -> None: - zarr.Group.from_store(v2_pipeline, zarr_format=None) + zarr.Group.from_store(v2_pipeline, zarr_format=2) group = zarr.Group.open(v2_pipeline, zarr_format=None) assert group.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): @@ -956,11 +963,17 @@ async def test_group_open(self, v2_pipeline: URLPipeline) -> None: async def test_array_open(self, v2_pipeline: URLPipeline, tmp_path: Path) -> None: zarr.create_array(v2_pipeline, name="x", shape=(2,), dtype="i4", zarr_format=None) - array_pipeline = URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:x") - arr = zarr.Array.open(array_pipeline, zarr_format=None) - assert arr.metadata.zarr_format == 2 + # Array.open opens zarr v3 only, so a v2 pipeline conflicts with it with pytest.raises(ValueError, match="conflicts with"): - zarr.Array.open(array_pipeline) + zarr.Array.open(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:x")) + zarr.create_array( + URLPipeline.from_url(f"file:{tmp_path.as_posix()}/v3|zarr3:"), + name="y", + shape=(2,), + dtype="i4", + ) + arr = zarr.Array.open(URLPipeline.from_url(f"file:{tmp_path.as_posix()}/v3|zarr3:y")) + assert arr.metadata.zarr_format == 3 async def test_open_like(self, v2_pipeline: URLPipeline) -> None: # open_like inherits v2 filters/compressor from the reference; the From 662cc692788713810acbc8189bb30ded5226bbdd Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 22 Oct 2026 21:52:54 +0200 Subject: [PATCH 15/20] fix(url-pipeline): validate URLPipeline.from_url input `from_url` failed with AttributeError on non-strings and silently accepted input no URL should contain. - Raise TypeError for anything but a `str`, and for a hand-built `URLPipeline` whose segments are not a tuple of `PipelineSegment`. - Raise URLPipelineError for ASCII control characters, whitespace at either end of a sub-URL, and a bare Windows drive root (`C:\x`, `C:/x`), which used to parse as scheme `c`. Spaces inside a sub-URL stay allowed, since zarr does not percent-decode. Co-Authored-By: Claude Opus 5.5 --- changes/4192.feature.md | 4 ++ src/zarr/storage/_url_pipeline.py | 52 +++++++++++++++-- tests/test_url_pipeline/test_parser.py | 56 ++++++++++++++++--- .../test_url_pipeline/test_pipeline_object.py | 18 +++++- 4 files changed, 116 insertions(+), 14 deletions(-) diff --git a/changes/4192.feature.md b/changes/4192.feature.md index bdcd786fe6..a9f0837a45 100644 --- a/changes/4192.feature.md +++ b/changes/4192.feature.md @@ -17,6 +17,10 @@ Behavior notes: - Plain strings are never interpreted as pipelines: `zarr.open("data|x.zarr")` still addresses a local directory of that name, whatever adapters are installed. Only `URLPipeline.from_url` reads a string as a pipeline. +- `URLPipeline.from_url` raises `TypeError` for anything but a `str`, and + `URLPipelineError` for a string it cannot parse: an empty sub-URL, a root without a + URL scheme (including a bare local or Windows drive path), an invalid adapter scheme, + a fragment, an ASCII control character, or whitespace at either end of a sub-URL. - A trailing `zarr2:`/`zarr3:` segment selects the zarr format of the open or create call, and its body is the path of the node to open. A `zarr_format` argument that conflicts with it raises `ValueError`, including the default of 3 in diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py index 0dcd907d07..c9cf995c51 100644 --- a/src/zarr/storage/_url_pipeline.py +++ b/src/zarr/storage/_url_pipeline.py @@ -180,6 +180,13 @@ async def resolve_preceding(self) -> StorePath: # absolute path in a `file:` pipeline root. _WINDOWS_DRIVE_RE = re.compile(r"[A-Za-z]:[/\\]") +# A bare Windows drive path as a pipeline root (C:\data, C:/data), as +# opposed to a single-letter scheme with an authority (x://host/path). +_BARE_DRIVE_ROOT_RE = re.compile(r"[A-Za-z]:(\\|/(?!/))") + +# ASCII control characters (C0 and DEL). No sub-URL may contain them. +_CONTROL_CHAR_RE = re.compile(r"[\x00-\x1f\x7f]") + def _root_scheme(root_sub_url: str) -> str: """ @@ -187,8 +194,9 @@ def _root_scheme(root_sub_url: str) -> str: Scheme extraction happens on the raw string, so nothing `urlparse` would strip or reject can desync the detected scheme from the body. A - single letter followed by `:` is a scheme on every platform: a Windows - drive path is spelled `file:/C:/...` in a pipeline, never bare. + single letter followed by `:` is a scheme on every platform; a bare + Windows drive path (`C:\\...`, `C:/...`) is rejected before this is + called, since a pipeline spells it `file:/C:/...`. """ match = _ROOT_SCHEME_RE.match(root_sub_url) if match is None: @@ -220,15 +228,34 @@ def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]: in one. Segment text is preserved verbatim (no case or percent-encoding - normalization) except that schemes are lowercased. + normalization) except that schemes are lowercased. ASCII control + characters, and whitespace at either end of a sub-URL, are rejected + rather than stripped. """ + if not isinstance(url, str): + raise TypeError(f"a URL pipeline must be a str, not {type(url).__name__}") + if (match := _CONTROL_CHAR_RE.search(url)) is not None: + raise URLPipelineError( + f"URL pipeline contains the control character {match.group()!r}: {url!r}" + ) parts = url.split("|") if any(not part for part in parts): raise URLPipelineError(f"URL pipeline contains an empty sub-URL: {url!r}") + for part in parts: + if part != part.strip(): + raise URLPipelineError( + f"sub-URL {part!r} of the URL pipeline {url!r} has leading or trailing " + "whitespace; percent-encode it ('%20') if it is part of the URL" + ) segments: list[PipelineSegment] = [] for index, part in enumerate(parts): if index == 0: + if _BARE_DRIVE_ROOT_RE.match(part): + raise URLPipelineError( + f"the root sub-URL {part!r} of the URL pipeline {url!r} is a Windows drive " + "path; spell it as a 'file:' URL (e.g. 'file:/C:/data')." + ) scheme = _root_scheme(part) if not scheme: raise URLPipelineError( @@ -277,6 +304,8 @@ class URLPipeline: Raises ------ + TypeError + If `segments` is not a tuple of `PipelineSegment`. URLPipelineError If there are no segments, if a format segment (`zarr2:`/`zarr3:`) appears anywhere but last, or if a format segment carries a query. @@ -285,6 +314,14 @@ class URLPipeline: segments: tuple[PipelineSegment, ...] def __post_init__(self) -> None: + segments: object = self.segments # runtime check of a hand-built pipeline + if not isinstance(segments, tuple) or not all( + isinstance(segment, PipelineSegment) for segment in segments + ): + raise TypeError( + "URLPipeline.segments must be a tuple of PipelineSegment; " + "use URLPipeline.from_url to parse a string" + ) if not self.segments: raise URLPipelineError("a URL pipeline needs at least one segment") for segment in self.segments[1:-1]: @@ -305,11 +342,18 @@ def from_url(cls, url: str) -> URLPipeline: Parse a URL pipeline string. A URL without a `|` is a trivial pipeline consisting of its root alone. + Parsing is purely syntactic: nothing is resolved, and no adapter is + loaded, until the pipeline is opened. Raises ------ + TypeError + If `url` is not a `str`. URLPipelineError - If the string cannot be parsed or the resulting pipeline is invalid + If the string cannot be parsed: an empty sub-URL, a root without a URL + scheme (including a bare local or Windows drive path), an invalid + adapter scheme, a fragment, an ASCII control character, or whitespace + at either end of a sub-URL; or if the resulting pipeline is invalid (see the class docstring). """ return cls(parse_pipeline(url)) diff --git a/tests/test_url_pipeline/test_parser.py b/tests/test_url_pipeline/test_parser.py index bcc2dd3283..0bf76d7764 100644 --- a/tests/test_url_pipeline/test_parser.py +++ b/tests/test_url_pipeline/test_parser.py @@ -17,7 +17,7 @@ def test_single_root_url() -> None: assert str(segment) == "s3://bucket/key" -@pytest.mark.parametrize("url", ["/local/path", "/local/path|zip:", "\tfile:/tmp/x|zip:"]) +@pytest.mark.parametrize("url", ["/local/path", "/local/path|zip:", "relative/path|zip:"]) def test_schemeless_root_rejected(url: str) -> None: """The root sub-URL must carry a URL scheme; a bare path (or anything urlparse would not read as a scheme) is an error.""" with pytest.raises(URLPipelineError, match="URL scheme"): @@ -72,12 +72,45 @@ def test_empty_query() -> None: assert segment.query == "" -def test_single_letter_scheme_is_a_scheme_everywhere() -> None: - """A drive-letter-like prefix is a URL scheme on every platform; bare Windows paths are not pipelines.""" - (segment,) = parse_pipeline(r"C:\data\store") +@pytest.mark.parametrize("url", [r"C:\data\store", "C:/data/store|zip:", "c:\\x"]) +def test_windows_drive_root_rejected(url: str) -> None: + """A bare Windows drive path is rejected on every platform, not read as scheme `c`.""" + with pytest.raises(URLPipelineError, match="Windows drive path"): + parse_pipeline(url) + + +def test_single_letter_scheme_without_slash_is_a_scheme() -> None: + (segment,) = parse_pipeline("c:data") assert segment.scheme == "c" - assert segment.body == r"\data\store" - assert segment.raw == r"C:\data\store" + assert segment.body == "data" + + +@pytest.mark.parametrize("url", [None, 42, b"s3://b|zip:", ["s3://b"]]) +def test_non_string_rejected(url: object) -> None: + with pytest.raises(TypeError, match="must be a str"): + parse_pipeline(url) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + "url", + ["s3://b/\x00a", "s3://b/a\n|zip:", "s3://b|zip:\r\n", "s3://b|zip:\x7f", "s3://b\t|zip:"], +) +def test_control_characters_rejected(url: str) -> None: + with pytest.raises(URLPipelineError, match="control character"): + parse_pipeline(url) + + +@pytest.mark.parametrize("url", [" s3://b", "s3://b |zip:", "s3://b| zip:", "s3://b|zip:x\u00a0"]) +def test_surrounding_whitespace_rejected(url: str) -> None: + with pytest.raises(URLPipelineError, match="leading or trailing whitespace"): + parse_pipeline(url) + + +def test_interior_whitespace_is_preserved() -> None: + """zarr does not percent-decode, so a key with a space can only be spelled with one.""" + segments = parse_pipeline("s3://b/My Data.zip|zip:a b") + assert segments[0].body == "//b/My Data.zip" + assert segments[1].body == "a b" @pytest.mark.parametrize("url", ["a||b:", "|zip:", "file:/tmp|", ""]) @@ -126,8 +159,13 @@ def test_round_trip() -> None: # two or more characters, so Windows drive-letter handling cannot reclassify # the root scheme as a local path on one platform but not another _ROOT_SCHEME = st.from_regex(r"[a-zA-Z]{2}[a-zA-Z0-9+.\-]{0,14}", fullmatch=True) -_BODY = st.text(st.characters(min_codepoint=32, max_codepoint=126, exclude_characters="|#?")) -_QUERY = st.text(st.characters(min_codepoint=32, max_codepoint=126, exclude_characters="|#")) +# no trailing space: whitespace at either end of a sub-URL is rejected +_BODY = st.text( + st.characters(min_codepoint=32, max_codepoint=126, exclude_characters="|#?") +).filter(lambda t: not t.endswith(" ")) +_QUERY = st.text( + st.characters(min_codepoint=32, max_codepoint=126, exclude_characters="|#") +).filter(lambda t: not t.endswith(" ")) @st.composite @@ -180,7 +218,7 @@ def test_property_invalid_adapter_scheme_rejected( # corrupt one adapter segment's scheme with a leading character the # scheme grammar forbids index = data.draw(st.integers(1, len(parts) - 1)) - parts[index] = data.draw(st.sampled_from(["1", " ", "@", "~"])) + parts[index] + parts[index] = data.draw(st.sampled_from(["1", "@", "~"])) + parts[index] with pytest.raises(URLPipelineError, match="invalid adapter scheme"): parse_pipeline("|".join(parts)) diff --git a/tests/test_url_pipeline/test_pipeline_object.py b/tests/test_url_pipeline/test_pipeline_object.py index 78c50ec655..62282233c6 100644 --- a/tests/test_url_pipeline/test_pipeline_object.py +++ b/tests/test_url_pipeline/test_pipeline_object.py @@ -3,7 +3,7 @@ import pytest from zarr.errors import URLPipelineError -from zarr.storage import URLPipeline +from zarr.storage import PipelineSegment, URLPipeline from zarr.storage._url_pipeline import parse_pipeline @@ -81,6 +81,22 @@ def test_empty_pipeline_rejected() -> None: URLPipeline(()) +@pytest.mark.parametrize( + "segments", + ["s3://b|zip:", ["s3://b"], ("s3://b",), [PipelineSegment("s3", "//b", None, "s3://b")]], +) +def test_direct_construction_rejects_non_segments(segments: object) -> None: + """A string or a list is not a pipeline; `from_url` is the way to parse one.""" + with pytest.raises(TypeError, match="tuple of PipelineSegment"): + URLPipeline(segments) # type: ignore[arg-type] + + +@pytest.mark.parametrize("url", [None, b"s3://b", URLPipeline.from_url("s3://b")]) +def test_from_url_rejects_non_strings(url: object) -> None: + with pytest.raises(TypeError, match="must be a str"): + URLPipeline.from_url(url) # type: ignore[arg-type] + + def test_hashable() -> None: """Pipelines are values: equal pipelines hash equal and can key a dict.""" a = URLPipeline.from_url("s3://b/d.zip|zip:|zarr3:") From 1a1275fe0dab3ea8fcbcdb6398c96a5edc4490a4 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 22 Oct 2026 22:46:27 +0200 Subject: [PATCH 16/20] fix(url-pipeline): address review findings on resolution and validation Validation and resolution: - make_store_path rejects an invalid mode before anything is resolved, so adapters never see one, and closes the adapter's store when opening the StorePath fails (read-only store in mode "a", "w-" on data, ...). - consolidate_metadata honors the pipeline's format segment. - A format scheme cannot be the root, and a format segment's node path is validated at construction: '.'/'..' components are rejected before any I/O. - Support the spec's `zarr:` segment (node path, format detected). - file: roots: reject '//' paths, which urlparse misread as an authority; reject '.'/'..' components; resolve via Path rather than a reconstructed URL; drive letters are special only on Windows. - Non-native roots go to fsspec with the lowercased scheme. - memory: roots follow the spec: one in-memory store, body is a path. - PipelineSegment validates field types and requires a lowercase, valid scheme; register_url_adapter rejects invalid schemes and the ones zarr handles itself (file, memory, zarr, zarr2, zarr3). Messages: no percent-encoding advice (nothing is decoded), a readable URLPipeline repr, and "invalid URL scheme" for e.g. `a_b://x`. Tests in test_open_path.py exercise these through zarr.open_*, make_store_path, make_store and consolidate_metadata. Co-Authored-By: Claude Opus 5.5 --- changes/4192.feature.md | 18 +- docs/user-guide/storage.md | 26 +- src/zarr/api/asynchronous.py | 1 + src/zarr/registry.py | 16 ++ src/zarr/storage/_common.py | 14 +- src/zarr/storage/_url_pipeline.py | 132 ++++++++-- tests/test_url_pipeline/test_open_path.py | 298 ++++++++++++++++++++++ tests/test_url_pipeline/test_resolver.py | 38 +-- 8 files changed, 479 insertions(+), 64 deletions(-) create mode 100644 tests/test_url_pipeline/test_open_path.py diff --git a/changes/4192.feature.md b/changes/4192.feature.md index a9f0837a45..5266b3d678 100644 --- a/changes/4192.feature.md +++ b/changes/4192.feature.md @@ -20,19 +20,25 @@ Behavior notes: - `URLPipeline.from_url` raises `TypeError` for anything but a `str`, and `URLPipelineError` for a string it cannot parse: an empty sub-URL, a root without a URL scheme (including a bare local or Windows drive path), an invalid adapter scheme, - a fragment, an ASCII control character, or whitespace at either end of a sub-URL. + a fragment, an ASCII control character, whitespace at either end of a sub-URL, a + format segment as the root, or a format-segment node path with `.` or `..` components. - A trailing `zarr2:`/`zarr3:` segment selects the zarr format of the open or create - call, and its body is the path of the node to open. A `zarr_format` argument that + call, and its body is the path of the node to open; a trailing `zarr:` segment gives + the node path and leaves the format to be detected. A `zarr_format` argument that conflicts with it raises `ValueError`, including the default of 3 in `zarr.create_array`, `Group.from_store`, `Group.open` and `Array.open`: pass the matching format (or `zarr_format=None` where it is accepted) to open a `zarr2:` pipeline with these. -- Inside a pipeline, `memory:` and `file:` roots follow the URL pipeline spec's spelling - rules (`memory:` ≡ `memory:/` ≡ `memory://`; `file:` must carry an absolute path with - at most a `localhost` authority). Percent-escapes are not decoded, matching - `LocalStore`. A pipeline's `memory:` root is zarr's `ManagedMemoryStore`, which does +- Inside a pipeline, `memory:` and `file:` roots follow the URL pipeline spec. Every + `memory:` root addresses one in-memory store and its body is a path within it + (`memory:a` ≡ `memory:/a` ≡ `memory://a`). A `file:` root must carry an absolute path + with at most a `localhost` authority; drive letters are only special on Windows. + Neither accepts `.` or `..` path components. Percent-escapes are not decoded, matching + `LocalStore`. A pipeline's `memory:` root is a zarr `ManagedMemoryStore`, which does not share data with the fsspec in-memory filesystem a plain `memory://name` string resolves to. +- `register_url_adapter` raises `ValueError` for an invalid URL scheme and for the + schemes zarr-python handles itself (`file`, `memory`, `zarr`, `zarr2`, `zarr3`). - Wrapper adapters can open a local *file* root only in mode `"r"` (via `dataclasses.replace(context, mode="r").resolve_preceding()`); writing through a container file is not yet supported. diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index fe568e8df3..ccadc80815 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -137,23 +137,27 @@ still names a directory, whatever adapters are installed. A trailing `zarr2:` or `zarr3:` segment is a *format segment*. It is not an adapter: zarr-python reads it from the pipeline itself and uses it as the `zarr_format` of the open or create call, and its body (`zarr3:path/to/node`) is the path of the node to -open. A `zarr_format` argument that conflicts with the format segment raises +open. The node path must not contain `.` or `..` components. A trailing `zarr:` segment +also gives a node path, but leaves the format to be detected. A `zarr_format` argument that conflicts with the format segment raises `ValueError`; functions whose `zarr_format` defaults to 3 (such as `zarr.create_array` and `Group.open`) need an explicit `zarr_format=None` (or the matching format) to defer to the pipeline. `storage_options` apply to the *root* sub-URL (e.g. fsspec options for `s3://...`); adapters may consume adapter-specific, namespaced keys. No percent-escape is decoded -anywhere in a pipeline. Registered adapters cannot intercept zarr's native `file:` and -`memory:` root schemes. - -Inside a pipeline, a `memory:` root is zarr's managed in-memory store (`memory:`, -`memory:/` and `memory://` are equivalent, and `memory:name` selects a named store). -When fsspec is installed, a plain `memory://name` string store specification is still -routed to fsspec's in-memory filesystem, which is a different store. A `file:` root -must carry an absolute path; percent-escapes are not decoded, matching the -[local store](#local-store). The root always carries a scheme: a local path is spelled -as a `file:` URL, never as a bare path. +anywhere in a pipeline. Adapters cannot be registered for the schemes zarr-python +handles itself: `file`, `memory`, `zarr`, `zarr2` and `zarr3`. + +Inside a pipeline, every `memory:` root addresses the same managed in-memory store, and +its body is a path within that store: `memory:a/b`, `memory:/a/b` and `memory://a/b` are +equivalent, as are `memory:`, `memory:/` and `memory://`. Data lives only while some +store object referring to it is alive. When fsspec is installed, a plain `memory://name` +string store specification is still routed to fsspec's in-memory filesystem, which is a +different store. A `file:` root must carry an absolute path, with at most a `localhost` +authority; percent-escapes are not decoded, matching the [local store](#local-store). +Drive letters are only special on Windows: elsewhere `file:/C:/data` is the absolute +path `/C:/data`. Neither root accepts `.` or `..` path components. The root always +carries a scheme: a local path is spelled as a `file:` URL, never as a bare path. ## Explicit Store Creation diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index f873c777d6..a4c254c1be 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -234,6 +234,7 @@ async def consolidate_metadata( consolidated metadata, this function raises a `TypeError`. See `Store.supports_consolidated_metadata`. """ + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, path=path) if not store_path.store.supports_consolidated_metadata: diff --git a/src/zarr/registry.py b/src/zarr/registry.py index 892e728da0..a568f97347 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -518,8 +518,24 @@ class for a URL scheme. Registering a scheme that already has an adapter replaces it and emits a [`ZarrUserWarning`][zarr.errors.ZarrUserWarning]. + + Raises + ------ + ValueError + If `scheme` is not a valid URL scheme, or is one zarr-python handles + itself (`file`, `memory`, `zarr`, `zarr2`, `zarr3`), so an adapter + registered for it could never be used. """ + from zarr.storage._url_pipeline import _RESERVED_SCHEMES, _SCHEME_RE # circular import + + if not _SCHEME_RE.match(scheme): + raise ValueError(f"{scheme!r} is not a valid URL scheme") key = scheme.lower() + if key in _RESERVED_SCHEMES: + raise ValueError( + f"the URL scheme {scheme!r} is handled by zarr-python itself and cannot be " + "given to a URL pipeline adapter" + ) previous = _url_adapter_registry.get(key) if previous is not None and previous is not cls: warnings.warn( diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index be84dd075a..b869bb1233 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -361,7 +361,7 @@ async def make_store( if isinstance(store_like, URLPipeline): result = await store_like._resolve_store_path(mode=mode, storage_options=storage_options) - if normalize_path(result.path): + if result.path: result.store.close() raise URLPipelineError( f"the URL pipeline {str(store_like)!r} resolves to a path inside a store; " @@ -483,12 +483,20 @@ async def make_store_path( -------- make_store """ + if mode is not None and mode not in ANY_ACCESS_MODE: + raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") path_normalized = normalize_path(path) if isinstance(store_like, URLPipeline): result = await store_like._resolve_store_path(mode=mode, storage_options=storage_options) - combined_path = _join_paths([normalize_path(result.path), path_normalized]) - return await StorePath.open(result.store, path=combined_path, mode=mode) + try: + return await StorePath.open( + result.store, path=_join_paths([result.path, path_normalized]), mode=mode + ) + except BaseException: + # the store was opened for this call; don't leak it + result.store.close() + raise if isinstance(store_like, StorePath): # Already a StorePath diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py index c9cf995c51..1fc8c5438c 100644 --- a/src/zarr/storage/_url_pipeline.py +++ b/src/zarr/storage/_url_pipeline.py @@ -18,6 +18,8 @@ import dataclasses import re +import sys +from pathlib import Path from typing import TYPE_CHECKING, Any from zarr.errors import URLPipelineError @@ -59,6 +61,22 @@ class PipelineSegment: query: str | None raw: str + def __post_init__(self) -> None: + for name in ("scheme", "body", "raw"): + if not isinstance(getattr(self, name), str): + raise TypeError( + f"PipelineSegment.{name} must be a str, " + f"not {type(getattr(self, name)).__name__}" + ) + if self.query is not None and not isinstance(self.query, str): + raise TypeError( + f"PipelineSegment.query must be a str or None, not {type(self.query).__name__}" + ) + if not _SCHEME_RE.match(self.scheme) or self.scheme != self.scheme.lower(): + raise URLPipelineError( + f"PipelineSegment.scheme must be a lowercase URL scheme, not {self.scheme!r}" + ) + def __str__(self) -> str: return self.raw @@ -161,8 +179,8 @@ async def resolve_preceding(self) -> StorePath: # Format segments are consumed by zarr-python itself rather than by a # registered adapter: they select the zarr format of the node the pipeline -# addresses, and their body is a path to that node. -_FORMAT_SCHEMES: dict[str, ZarrFormat] = {"zarr2": 2, "zarr3": 3} +# addresses (`zarr:` detects it), and their body is a path to that node. +_FORMAT_SCHEMES: dict[str, ZarrFormat | None] = {"zarr": None, "zarr2": 2, "zarr3": 3} # Adapter scheme per RFC 3986 plus "." to permit vendor-prefixed # nonstandard schemes (e.g. "earthmover.myscheme"). @@ -171,11 +189,19 @@ async def resolve_preceding(self) -> StorePath: # Root scheme at the very start of the sub-URL. _ROOT_SCHEME_RE = re.compile(r"([a-zA-Z][a-zA-Z0-9+.\-]*):") +# Something shaped like a scheme (text up to the first ':' with no '/', '?' +# or '#' before it) that is not a valid one, e.g. `a_b://x`. +_SCHEME_LIKE_RE = re.compile(r"[^:/?#]+:") + # Schemes zarr resolves natively at the pipeline root. These are never # dispatched to a registered root adapter, so an installed package cannot # intercept zarr's own local-path and in-memory routing. _NATIVE_ROOT_SCHEMES = frozenset({"file", "memory"}) +# Schemes zarr-python handles itself, which therefore cannot be given to a +# registered adapter. +_RESERVED_SCHEMES = _NATIVE_ROOT_SCHEMES | frozenset(_FORMAT_SCHEMES) + # An absolute Windows drive path (C:\... or C:/...), which counts as an # absolute path in a `file:` pipeline root. _WINDOWS_DRIVE_RE = re.compile(r"[A-Za-z]:[/\\]") @@ -208,8 +234,8 @@ def _split_query(sub_url: str) -> tuple[str, str | None]: """Split a sub-URL on the first `?`. Fragments are not supported.""" if "#" in sub_url: raise URLPipelineError( - f"URL pipeline sub-URLs do not support fragments: {sub_url!r}. " - "Percent-encode '#' as '%23' if it is part of the path." + f"URL pipeline sub-URLs do not support fragments ('#'): {sub_url!r}. " + "Percent-escapes are not decoded, so a '#' in a path cannot be addressed." ) body, sep, query = sub_url.partition("?") return body, query if sep else None @@ -245,7 +271,7 @@ def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]: if part != part.strip(): raise URLPipelineError( f"sub-URL {part!r} of the URL pipeline {url!r} has leading or trailing " - "whitespace; percent-encode it ('%20') if it is part of the URL" + "whitespace; remove it" ) segments: list[PipelineSegment] = [] @@ -257,11 +283,18 @@ def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]: "path; spell it as a 'file:' URL (e.g. 'file:/C:/data')." ) scheme = _root_scheme(part) + if not scheme and _SCHEME_LIKE_RE.match(part): + invalid = part.partition(":")[0] + raise URLPipelineError( + f"invalid URL scheme {invalid!r} in the root sub-URL {part!r} " + f"of the URL pipeline {url!r}" + ) if not scheme: raise URLPipelineError( f"the root sub-URL {part!r} of the URL pipeline {url!r} has no URL scheme. " - "Spell a local path as an absolute 'file:' URL, and a literal '|' in " - "it as '%7C'." + "Spell a local path as an absolute 'file:' URL. Percent-escapes are not " + "decoded, so a local path containing '|' cannot be addressed by a URL " + "pipeline; open it with a plain path or a store object instead." ) body_and_scheme, query = _split_query(part) body = body_and_scheme[len(scheme) + 1 :] @@ -307,8 +340,9 @@ class URLPipeline: TypeError If `segments` is not a tuple of `PipelineSegment`. URLPipelineError - If there are no segments, if a format segment (`zarr2:`/`zarr3:`) - appears anywhere but last, or if a format segment carries a query. + If there are no segments, if a format segment (`zarr:`/`zarr2:`/`zarr3:`) + appears anywhere but last (including as the root), or if a format segment + carries a query or a node path with `.` or `..` components. """ segments: tuple[PipelineSegment, ...] @@ -324,6 +358,11 @@ def __post_init__(self) -> None: ) if not self.segments: raise URLPipelineError("a URL pipeline needs at least one segment") + if self.segments[0].scheme in _FORMAT_SCHEMES: + raise URLPipelineError( + f"the format segment {self.segments[0].raw!r} cannot be the root of " + f"the URL pipeline {str(self)!r}; the root must locate a resource" + ) for segment in self.segments[1:-1]: if segment.scheme in _FORMAT_SCHEMES: raise URLPipelineError( @@ -335,6 +374,17 @@ def __post_init__(self) -> None: raise URLPipelineError( f"'{format_segment.scheme}:' segments do not accept a query: {format_segment.raw!r}" ) + if format_segment is not None: + # validate the node path now, before anything is resolved + try: + normalize_path(format_segment.body) + except ValueError as exc: + raise URLPipelineError( + f"invalid node path in the format segment {format_segment.raw!r}: {exc}" + ) from None + + def __repr__(self) -> str: + return f"URLPipeline({str(self)!r})" @classmethod def from_url(cls, url: str) -> URLPipeline: @@ -377,7 +427,8 @@ def _store_segments(self) -> tuple[PipelineSegment, ...]: @property def _zarr_format(self) -> ZarrFormat | None: - """The format selected by a trailing `zarr2:`/`zarr3:` segment, or None.""" + """The format selected by a trailing `zarr2:`/`zarr3:` segment, or None + (also for a trailing `zarr:` segment, which detects the format).""" format_segment = self._format_segment if format_segment is None: return None @@ -467,9 +518,9 @@ async def _resolve_store_path( resolved = await _resolve(self._store_segments, mode=mode, storage_options=storage_options) if not self._node_path: return resolved - return StorePath( - resolved.store, _join_paths([normalize_path(resolved.path), self._node_path]) - ) + # both paths are already normalized: StorePath normalizes the adapter's, + # and __post_init__ validated the format segment's + return StorePath(resolved.store, _join_paths([resolved.path, self._node_path])) def __str__(self) -> str: return "|".join(segment.raw for segment in self.segments) @@ -561,24 +612,50 @@ async def _resolve_root( "'localhost' is allowed" ) body = f"/{rest}" if sep else "" - # a file URL spells a Windows drive path as /C:/...; strip the - # leading slash so the local-path machinery sees the drive - if body.startswith("/") and _WINDOWS_DRIVE_RE.match(body[1:]): - body = body[1:] - if not (body.startswith("/") or _WINDOWS_DRIVE_RE.match(body)): + if body.startswith("//"): + # the spec's file-path-absolute grammar forbids an empty first + # path component + raise URLPipelineError( + f"'file:' pipeline root paths must not start with '//': {segment.raw!r}" + ) + if sys.platform == "win32": + # a file URL spells a Windows drive path as /C:/...; strip the + # leading slash so the local-path machinery sees the drive + if body.startswith("/") and _WINDOWS_DRIVE_RE.match(body[1:]): + body = body[1:] + is_absolute = body.startswith("/") or bool(_WINDOWS_DRIVE_RE.match(body)) + else: + is_absolute = body.startswith("/") + if not is_absolute: raise URLPipelineError( f"'file:' pipeline roots must carry an absolute path: {segment.raw!r}" ) - return await make_store(f"file:{body}", mode=mode, storage_options=storage_options) + _reject_dot_segments(body, segment) + return await make_store(Path(body), mode=mode, storage_options=storage_options) + # fsspec matches protocols case-sensitively; schemes are case-insensitive + url = segment.scheme + segment.raw[len(segment.scheme) :] try: - return await make_store(segment.raw, mode=mode, storage_options=storage_options) + return await make_store(url, mode=mode, storage_options=storage_options) except ValueError as exc: raise URLPipelineError( f"could not resolve the pipeline root {segment.raw!r}: {exc}" ) from exc +def _reject_dot_segments(path: str, segment: PipelineSegment) -> None: + """Reject `.` and `..` path components in a `file:` or `memory:` root.""" + if any(part in (".", "..") for part in re.split(r"[/\\]", path)): + raise URLPipelineError( + f"'{segment.scheme}:' pipeline root paths must not contain '.' or '..' " + f"components: {segment.raw!r}" + ) + + +# The single in-memory store every `memory:` pipeline root addresses. +_PIPELINE_MEMORY_STORE_NAME = "" + + def _resolve_memory_root( segment: PipelineSegment, *, @@ -586,17 +663,18 @@ def _resolve_memory_root( storage_options: dict[str, Any] | None, ) -> Store: """ - Resolve a `memory:` pipeline root per the spec's spelling equivalences: - `memory:` ≡ `memory:/` ≡ `memory://`, and `memory:a` ≡ `memory:/a` ≡ - `memory://a`. The first path component names the managed store; the - remainder is a path within it. + Resolve a `memory:` pipeline root per the spec: there is a single + in-memory store, and the body is a path within it. Spellings follow the + spec's equivalences: `memory:` ≡ `memory:/` ≡ `memory://`, and + `memory:a/b` ≡ `memory:/a/b` ≡ `memory://a/b`. """ if segment.query is not None: raise URLPipelineError(f"'memory:' pipeline roots do not accept a query: {segment.raw!r}") - if storage_options: + if storage_options is not None: raise TypeError( "'storage_options' was provided but unused. " "'storage_options' is only used when the store is passed as an FSSpec URI string.", ) - name, _, path = segment.body.lstrip("/").partition("/") - return ManagedMemoryStore(name=name, path=path, read_only=mode == "r") + path = segment.body.lstrip("/") + _reject_dot_segments(path, segment) + return ManagedMemoryStore(name=_PIPELINE_MEMORY_STORE_NAME, path=path, read_only=mode == "r") diff --git a/tests/test_url_pipeline/test_open_path.py b/tests/test_url_pipeline/test_open_path.py new file mode 100644 index 0000000000..8e294d273f --- /dev/null +++ b/tests/test_url_pipeline/test_open_path.py @@ -0,0 +1,298 @@ +""" +URL pipelines through the public open path: `zarr.open_*`, `make_store_path`, +`make_store` and `consolidate_metadata`, rather than the private resolver. +""" + +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING, ClassVar + +import pytest + +import zarr +import zarr.registry +from zarr.abc.url_pipeline import URLPipelineAdapter +from zarr.errors import GroupNotFoundError, URLPipelineError, ZarrUserWarning +from zarr.registry import register_url_adapter +from zarr.storage import MemoryStore, PipelineSegment, StorePath, URLPipeline +from zarr.storage._common import make_store, make_store_path + +if TYPE_CHECKING: + from pathlib import Path + + from zarr.storage import PipelineContext + +pytestmark = pytest.mark.usefixtures("clean_url_adapter_registry") + + +class ClosingStore(MemoryStore): + """A memory store that records how often it is closed.""" + + closes: ClassVar[list[ClosingStore]] = [] + + def close(self) -> None: + ClosingStore.closes.append(self) + super().close() + + +class RecordingAdapter(URLPipelineAdapter): + """Records each call; returns a fresh `ClosingStore` honoring `read_only`.""" + + calls: ClassVar[list[PipelineContext]] = [] + read_only_store: ClassVar[bool] = False + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> StorePath: + cls.calls.append(context) + store = await ClosingStore.open(read_only=cls.read_only_store or context.read_only) + return StorePath(store, segment.body) + + +@pytest.fixture +def recording() -> type[RecordingAdapter]: + RecordingAdapter.calls = [] + RecordingAdapter.read_only_store = False + ClosingStore.closes = [] + register_url_adapter("rec", RecordingAdapter) + return RecordingAdapter + + +class TestModeValidation: + @pytest.mark.parametrize("mode", ["bogus", "R", "rw"]) + async def test_invalid_mode_rejected_before_adapters_run( + self, recording: type[RecordingAdapter], mode: str + ) -> None: + with pytest.raises(ValueError, match="Invalid mode"): + zarr.open_group(URLPipeline.from_url("memory:|rec:"), mode=mode) # type: ignore[arg-type] + with pytest.raises(ValueError, match="Invalid mode"): + await make_store_path(URLPipeline.from_url("memory:|rec:"), mode=mode) # type: ignore[arg-type] + assert recording.calls == [] + + async def test_invalid_mode_rejected_for_plain_stores(self) -> None: + with pytest.raises(ValueError, match="Invalid mode"): + await make_store_path({}, mode="bogus") # type: ignore[arg-type] + + +class TestStoresClosedOnError: + async def test_read_only_store_in_mode_a(self, recording: type[RecordingAdapter]) -> None: + recording.read_only_store = True + with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"): + zarr.open_group(URLPipeline.from_url("memory:|rec:"), mode="a") + assert len(ClosingStore.closes) == 1 + + async def test_mode_w_minus_on_existing_data(self, recording: type[RecordingAdapter]) -> None: + class NonEmptyAdapter(RecordingAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> StorePath: + store_path = await super().open_pipeline_segment(segment, context) + zarr.create_group(store_path.store) + return store_path + + register_url_adapter("full", NonEmptyAdapter) + with pytest.raises(FileExistsError): + await make_store_path(URLPipeline.from_url("memory:|full:"), mode="w-") + assert len(ClosingStore.closes) == 1 + + async def test_make_store_residual_path(self, recording: type[RecordingAdapter]) -> None: + with pytest.raises(URLPipelineError, match="path inside a store"): + await make_store(URLPipeline.from_url("memory:|rec:sub")) + assert len(ClosingStore.closes) == 1 + + async def test_success_does_not_close(self, recording: type[RecordingAdapter]) -> None: + zarr.open_group(URLPipeline.from_url("memory:|rec:"), mode="w") + assert ClosingStore.closes == [] + + +class TestConsolidateMetadata: + async def test_honors_format_segment(self, tmp_path: Path) -> None: + root = tmp_path.as_posix() + zarr.create_group(tmp_path, zarr_format=3).create_array("x", shape=(2,), dtype="i4") + # a zarr2: pipeline over v3 data finds no v2 group + with pytest.raises(FileNotFoundError): + zarr.consolidate_metadata(URLPipeline.from_url(f"file:{root}|zarr2:")) + with pytest.warns(ZarrUserWarning, match="Consolidated metadata"): + group = zarr.consolidate_metadata(URLPipeline.from_url(f"file:{root}|zarr3:")) + assert group.metadata.consolidated_metadata is not None + + async def test_conflicting_format_raises(self, tmp_path: Path) -> None: + zarr.create_group(tmp_path, zarr_format=2) + with pytest.raises(ValueError, match="conflicts with"): + zarr.consolidate_metadata( + URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:"), zarr_format=3 + ) + + +class TestFormatSegments: + @pytest.mark.parametrize("url", ["zarr3:", "zarr2:/data|zarr3:", "zarr:x|rec:", "ZARR3:"]) + def test_format_scheme_cannot_be_root(self, url: str) -> None: + with pytest.raises(URLPipelineError, match="cannot be the root"): + URLPipeline.from_url(url) + + @pytest.mark.parametrize("body", ["../x", "a/../../x", "./x", "a/./b"]) + def test_node_path_dot_segments_rejected_at_parse(self, body: str) -> None: + with pytest.raises(URLPipelineError, match="invalid node path"): + URLPipeline.from_url(f"memory:|zarr3:{body}") + + async def test_node_path_traversal_creates_nothing(self, tmp_path: Path) -> None: + target = tmp_path / "newdir" + with pytest.raises(URLPipelineError): + zarr.open_group(URLPipeline.from_url(f"file:{target.as_posix()}|zarr3:../x"), mode="w") + assert not target.exists() + + async def test_zarr_segment_detects_format(self, tmp_path: Path) -> None: + root = tmp_path.as_posix() + zarr.create_group(URLPipeline.from_url(f"file:{root}|zarr2:g"), zarr_format=2) + group = zarr.open_group(URLPipeline.from_url(f"file:{root}|zarr:g"), mode="r") + assert group.metadata.zarr_format == 2 + assert group.path == "g" + # no format is selected, so a default zarr_format=3 does not conflict + zarr.create_array( + URLPipeline.from_url(f"file:{root}|zarr:"), name="a", shape=(2,), dtype="i4" + ) + assert ( + zarr.open_array(URLPipeline.from_url(f"file:{root}|zarr:a")).metadata.zarr_format == 3 + ) + + def test_zarr_segment_must_be_last(self) -> None: + with pytest.raises(URLPipelineError, match="must be the last segment"): + URLPipeline.from_url("memory:|zarr:|rec:") + + +class TestFileRoots: + @pytest.mark.parametrize( + "url", ["file:////data", "file:////nonexistent/data", "file://localhost//data"] + ) + async def test_double_slash_path_rejected( + self, url: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.chdir(tmp_path) + with pytest.raises(URLPipelineError, match="must not start with '//'"): + zarr.open_group(URLPipeline.from_url(url), mode="w") + assert list(tmp_path.iterdir()) == [] + + @pytest.mark.parametrize("suffix", ["/../x", "/./x", "/a/../../x"]) + async def test_dot_segments_rejected(self, suffix: str, tmp_path: Path) -> None: + with pytest.raises(URLPipelineError, match=r"'\.' or '\.\.'"): + zarr.open_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}{suffix}"), mode="w") + + async def test_root_opens_absolute_local_path(self, tmp_path: Path) -> None: + group = zarr.open_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}/g"), mode="w") + group.attrs["x"] = 1 + assert (tmp_path / "g" / "zarr.json").exists() + + async def test_storage_options_rejected(self, tmp_path: Path) -> None: + with pytest.raises(TypeError, match="'storage_options' was provided but unused"): + zarr.open_group( + URLPipeline.from_url(f"file:{tmp_path.as_posix()}"), + mode="w", + storage_options={"anon": True}, + ) + + +class TestMemoryRoot: + async def test_single_store_body_is_a_path(self) -> None: + """`memory:n/a/b`, `memory:n|...a/b` and `memory:|...n/a/b` address the same node.""" + + class PathAdapter(URLPipelineAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> StorePath: + preceding = await context.resolve_preceding() + return StorePath(preceding.store, f"{preceding.path}/{segment.body}".strip("/")) + + register_url_adapter("sub", PathAdapter) + # keep the writer alive: managed memory stores drop their data once unreferenced + writer = zarr.open_group(URLPipeline.from_url("memory:mem-one/a/b|sub:"), mode="w") + writer.attrs["k"] = 1 + for url in ("memory:mem-one|sub:a/b", "memory:|sub:mem-one/a/b", "memory://mem-one/a/b"): + assert zarr.open_group(URLPipeline.from_url(url), mode="r").attrs["k"] == 1 + + @pytest.mark.parametrize("body", ["../x", "a/../b", "./a"]) + async def test_dot_segments_rejected(self, body: str) -> None: + with pytest.raises(URLPipelineError, match=r"'\.' or '\.\.'"): + await make_store_path(URLPipeline.from_url(f"memory:{body}")) + + async def test_empty_storage_options_rejected(self) -> None: + with pytest.raises(TypeError, match="'storage_options' was provided but unused"): + await make_store_path(URLPipeline.from_url("memory:x"), storage_options={}) + + +class TestRootSchemeCase: + async def test_uppercase_native_root(self, tmp_path: Path) -> None: + group = zarr.open_group(URLPipeline.from_url(f"FILE:{tmp_path.as_posix()}"), mode="w") + assert isinstance(group, zarr.Group) + + async def test_uppercase_fsspec_root(self) -> None: + """fsspec matches protocols case-sensitively, so the root gets the lowercased scheme.""" + pytest.importorskip("fsspec") + pytest.importorskip("aiohttp") + from zarr.storage import FsspecStore + + store_path = await make_store_path(URLPipeline.from_url("HTTPS://example.com/d")) + assert isinstance(store_path.store, FsspecStore) + assert store_path.store.path.startswith("https://") or "example.com" in str( + store_path.store + ) + + +class TestRegisterURLAdapter: + @pytest.mark.parametrize("scheme", ["", "a b", "1abc", "a_b", "zip:"]) + def test_invalid_scheme_rejected(self, scheme: str) -> None: + with pytest.raises(ValueError, match="not a valid URL scheme"): + register_url_adapter(scheme, RecordingAdapter) + + @pytest.mark.parametrize("scheme", ["file", "memory", "zarr", "zarr2", "ZARR3"]) + def test_reserved_scheme_rejected(self, scheme: str) -> None: + with pytest.raises(ValueError, match="handled by zarr-python itself"): + register_url_adapter(scheme, RecordingAdapter) + + def test_vendor_scheme_accepted(self) -> None: + register_url_adapter("Vendor.My-Scheme+x", RecordingAdapter) + assert zarr.registry._get_url_adapter("vendor.my-scheme+x") is RecordingAdapter + + +class TestSegmentValidation: + @pytest.mark.parametrize( + ("fields", "error"), + [ + ((1, "b", None, "r"), TypeError), + (("s", 2, None, "r"), TypeError), + (("s", "b", 3, "r"), TypeError), + (("s", "b", None, None), TypeError), + (("S3", "b", None, "r"), URLPipelineError), + (("", "b", None, "r"), URLPipelineError), + (("a b", "b", None, "r"), URLPipelineError), + ], + ) + def test_invalid_fields(self, fields: tuple[object, ...], error: type[Exception]) -> None: + with pytest.raises(error): + PipelineSegment(*fields) # type: ignore[arg-type] + + +class TestMessages: + def test_repr_is_the_url(self) -> None: + assert repr(URLPipeline.from_url("memory:|zarr3:a")) == "URLPipeline('memory:|zarr3:a')" + + def test_invalid_root_scheme_message(self) -> None: + with pytest.raises(URLPipelineError, match="invalid URL scheme 'a_b'"): + URLPipeline.from_url("a_b://x") + + @pytest.mark.parametrize("url", ["/a|b", "file:/a#b", " s3://b"]) + def test_no_percent_encoding_advice(self, url: str) -> None: + with pytest.raises(URLPipelineError) as info: + URLPipeline.from_url(url) + assert "%7C" not in str(info.value) + assert "%23" not in str(info.value) + assert "%20" not in str(info.value) + + @pytest.mark.skipif(sys.platform == "win32", reason="'|' is not allowed in Windows file names") + async def test_group_not_found_message_is_readable(self, tmp_path: Path) -> None: + with pytest.raises(GroupNotFoundError) as info: + zarr.open_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr3:"), mode="r") + assert "PipelineSegment(" not in str(info.value) diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index 3460b4b87e..3f2825412e 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -347,9 +347,11 @@ async def open_pipeline_segment( assert calls == [] def test_native_schemes_do_not_route_to_adapters(self) -> None: - """An installed package cannot intercept `file:`/`memory:` roots by registering them.""" - register_url_adapter("file", RootAdapter) - register_url_adapter("memory", RootAdapter) + """An installed package cannot intercept `file:`/`memory:` roots, even via the registry.""" + # register_url_adapter refuses these schemes; an entry point could still + # name them, so put them in the registry directly + zarr.registry._url_adapter_registry.register(RootAdapter, "file") + zarr.registry._url_adapter_registry.register(RootAdapter, "memory") pipeline = URLPipeline.from_url("memory://x") store = zarr.core.sync.sync(make_store(pipeline)) assert isinstance(store, ManagedMemoryStore) @@ -616,14 +618,15 @@ async def test_memory_root_spellings_equivalent(self, root: str) -> None: assert inner.path == "" @pytest.mark.parametrize("root", ["memory:a/b", "memory:/a/b", "memory://a/b"]) - async def test_memory_root_named_spellings_equivalent(self, root: str) -> None: + async def test_memory_root_path_spellings_equivalent(self, root: str) -> None: + """There is one in-memory store; the whole body is a path within it.""" register_url_adapter("wrap", WrapperAdapter) result = await URLPipeline.from_url(f"{root}|wrap:")._resolve_store_path() assert isinstance(result.store, TracingStore) inner = result.store._store assert isinstance(inner, ManagedMemoryStore) - assert inner._name == "a" - assert inner.path == "b" + assert inner._name == "" + assert inner.path == "a/b" async def test_memory_root_shares_data_across_spellings(self) -> None: register_url_adapter("wrap", WrapperAdapter) @@ -655,24 +658,25 @@ async def test_file_root_localhost_authority(self, tmp_path: Path) -> None: assert isinstance(opened, zarr.Group) zarr.open_group(URLPipeline.from_url(f"file://{url_path}|wrap:"), mode="r") - async def test_file_root_windows_drive_spellings( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - # Windows drive paths count as absolute, and the URL spelling + @pytest.mark.skipif(sys.platform != "win32", reason="drive paths are Windows-only") + async def test_file_root_windows_drive_spellings(self, tmp_path: Path) -> None: + # on Windows, drive paths count as absolute, and the URL spelling # file:///C:/... has its leading slash stripped before the drive register_url_adapter("wrap", WrapperAdapter) - if sys.platform == "win32": - drive_path = tmp_path.as_posix() # C:/Users/... - else: - # off Windows a drive path is just an odd relative directory; - # chdir into tmp so it is created there - monkeypatch.chdir(tmp_path) - drive_path = "C:/drive/data" + drive_path = tmp_path.as_posix() # C:/Users/... r1 = await URLPipeline.from_url(f"file:{drive_path}|wrap:")._resolve_store_path() r2 = await URLPipeline.from_url(f"file:///{drive_path}|wrap:")._resolve_store_path() assert isinstance(r1.store, TracingStore) assert isinstance(r2.store, TracingStore) + @pytest.mark.skipif(sys.platform == "win32", reason="POSIX path semantics") + async def test_file_root_drive_letter_is_not_special_on_posix(self) -> None: + """Off Windows, `/C:/x` is an ordinary absolute path and `C:/x` is relative.""" + with pytest.raises(URLPipelineError, match="absolute path"): + await make_store_path(URLPipeline.from_url("file:C:/drive/data"), mode="r") + with pytest.raises(FileNotFoundError, match="/C:/drive/data"): + await make_store_path(URLPipeline.from_url("file:///C:/drive/data"), mode="r") + async def test_file_root_rejects_other_authority(self) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="unsupported authority"): From 893b9049b80dd67afaf2fc5f6569082228ddd8a0 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 22 Oct 2026 23:37:16 +0200 Subject: [PATCH 17/20] fix(url-pipeline): harden parsing and resolution; trim down-scope leftovers Security (round-2 review): - Redact query strings and URL userinfo in every error message and in the URLPipeline/PipelineSegment reprs; PipelineContext no longer reprs its storage_options. str() stays lossless. - zarr.url_adapters entry points with invalid or reserved names (file, memory, zarr, zarr2, zarr3) are ignored with a warning, and file:/memory: can no longer be adapter segments. - At most 32 segments, which bounds resolution recursion. - Single-letter root schemes are rejected (they read as drive paths on Windows). On Windows, file: roots are checked with '\' as a separator, so UNC paths are rejected, and ':' is allowed only in the drive. - Schemes are validated with fullmatch and before lowercasing, so a trailing newline or a non-ASCII look-alike cannot alias another scheme. - file:// authorities compare case-insensitively; adapter-not-found errors no longer list installed schemes. - Docs: every adapter sees all storage_options; memory: is process-wide. Simplification and leftovers: - One scheme check, one format-merge function (resolve_zarr_format), and inlined single-use helpers; parse_pipeline is private. - make_store keeps its URLPipeline branch but checks mode first only there; save/create functions keep their original defaulting order. - Drop PipelineSegment.__str__, tests of removed behavior and duplicate tests; fix the ABC docstring's TOML and the PipelineContext.mode text. - Cut the changelog and the user-guide sections to what they need. Co-Authored-By: Claude Opus 5.5 --- changes/4192.feature.md | 54 +- docs/user-guide/extending.md | 47 +- docs/user-guide/storage.md | 65 +- src/zarr/abc/url_pipeline.py | 65 +- src/zarr/api/asynchronous.py | 27 +- src/zarr/registry.py | 56 +- src/zarr/storage/_common.py | 43 +- src/zarr/storage/_url_pipeline.py | 605 ++++++------------ tests/test_url_pipeline/test_open_path.py | 122 +++- tests/test_url_pipeline/test_parser.py | 88 +-- .../test_url_pipeline/test_pipeline_object.py | 84 +-- tests/test_url_pipeline/test_resolver.py | 92 +-- tests/test_url_pipeline/test_spec_examples.py | 8 +- 13 files changed, 535 insertions(+), 821 deletions(-) diff --git a/changes/4192.feature.md b/changes/4192.feature.md index 5266b3d678..b722ca82c6 100644 --- a/changes/4192.feature.md +++ b/changes/4192.feature.md @@ -1,47 +1,9 @@ Added core support for URL pipelines (https://github.com/jbms/url-pipeline): -`|`-chained URLs that address zarr data through nested storage layers, e.g. -`s3://bucket/data.zip|zip:|zarr3:`. A parsed `zarr.storage.URLPipeline` (built with -`URLPipeline.from_url`) can be passed to any function that accepts a `StoreLike`, -including `zarr.open`, `zarr.open_group`, `zarr.open_array` and -`zarr.open_consolidated`. Adapters for the -sub-URLs after the first are provided by third-party packages (e.g. Icechunk) through -the `zarr.url_adapters` entry-point group and the single-method -`zarr.abc.url_pipeline.URLPipelineAdapter` interface, which receives a -`zarr.storage.PipelineSegment` and `zarr.storage.PipelineContext` and returns a -`zarr.storage.StorePath`; adapters are loaded lazily and individually, and -`zarr.registry.register_url_adapter` registers one without an entry point. A builtin -`zip:` adapter will follow separately. - -Behavior notes: - -- Plain strings are never interpreted as pipelines: `zarr.open("data|x.zarr")` still - addresses a local directory of that name, whatever adapters are installed. Only - `URLPipeline.from_url` reads a string as a pipeline. -- `URLPipeline.from_url` raises `TypeError` for anything but a `str`, and - `URLPipelineError` for a string it cannot parse: an empty sub-URL, a root without a - URL scheme (including a bare local or Windows drive path), an invalid adapter scheme, - a fragment, an ASCII control character, whitespace at either end of a sub-URL, a - format segment as the root, or a format-segment node path with `.` or `..` components. -- A trailing `zarr2:`/`zarr3:` segment selects the zarr format of the open or create - call, and its body is the path of the node to open; a trailing `zarr:` segment gives - the node path and leaves the format to be detected. A `zarr_format` argument that - conflicts with it raises `ValueError`, including the default of 3 in - `zarr.create_array`, `Group.from_store`, `Group.open` and `Array.open`: pass the - matching format (or `zarr_format=None` where it is accepted) to open a `zarr2:` - pipeline with these. -- Inside a pipeline, `memory:` and `file:` roots follow the URL pipeline spec. Every - `memory:` root addresses one in-memory store and its body is a path within it - (`memory:a` ≡ `memory:/a` ≡ `memory://a`). A `file:` root must carry an absolute path - with at most a `localhost` authority; drive letters are only special on Windows. - Neither accepts `.` or `..` path components. Percent-escapes are not decoded, matching - `LocalStore`. A pipeline's `memory:` root is a zarr `ManagedMemoryStore`, which does - not share data with the fsspec in-memory filesystem a plain `memory://name` string - resolves to. -- `register_url_adapter` raises `ValueError` for an invalid URL scheme and for the - schemes zarr-python handles itself (`file`, `memory`, `zarr`, `zarr2`, `zarr3`). -- Wrapper adapters can open a local *file* root only in mode `"r"` (via - `dataclasses.replace(context, mode="r").resolve_preceding()`); writing through a - container file is not yet supported. -- A `zarr.url_adapters` entry point whose scheme is already registered is ignored with a - `ZarrUserWarning`, as are duplicate entry-point names; an entry point that fails to - import raises `URLPipelineError` naming it, and stays discoverable for a retry. +`|`-chained URLs such as `s3://bucket/data.zip|zip:|zarr3:` that address zarr data +through nested storage layers. Parse one with `zarr.storage.URLPipeline.from_url` and +pass it to any function that accepts a `StoreLike`; plain strings are never read as +pipelines. A trailing `zarr:`, `zarr2:` or `zarr3:` segment gives the node path and +selects (or, for `zarr:`, detects) the zarr format. Packages such as Icechunk provide +adapters for other schemes by subclassing `zarr.abc.url_pipeline.URLPipelineAdapter` +and registering it under the `zarr.url_adapters` entry-point group, or with +`zarr.registry.register_url_adapter`. See the storage and extending user guides. diff --git a/docs/user-guide/extending.md b/docs/user-guide/extending.md index 5ecc90b2b8..b104199159 100644 --- a/docs/user-guide/extending.md +++ b/docs/user-guide/extending.md @@ -87,12 +87,12 @@ See [developing custom stores](storage.md#developing-custom-stores) for more inf ## Custom URL pipeline adapters -[URL pipelines](storage.md#user-guide-url-pipelines) are `|`-chained URLs such as -`s3://bucket/data.zip|zip:|zarr3:`. Each sub-URL after the first names an *adapter* -that reinterprets everything to its left. A package provides an adapter for a scheme by -subclassing [`zarr.abc.url_pipeline.URLPipelineAdapter`][] and implementing its single -classmethod, which resolves the adapter's segment into an open store and a residual path, -returned as a [`StorePath`][zarr.storage.StorePath]: +In a [URL pipeline](storage.md#user-guide-url-pipelines) such as +`s3://bucket/data.zip|zip:|zarr3:`, each sub-URL after the first names an *adapter* +that reinterprets everything to its left. A package provides an adapter by subclassing +[`zarr.abc.url_pipeline.URLPipelineAdapter`][] and implementing one classmethod, which +returns an open store and the path addressed inside it as a +[`StorePath`][zarr.storage.StorePath]: ```python test="true" session="url-adapter" from zarr.abc.url_pipeline import URLPipelineAdapter @@ -112,12 +112,9 @@ class MyAdapter(URLPipelineAdapter): return StorePath(store, path) ``` -To resolve the preceding pipeline with a different mode or options (for example -read-only, or without the adapter's own `storage_options` keys), resolve through a derived -context: `await dataclasses.replace(context, mode="r").resolve_preceding()`. - -Wrapper adapters compose: in `memory://x|mypackage.myscheme:a|mypackage.myscheme:b` the -inner adapter sees the outer one's residual path `a` and the pipeline resolves to `a/b`. +To resolve the preceding pipeline with another mode or options, use a derived context: +`await dataclasses.replace(context, mode="r").resolve_preceding()`. Wrapper adapters +compose: `memory://x|mypackage.myscheme:a|mypackage.myscheme:b` resolves to `a/b`. ```python test="true" session="url-adapter" import zarr @@ -131,23 +128,27 @@ group = zarr.open_group(URLPipeline.from_url(url), mode="r") assert group.path == "a/b" ``` -Register the class under the `zarr.url_adapters` entry-point group, using the URL -scheme as the entry-point name. Nonstandard schemes should be vendor-prefixed -(`vendor.scheme`), per the URL pipeline specification: +Register the class under the `zarr.url_adapters` entry-point group with the scheme as +the entry-point name, vendor-prefixed for nonstandard schemes. Adapters are imported +only when a pipeline naming their scheme is resolved. ```toml [project.entry-points."zarr.url_adapters"] "mypackage.myscheme" = "mypackage.zarr_adapter:MyAdapter" ``` -Adapters are loaded lazily, only when a pipeline naming their scheme is resolved. An -adapter may also be used as a *root* scheme (e.g. `mypackage.myscheme://org/repo`), in -which case `context.preceding` is empty; such a URL reaches the adapter only through -a `URLPipeline`, never as a plain string store specification. The returned store must honor -`context.read_only`, and `open_pipeline_segment` runs on zarr's internal event loop, so -it must not block or call zarr's synchronous API. See -[`zarr.abc.url_pipeline`][] and [`PipelineContext`][zarr.storage.PipelineContext] for the -full contract. +An adapter can also be a *root* scheme (`mypackage.myscheme://org/repo`), with an empty +`context.preceding`. Installing a root adapter for a scheme fsspec also handles takes +that scheme over at the root of every pipeline. The schemes zarr handles itself +(`file`, `memory`, `zarr`, `zarr2`, `zarr3`) cannot be registered, and entry points +with those names are ignored. + +Every adapter sees all of the caller's `storage_options`, including credentials meant +for the root: consume only keys namespaced for your adapter, never log the options, and +strip your keys before resolving the rest of the pipeline. The returned store must honor +`context.read_only`. `open_pipeline_segment` runs on zarr's event loop, so it must not +block or call zarr's synchronous API. See [`PipelineContext`][zarr.storage.PipelineContext] +for the full contract. ## Custom array buffers diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index ccadc80815..21ced15b41 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -110,54 +110,43 @@ print(group) ## URL Pipelines {#user-guide-url-pipelines} Zarr supports [URL pipelines](https://github.com/jbms/url-pipeline): `|`-chained URLs -that address zarr data through nested storage layers, read left to right. The first -sub-URL locates a resource with a conventional URL; each subsequent sub-URL names an -*adapter* that reinterprets everything to its left (e.g. -`s3://bucket/data.zip|zip:|zarr3:`). Adapters are provided by packages through the -`zarr.url_adapters` entry-point group — see -[`zarr.abc.url_pipeline`][zarr.abc.url_pipeline] for the adapter interface. A builtin -`zip:` adapter is under development and will expand this section. +read left to right. The first sub-URL locates a resource with a conventional URL; each +later sub-URL names an *adapter* that reinterprets everything to its left (e.g. +`s3://bucket/data.zip|zip:|zarr3:`). Adapters are provided by packages; see +[custom URL pipeline adapters](extending.md#custom-url-pipeline-adapters). -A parsed [`URLPipeline`][zarr.storage.URLPipeline] can be passed to any function that -accepts a `StoreLike`: +Parse a pipeline with [`URLPipeline.from_url`][zarr.storage.URLPipeline.from_url] and +pass it to any function that accepts a `StoreLike`. This is the only way a string is +read as a pipeline: a plain string store specification keeps its existing meaning, so a +local path containing `|` still names a directory. ```python exec="true" session="storage" source="above" result="ansi" from zarr.storage import URLPipeline pipeline = URLPipeline.from_url("memory://pipeline-demo|zarr2:") group = zarr.open_group(pipeline, mode="w") -print(group.metadata.zarr_format) print(zarr.open_group(pipeline, mode="r").metadata.zarr_format) ``` -Only `URLPipeline.from_url` reads a string as a pipeline. Plain string store -specifications keep their existing meaning, so a local path that happens to contain `|` -still names a directory, whatever adapters are installed. - -A trailing `zarr2:` or `zarr3:` segment is a *format segment*. It is not an adapter: -zarr-python reads it from the pipeline itself and uses it as the `zarr_format` of the -open or create call, and its body (`zarr3:path/to/node`) is the path of the node to -open. The node path must not contain `.` or `..` components. A trailing `zarr:` segment -also gives a node path, but leaves the format to be detected. A `zarr_format` argument that conflicts with the format segment raises -`ValueError`; functions whose `zarr_format` defaults to 3 (such as `zarr.create_array` -and `Group.open`) need an explicit `zarr_format=None` (or the matching format) to defer -to the pipeline. - -`storage_options` apply to the *root* sub-URL (e.g. fsspec options for `s3://...`); -adapters may consume adapter-specific, namespaced keys. No percent-escape is decoded -anywhere in a pipeline. Adapters cannot be registered for the schemes zarr-python -handles itself: `file`, `memory`, `zarr`, `zarr2` and `zarr3`. - -Inside a pipeline, every `memory:` root addresses the same managed in-memory store, and -its body is a path within that store: `memory:a/b`, `memory:/a/b` and `memory://a/b` are -equivalent, as are `memory:`, `memory:/` and `memory://`. Data lives only while some -store object referring to it is alive. When fsspec is installed, a plain `memory://name` -string store specification is still routed to fsspec's in-memory filesystem, which is a -different store. A `file:` root must carry an absolute path, with at most a `localhost` -authority; percent-escapes are not decoded, matching the [local store](#local-store). -Drive letters are only special on Windows: elsewhere `file:/C:/data` is the absolute -path `/C:/data`. Neither root accepts `.` or `..` path components. The root always -carries a scheme: a local path is spelled as a `file:` URL, never as a bare path. +A trailing `zarr2:` or `zarr3:` *format segment* selects the zarr format of the call, +and its body (`zarr3:path/to/node`) is the path of the node; `zarr:` gives a node path +and leaves the format to be detected. A conflicting `zarr_format` argument raises +`ValueError`. That includes the default of 3 of `zarr.create_array`, `Group.open`, +`Group.from_store` and `Array.open`: pass the matching format, or use +`zarr.open_group` / `zarr.open_array`, which follow the pipeline. + +The root sub-URL must carry a scheme, so a local path is spelled as an absolute `file:` +URL (`file:/data/x.zarr`, with at most a `localhost` authority). Every `memory:` root +addresses one managed in-memory store, and its body is a path within it (`memory:a` ≡ +`memory:/a` ≡ `memory://a`); data lives only while some store object refers to it. +Neither root accepts `.` or `..` path components, and drive letters are only special on +Windows. No percent-escape is decoded anywhere in a pipeline. Other roots, such as +`s3://...`, are opened exactly as the same plain string would be. + +`storage_options` configure the root sub-URL, and every adapter in the pipeline can +see them. When a pipeline may come from an untrusted source, keep credentials out of +`storage_options` unless you trust every installed adapter, and don't accept `memory:` +roots in a process that serves several users: the in-memory store is process-wide. ## Explicit Store Creation diff --git a/src/zarr/abc/url_pipeline.py b/src/zarr/abc/url_pipeline.py index 8280e1cca6..d00db407e8 100644 --- a/src/zarr/abc/url_pipeline.py +++ b/src/zarr/abc/url_pipeline.py @@ -1,20 +1,7 @@ """ -Abstract base class for URL pipeline adapters. - -A URL pipeline is a `|`-separated chain of sub-URLs, read outer-to-inner, -as specified by https://github.com/jbms/url-pipeline. The first sub-URL (the -*root*) locates a resource using a conventional URL, and each subsequent -sub-URL names an *adapter* that reinterprets everything to its left: - - s3://bucket/data.zip|zip:path/inside|zarr3: - -Third-party packages provide adapters by subclassing -[`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] and -registering the class under the `zarr.url_adapters` entry-point group, -using the URL scheme as the entry-point name. The values an adapter receives -and returns, [`PipelineSegment`][zarr.storage.PipelineSegment], -[`PipelineContext`][zarr.storage.PipelineContext] and -[`StorePath`][zarr.storage.StorePath], live in `zarr.storage`. +Abstract base class for URL pipeline adapters +(https://github.com/jbms/url-pipeline). See +[`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter]. """ from __future__ import annotations @@ -32,22 +19,21 @@ class URLPipelineAdapter(ABC): """ Handler for one URL pipeline scheme. - Subclasses implement a single classmethod, - [`open_pipeline_segment`][zarr.abc.url_pipeline.URLPipelineAdapter.open_pipeline_segment], - and are registered under the `zarr.url_adapters` entry-point group with + In a pipeline such as `s3://bucket/data.zip|zip:inner|zarr3:`, each sub-URL + after the first names an adapter that reinterprets everything to its left. + Subclasses implement + [`open_pipeline_segment`][zarr.abc.url_pipeline.URLPipelineAdapter.open_pipeline_segment] + and are registered under the `zarr.url_adapters` entry-point group, with the URL scheme as the entry-point name: [project.entry-points."zarr.url_adapters"] - mypackage.myscheme = "mypackage.zarr_adapter:MyAdapter" - - Nonstandard schemes should be vendor-prefixed (`vendor.scheme`) per the - URL pipeline specification. + "mypackage.myscheme" = "mypackage.zarr_adapter:MyAdapter" - An adapter is used in two positions: - - - as an *adapter segment*: `s3://bucket/repo|icechunk://tag.v1` — the - context carries the preceding sub-URLs; - - as a *root scheme*: `gh://org/repo` — `context.preceding` is empty. + Nonstandard schemes should be vendor-prefixed (`vendor.scheme`), per the + URL pipeline specification. An adapter is used either as an *adapter + segment* (`s3://bucket/repo|icechunk://tag.v1`), where the context carries + the preceding sub-URLs, or as a *root scheme* (`gh://org/repo`), where + `context.preceding` is empty. """ @classmethod @@ -56,21 +42,14 @@ async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext ) -> StorePath: """ - Resolve `segment` (in the context of the pipeline to its left) - into a store and a residual path within that store. - - The returned store must already be open and must honor - `context.read_only` (the resolver additionally enforces it by - downgrading — or rejecting — a writable store when the caller - required read-only). The path of the returned - [`StorePath`][zarr.storage.StorePath] is the location the pipeline - addresses inside the store; it is empty for the store root. - - This coroutine runs on zarr's internal I/O event loop. It must not - block (do I/O through async APIs or a thread executor) and must not - call zarr's synchronous API (`zarr.open`, `Group.open`, or anything - else that uses `zarr.core.sync.sync`) — doing so raises - `SyncError`. To open the preceding pipeline, use + Resolve `segment`, in the context of the pipeline to its left, into an + open store and the path the pipeline addresses inside it (empty for + the store root). + + The store must honor `context.read_only`; the resolver also enforces + it. This coroutine runs on zarr's I/O event loop, so it must not + block, and must not call zarr's synchronous API (which raises + `SyncError`). Open the preceding pipeline with [`resolve_preceding`][zarr.storage.PipelineContext.resolve_preceding]. """ ... diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index a4c254c1be..588ce5850a 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -528,14 +528,14 @@ async def save_array( **kwargs Passed through to [`create`][zarr.api.asynchronous.create], e.g., compressor. """ + zarr_format = resolve_zarr_format(store, zarr_format) + if zarr_format is None: + zarr_format = _default_zarr_format() if not isinstance(arr, NDArrayLike): raise TypeError("arr argument must be numpy or other NDArrayLike array") mode = kwargs.pop("mode", "a") - zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - if zarr_format is None: - zarr_format = _default_zarr_format() if np.isscalar(arr): arr = np.array(arr) shape = arr.shape @@ -584,6 +584,10 @@ async def save_group( NumPy arrays with data to save. """ + zarr_format = resolve_zarr_format(store, zarr_format) + if zarr_format is None: + zarr_format = _default_zarr_format() + for arg in args: if not isinstance(arg, NDArrayLike): raise TypeError( @@ -601,10 +605,7 @@ async def save_group( for arr in (*args, *kwargs.values()): get_data_type_from_native_dtype(arr.dtype) - zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, path=path, mode="w", storage_options=storage_options) - if zarr_format is None: - zarr_format = _default_zarr_format() # The group replaces whatever is stored under the path, now that the arguments are known # to be valid. await AsyncGroup.from_store(store_path, zarr_format=zarr_format, overwrite=True) @@ -799,13 +800,14 @@ async def create_group( The new group. """ - mode: Literal["a"] = "a" - zarr_format = resolve_zarr_format(store, zarr_format) - store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) if zarr_format is None: zarr_format = _default_zarr_format() + mode: Literal["a"] = "a" + + store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) + return await AsyncGroup.from_store( store=store_path, zarr_format=zarr_format, @@ -1102,6 +1104,10 @@ async def create( z : array The array. """ + zarr_format = resolve_zarr_format(store, zarr_format) + if zarr_format is None: + zarr_format = _default_zarr_format() + _warn_unimplemented_kwargs( { "synchronizer": synchronizer, @@ -1120,10 +1126,7 @@ async def create( if mode is None: mode = "a" overwrite = overwrite or _infer_overwrite(mode) - zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - if zarr_format is None: - zarr_format = _default_zarr_format() config_parsed = parse_array_config(config) diff --git a/src/zarr/registry.py b/src/zarr/registry.py index a568f97347..c1d94e9095 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re import threading import warnings from collections import defaultdict @@ -307,7 +308,16 @@ def _collect_entrypoints() -> list[Registry[Any]]: entry_points.select(group="zarr", name="chunk_key_encoding") ) - _url_adapter_registry.lazy_load_list.extend(entry_points.select(group="zarr.url_adapters")) + for entry_point in entry_points.select(group="zarr.url_adapters"): + if _is_adapter_scheme(entry_point.name): + _url_adapter_registry.lazy_load_list.append(entry_point) + else: + warnings.warn( + f"ignoring the 'zarr.url_adapters' entry point {entry_point.name!r} " + f"({entry_point.value!r}): it is not a URL scheme an adapter can handle", + category=ZarrUserWarning, + stacklevel=2, + ) _pipeline_registry.lazy_load_list.extend(entry_points.select(group="zarr.codec_pipeline")) _pipeline_registry.lazy_load_list.extend( @@ -511,10 +521,23 @@ def get_chunk_key_encoding_class(key: str) -> type[ChunkKeyEncoding]: return _chunk_key_encoding_registry[key] +# A URL scheme (RFC 3986, ASCII only), plus "." for vendor-prefixed schemes +# such as "earthmover.myscheme". Use with fullmatch. +_URL_SCHEME_RE = re.compile(r"[a-zA-Z][a-zA-Z0-9+.\-]*") + +# Schemes zarr-python handles itself: the native `file:` and `memory:` roots +# and the `zarr:`/`zarr2:`/`zarr3:` format segments. +_RESERVED_URL_SCHEMES = frozenset({"file", "memory", "zarr", "zarr2", "zarr3"}) + + +def _is_adapter_scheme(scheme: str) -> bool: + return bool(_URL_SCHEME_RE.fullmatch(scheme)) and scheme.lower() not in _RESERVED_URL_SCHEMES + + def register_url_adapter(scheme: str, cls: type[URLPipelineAdapter]) -> None: """ Register a [`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] - class for a URL scheme. + class for a URL scheme (case-insensitive). Registering a scheme that already has an adapter replaces it and emits a [`ZarrUserWarning`][zarr.errors.ZarrUserWarning]. @@ -523,19 +546,16 @@ class for a URL scheme. ------ ValueError If `scheme` is not a valid URL scheme, or is one zarr-python handles - itself (`file`, `memory`, `zarr`, `zarr2`, `zarr3`), so an adapter - registered for it could never be used. + itself (`file`, `memory`, `zarr`, `zarr2`, `zarr3`). """ - from zarr.storage._url_pipeline import _RESERVED_SCHEMES, _SCHEME_RE # circular import - - if not _SCHEME_RE.match(scheme): + if not _URL_SCHEME_RE.fullmatch(scheme): raise ValueError(f"{scheme!r} is not a valid URL scheme") - key = scheme.lower() - if key in _RESERVED_SCHEMES: + if not _is_adapter_scheme(scheme): raise ValueError( f"the URL scheme {scheme!r} is handled by zarr-python itself and cannot be " "given to a URL pipeline adapter" ) + key = scheme.lower() previous = _url_adapter_registry.get(key) if previous is not None and previous is not cls: warnings.warn( @@ -555,9 +575,10 @@ def _list_url_adapter_schemes() -> set[str]: entry points; consulting this does not import any adapter code. Schemes are case-insensitive and reported lowercased. """ - return set(_url_adapter_registry) | { + schemes = set(_url_adapter_registry) | { e.name.lower() for e in _url_adapter_registry.lazy_load_list } + return {scheme for scheme in schemes if _is_adapter_scheme(scheme)} _url_adapter_lock = threading.Lock() @@ -569,7 +590,7 @@ def _get_url_adapter(scheme: str) -> type[URLPipelineAdapter]: Loads pending `zarr.url_adapters` entry points for this scheme only, so resolving one scheme never imports other providers' packages. When the - scheme is already registered (e.g. a builtin adapter), a same-named entry + scheme is already registered (e.g. with `register_url_adapter`), a same-named entry point is discarded with a [`ZarrUserWarning`][zarr.errors.ZarrUserWarning]; when several entry points share the name, the first one wins and a warning names the ones ignored. @@ -581,6 +602,10 @@ def _get_url_adapter(scheme: str) -> type[URLPipelineAdapter]: providing it fails to import. """ key = scheme.lower() + if not _is_adapter_scheme(key): + raise URLPipelineError( + f"'{key}:' is handled by zarr-python itself and cannot be an adapter segment" + ) # Take the matching entry points out of the pending list under the lock, # so concurrent first-time resolutions of different schemes never clobber # each other's view of the list. The import itself runs *outside* the lock: @@ -617,8 +642,7 @@ def _get_url_adapter(scheme: str) -> type[URLPipelineAdapter]: with _url_adapter_lock: _url_adapter_registry.lazy_load_list.extend(pending) raise URLPipelineError( - f"the 'zarr.url_adapters' entry point {entry_point.name!r} " - f"({entry_point.value!r}) could not be loaded: {exc!r}" + f"the 'zarr.url_adapters' entry point {entry_point.name!r} could not be loaded" ) from exc with _url_adapter_lock: if key not in _url_adapter_registry: @@ -626,11 +650,9 @@ def _get_url_adapter(scheme: str) -> type[URLPipelineAdapter]: try: return _url_adapter_registry[key] except KeyError: - registered_schemes = sorted(_list_url_adapter_schemes()) raise URLPipelineError( - f"no URL pipeline adapter is registered for scheme {scheme!r}. " - f"Registered schemes: {registered_schemes}. Adapters are provided by " - "packages via the 'zarr.url_adapters' entry-point group." + f"no URL pipeline adapter is registered for scheme {scheme!r}. Adapters are " + "provided by packages via the 'zarr.url_adapters' entry-point group." ) from None diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index b869bb1233..88cbef1a8f 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -29,7 +29,7 @@ ) from zarr.storage._local import LocalStore from zarr.storage._memory import ManagedMemoryStore, MemoryStore -from zarr.storage._url_pipeline import URLPipeline +from zarr.storage._url_pipeline import _FORMAT_SCHEMES, URLPipeline from zarr.storage._utils import UPath, _join_paths, normalize_path, parse_store_url _has_fsspec = importlib.util.find_spec("fsspec") @@ -326,9 +326,8 @@ async def make_store( - `None` = `MemoryStore` object. - `FSMap` = `FsspecStore` object. - `UPath` = `FsspecStore` object, or `LocalStore` for a local `UPath`. - - `URLPipeline` = the store resolved by the [URL pipeline][user-guide-url-pipelines]; - the pipeline must address the store root (use `make_store_path` for pipelines that - address a path within a store). Plain strings are never interpreted as pipelines. + - `URLPipeline` = the store the [URL pipeline][user-guide-url-pipelines] resolves to, + which must be the store root (`make_store_path` accepts any pipeline). Parameters ---------- @@ -355,16 +354,14 @@ async def make_store( """ from zarr.storage._fsspec import FsspecStore # circular import - if mode is not None and mode not in ANY_ACCESS_MODE: - raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") - _read_only = mode == "r" - if isinstance(store_like, URLPipeline): + if mode is not None and mode not in ANY_ACCESS_MODE: + raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") result = await store_like._resolve_store_path(mode=mode, storage_options=storage_options) if result.path: result.store.close() raise URLPipelineError( - f"the URL pipeline {str(store_like)!r} resolves to a path inside a store; " + f"{store_like!r} resolves to a path inside a store; " "use zarr.open() or make_store_path() instead of make_store()" ) return result.store @@ -381,6 +378,10 @@ async def make_store( "'storage_options' is only used when the store is passed as an FSSpec URI string.", ) + if mode is not None and mode not in ANY_ACCESS_MODE: + raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") + _read_only = mode == "r" + if isinstance(store_like, StorePath): # Get underlying store return store_like.store @@ -531,21 +532,27 @@ def resolve_zarr_format( store_like: StoreLike | None, zarr_format: ZarrFormat | None ) -> ZarrFormat | None: """ - Merge the zarr format selected by a `URLPipeline` store specification with - the format a caller asked for. - - For any other `StoreLike`, `zarr_format` is returned unchanged. A pipeline - with a trailing `zarr2:`/`zarr3:` segment selects that format when the - caller passes None. + Merge the format selected by a `URLPipeline`'s trailing `zarr2:`/`zarr3:` + segment with the caller's `zarr_format`. Other `StoreLike`s, and a trailing + `zarr:` segment, leave `zarr_format` unchanged. Raises ------ ValueError If `zarr_format` is not None and differs from the pipeline's format. """ - if isinstance(store_like, URLPipeline): - return store_like._resolve_zarr_format(zarr_format) - return zarr_format + if not isinstance(store_like, URLPipeline) or store_like._format_segment is None: + return zarr_format + pipeline_format = _FORMAT_SCHEMES[store_like._format_segment.scheme] + if pipeline_format is None: + return zarr_format + if zarr_format is not None and zarr_format != pipeline_format: + raise ValueError( + f"zarr_format={zarr_format} conflicts with the 'zarr{pipeline_format}:' segment " + f"of {store_like!r}; pass zarr_format={pipeline_format} (or None where accepted), " + "or open it with zarr.open_group / zarr.open_array, which follow the pipeline" + ) + return pipeline_format async def ensure_no_existing_node( diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py index 1fc8c5438c..63cb75fa3f 100644 --- a/src/zarr/storage/_url_pipeline.py +++ b/src/zarr/storage/_url_pipeline.py @@ -1,17 +1,10 @@ """ Parsing and resolution of URL pipelines (https://github.com/jbms/url-pipeline). -Importing this module is cheap and has no side effects: third-party adapters -(registered through the `zarr.url_adapters` entry-point group) are loaded -only when a pipeline URL naming their scheme is actually resolved. - -A string is never interpreted as a pipeline on its own: the user wraps it -in a [`URLPipeline`][zarr.storage.URLPipeline], and that object is what the `StoreLike` machinery resolves. Plain string -store specifications, including local paths that happen to contain `|`, -keep their pre-existing meaning. No percent-escape is decoded. - -As the specification requires, the root sub-URL carries a URL scheme: a -local path is spelled as an absolute `file:` URL. +Only [`URLPipeline.from_url`][zarr.storage.URLPipeline.from_url] reads a string +as a pipeline; plain string store specifications keep their meaning. Adapters +registered through the `zarr.url_adapters` entry-point group are imported only +when a pipeline naming their scheme is resolved. No percent-escape is decoded. """ from __future__ import annotations @@ -19,11 +12,11 @@ import dataclasses import re import sys -from pathlib import Path +from pathlib import Path, PureWindowsPath from typing import TYPE_CHECKING, Any from zarr.errors import URLPipelineError -from zarr.registry import _get_url_adapter, _list_url_adapter_schemes +from zarr.registry import _URL_SCHEME_RE, _get_url_adapter, _list_url_adapter_schemes from zarr.storage._memory import ManagedMemoryStore from zarr.storage._utils import _join_paths, normalize_path @@ -34,6 +27,30 @@ __all__ = ["PipelineContext", "PipelineSegment", "URLPipeline"] +# Format segments are read by zarr-python itself, not by an adapter: they +# select the zarr format (`zarr:` detects it), and their body is a node path. +_FORMAT_SCHEMES: dict[str, ZarrFormat | None] = {"zarr": None, "zarr2": 2, "zarr3": 3} + +# Roots zarr resolves itself; an adapter can never claim them. +_NATIVE_ROOT_SCHEMES = frozenset({"file", "memory"}) + +# More segments than any real pipeline needs; bounds resolution recursion. +_MAX_SEGMENTS = 32 + +_CONTROL_CHAR_RE = re.compile(r"[\x00-\x1f\x7f]") +_WINDOWS_DRIVE_RE = re.compile(r"[A-Za-z]:/") + + +def _redact(text: str) -> str: + """Hide query strings and URL userinfo, which may carry credentials.""" + text = re.sub(r"//[^/@|?]*@", "//…@", text) + return re.sub(r"\?[^|]*", "?…", text) + + +def _check_scheme(scheme: str, raw: str) -> None: + if not _URL_SCHEME_RE.fullmatch(scheme): + raise URLPipelineError(f"invalid URL scheme {scheme!r} in the sub-URL {_redact(raw)!r}") + @dataclasses.dataclass(frozen=True) class PipelineSegment: @@ -45,15 +62,12 @@ class PipelineSegment: scheme : str The lowercased URL scheme. body : str - The text after `scheme:` and before any `?`. Interpretation is - scheme-defined; it is **not** URL-normalized, so case-significant - content (e.g. icechunk snapshot IDs) is preserved. + The text after `scheme:` and before any `?`, verbatim (case-significant + content such as Icechunk snapshot IDs is preserved). query : str | None - The raw query string after `?`, or None. Interpretation is - scheme-defined. + The raw query string after `?`, or None. raw : str - The exact original sub-URL text, preserved for lossless - reconstruction of the pipeline. + The original sub-URL text. """ scheme: str @@ -72,13 +86,12 @@ def __post_init__(self) -> None: raise TypeError( f"PipelineSegment.query must be a str or None, not {type(self.query).__name__}" ) - if not _SCHEME_RE.match(self.scheme) or self.scheme != self.scheme.lower(): - raise URLPipelineError( - f"PipelineSegment.scheme must be a lowercase URL scheme, not {self.scheme!r}" - ) + _check_scheme(self.scheme, self.raw) + if self.scheme != self.scheme.lower(): + raise URLPipelineError(f"PipelineSegment.scheme must be lowercase: {self.scheme!r}") - def __str__(self) -> str: - return self.raw + def __repr__(self) -> str: + return f"PipelineSegment({_redact(self.raw)!r})" @dataclasses.dataclass(frozen=True, eq=False) @@ -87,43 +100,34 @@ class PipelineContext: Context handed to a [`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] describing the pipeline to the left of its segment. - Instances are immutable but compare and hash by identity. An adapter that - needs to resolve the preceding pipeline with a different mode or options - derives a new context with `dataclasses.replace` (see - [`resolve_preceding`][zarr.storage.PipelineContext.resolve_preceding]). + Instances are immutable and compare by identity. Derive a context with + other values with `dataclasses.replace`. Attributes ---------- preceding : tuple[PipelineSegment, ...] - The parsed sub-URLs to the left of the adapter's segment, outer to - inner. Empty when the adapter's segment is the pipeline root. + The sub-URLs to the left of the adapter's segment, outer to inner. + Empty when the adapter's segment is the pipeline root. mode : AccessModeLiteral | None - The access mode requested by the caller (e.g. - `zarr.open(mode=...)`), or None when unspecified. Adapters for - read-only resources should raise for every mode that may write, and - open read-only for `"r"` and None. + The caller's access mode, or None. Adapters for read-only resources + should raise for every mode except `"r"`. storage_options : dict[str, Any] | None - Options passed by the caller. By convention these configure the - *root* sub-URL (e.g. fsspec options); adapters may consume - adapter-specific keys, and should namespace them (e.g. - `myscheme_credentials`) to avoid collisions with other segments' - backends. An adapter that consumes keys should strip them before - resolving the rest of the pipeline. + The caller's options. They configure the root sub-URL (e.g. fsspec + options), and every adapter in the pipeline sees all of them: an + adapter should consume only keys namespaced for it (e.g. + `myscheme_token`), and strip them before resolving the rest of the + pipeline. """ preceding: tuple[PipelineSegment, ...] mode: AccessModeLiteral | None - storage_options: dict[str, Any] | None + storage_options: dict[str, Any] | None = dataclasses.field(repr=False) @property def read_only(self) -> bool: """ True when the caller requires a read-only store (`mode == "r"`). - - Adapters must construct their store read-only when this is set - (the resolver enforces it afterwards); when it is False, they may - construct a writable store if the underlying resource supports - writing. + The resolver enforces this on the store the adapter returns. """ return self.mode == "r" @@ -133,41 +137,20 @@ def preceding_url(self) -> str: The pipeline to the left of this segment, reconstructed exactly. An adapter that consumes this string instead of calling - [`resolve_preceding`][zarr.storage.PipelineContext.resolve_preceding] - takes ownership of the *entire* preceding pipeline: it must - validate every preceding segment itself and raise - [`URLPipelineError`][zarr.errors.URLPipelineError] for segments it - does not understand, so that no segment is ever silently ignored. + `resolve_preceding` must validate every preceding segment itself, + and raise `URLPipelineError` for any it does not understand. """ return "|".join(segment.raw for segment in self.preceding) async def resolve_preceding(self) -> StorePath: """ - Resolve the preceding pipeline into a store and a residual path. - - This is the entry point for *wrapper* adapters (e.g. `zip:`) that - operate on the resource produced by the segments to their left. It - composes with any preceding adapters, because each segment is - resolved by its own adapter. Adapters backed by their own I/O - machinery (e.g. `icechunk:`) may instead consume - [`preceding_url`][zarr.storage.PipelineContext.preceding_url] - and never materialize the intermediate store — subject to the - ownership contract documented there. - - The preceding pipeline is resolved with this context's `mode` and - `storage_options`. To use others, resolve through a derived - context: - - preceding = await dataclasses.replace( - context, mode="r", storage_options=remaining_options - ).resolve_preceding() - - Wrapper adapters that only read the preceding resource should pass - `mode="r"`, so the root is opened read-only and without create-on-open - side effects. Note that a local *file* root (e.g. `file:/data.zip|zip:`) - can currently only be resolved in mode `"r"`: every other mode creates - a directory at the root and fails with `FileExistsError`, because - there is no file-resource primitive yet. + Resolve the preceding pipeline, with this context's `mode` and + `storage_options`, into a store and a residual path. + + Wrapper adapters (e.g. `zip:`) that only read the preceding resource + should resolve it read-only: + `await dataclasses.replace(context, mode="r").resolve_preceding()`. + A local *file* root can currently only be resolved in mode `"r"`. """ if not self.preceding: raise URLPipelineError( @@ -177,172 +160,80 @@ async def resolve_preceding(self) -> StorePath: return await _resolve(self.preceding, mode=self.mode, storage_options=self.storage_options) -# Format segments are consumed by zarr-python itself rather than by a -# registered adapter: they select the zarr format of the node the pipeline -# addresses (`zarr:` detects it), and their body is a path to that node. -_FORMAT_SCHEMES: dict[str, ZarrFormat | None] = {"zarr": None, "zarr2": 2, "zarr3": 3} - -# Adapter scheme per RFC 3986 plus "." to permit vendor-prefixed -# nonstandard schemes (e.g. "earthmover.myscheme"). -_SCHEME_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.\-]*$") - -# Root scheme at the very start of the sub-URL. -_ROOT_SCHEME_RE = re.compile(r"([a-zA-Z][a-zA-Z0-9+.\-]*):") - -# Something shaped like a scheme (text up to the first ':' with no '/', '?' -# or '#' before it) that is not a valid one, e.g. `a_b://x`. -_SCHEME_LIKE_RE = re.compile(r"[^:/?#]+:") - -# Schemes zarr resolves natively at the pipeline root. These are never -# dispatched to a registered root adapter, so an installed package cannot -# intercept zarr's own local-path and in-memory routing. -_NATIVE_ROOT_SCHEMES = frozenset({"file", "memory"}) - -# Schemes zarr-python handles itself, which therefore cannot be given to a -# registered adapter. -_RESERVED_SCHEMES = _NATIVE_ROOT_SCHEMES | frozenset(_FORMAT_SCHEMES) - -# An absolute Windows drive path (C:\... or C:/...), which counts as an -# absolute path in a `file:` pipeline root. -_WINDOWS_DRIVE_RE = re.compile(r"[A-Za-z]:[/\\]") - -# A bare Windows drive path as a pipeline root (C:\data, C:/data), as -# opposed to a single-letter scheme with an authority (x://host/path). -_BARE_DRIVE_ROOT_RE = re.compile(r"[A-Za-z]:(\\|/(?!/))") - -# ASCII control characters (C0 and DEL). No sub-URL may contain them. -_CONTROL_CHAR_RE = re.compile(r"[\x00-\x1f\x7f]") - - -def _root_scheme(root_sub_url: str) -> str: - """ - Detect the scheme of the root sub-URL, or `""` when it has none. - - Scheme extraction happens on the raw string, so nothing `urlparse` - would strip or reject can desync the detected scheme from the body. A - single letter followed by `:` is a scheme on every platform; a bare - Windows drive path (`C:\\...`, `C:/...`) is rejected before this is - called, since a pipeline spells it `file:/C:/...`. - """ - match = _ROOT_SCHEME_RE.match(root_sub_url) - if match is None: - return "" - return match.group(1).lower() - - def _split_query(sub_url: str) -> tuple[str, str | None]: - """Split a sub-URL on the first `?`. Fragments are not supported.""" if "#" in sub_url: raise URLPipelineError( - f"URL pipeline sub-URLs do not support fragments ('#'): {sub_url!r}. " - "Percent-escapes are not decoded, so a '#' in a path cannot be addressed." + f"URL pipeline sub-URLs do not support fragments ('#'): {_redact(sub_url)!r}" ) body, sep, query = sub_url.partition("?") return body, query if sep else None -def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]: +def _parse_pipeline(url: str) -> tuple[PipelineSegment, ...]: """ - Parse a URL pipeline into its `|`-delimited segments. - - The first segment is the *root* sub-URL. Subsequent segments are *adapter* sub-URLs of the form - `scheme:body` where the trailing colon is optional when the body is - empty (`zip` is equivalent to `zip:`). - - The root must carry a scheme; a bare local path is rejected, since the - specification has no schemeless sub-URLs and `|` could not be escaped - in one. - - Segment text is preserved verbatim (no case or percent-encoding - normalization) except that schemes are lowercased. ASCII control - characters, and whitespace at either end of a sub-URL, are rejected - rather than stripped. + Split a URL pipeline into segments. Adapter segments are `scheme:body`, + where the colon is optional when the body is empty (`zip` ≡ `zip:`). """ if not isinstance(url, str): raise TypeError(f"a URL pipeline must be a str, not {type(url).__name__}") if (match := _CONTROL_CHAR_RE.search(url)) is not None: - raise URLPipelineError( - f"URL pipeline contains the control character {match.group()!r}: {url!r}" - ) + raise URLPipelineError(f"URL pipeline contains the control character {match.group()!r}") parts = url.split("|") + if len(parts) > _MAX_SEGMENTS: + raise URLPipelineError(f"URL pipeline has more than {_MAX_SEGMENTS} segments") if any(not part for part in parts): - raise URLPipelineError(f"URL pipeline contains an empty sub-URL: {url!r}") - for part in parts: - if part != part.strip(): - raise URLPipelineError( - f"sub-URL {part!r} of the URL pipeline {url!r} has leading or trailing " - "whitespace; remove it" - ) + raise URLPipelineError(f"URL pipeline contains an empty sub-URL: {_redact(url)!r}") - segments: list[PipelineSegment] = [] + segments = [] for index, part in enumerate(parts): + if part != part.strip(): + raise URLPipelineError(f"sub-URL {_redact(part)!r} has leading or trailing whitespace") + body_and_scheme, query = _split_query(part) + scheme, sep, body = body_and_scheme.partition(":") if index == 0: - if _BARE_DRIVE_ROOT_RE.match(part): - raise URLPipelineError( - f"the root sub-URL {part!r} of the URL pipeline {url!r} is a Windows drive " - "path; spell it as a 'file:' URL (e.g. 'file:/C:/data')." - ) - scheme = _root_scheme(part) - if not scheme and _SCHEME_LIKE_RE.match(part): - invalid = part.partition(":")[0] + if not sep or not scheme or "/" in scheme or "\\" in scheme: raise URLPipelineError( - f"invalid URL scheme {invalid!r} in the root sub-URL {part!r} " - f"of the URL pipeline {url!r}" + f"the root sub-URL {_redact(part)!r} has no URL scheme. Spell a local path " + "as an absolute 'file:' URL; a local path containing '|' cannot be " + "addressed by a URL pipeline, so open it with a plain path instead." ) - if not scheme: + _check_scheme(scheme, part) + if len(scheme) == 1: raise URLPipelineError( - f"the root sub-URL {part!r} of the URL pipeline {url!r} has no URL scheme. " - "Spell a local path as an absolute 'file:' URL. Percent-escapes are not " - "decoded, so a local path containing '|' cannot be addressed by a URL " - "pipeline; open it with a plain path or a store object instead." + f"the root sub-URL {_redact(part)!r} looks like a Windows drive path; " + "spell it as a 'file:' URL (e.g. 'file:/C:/data')" ) - body_and_scheme, query = _split_query(part) - body = body_and_scheme[len(scheme) + 1 :] - segments.append(PipelineSegment(scheme=scheme, body=body, query=query, raw=part)) else: - body_and_scheme, query = _split_query(part) - scheme, _, body = body_and_scheme.partition(":") - scheme = scheme.lower() - if not _SCHEME_RE.match(scheme): - raise URLPipelineError( - f"invalid adapter scheme {scheme!r} in pipeline segment {part!r}" - ) - segments.append(PipelineSegment(scheme=scheme, body=body, query=query, raw=part)) + _check_scheme(scheme, part) + segments.append(PipelineSegment(scheme=scheme.lower(), body=body, query=query, raw=part)) return tuple(segments) @dataclasses.dataclass(frozen=True) class URLPipeline: """ - A parsed URL pipeline, ready to be passed wherever a `StoreLike` is accepted. + A parsed URL pipeline, accepted wherever a `StoreLike` is. - Construct one with [`from_url`][zarr.storage.URLPipeline.from_url] and open it - through any `StoreLike`-taking function. Because a - `URLPipeline` is an explicit object, no plain string store specification is - ever interpreted as a pipeline: `zarr.open("data|x.zarr")` still addresses a - local directory of that name, while - `zarr.open(URLPipeline.from_url("s3://bucket/data.zip|zip:|zarr3:"))` resolves - the pipeline through registered adapters. - - A pipeline is a value: instances are immutable, hashable, and compare equal - when they hold the same segments. Nothing is resolved until the pipeline is - opened; resolution needs the access mode and storage options of the open - call. + Build one with [`from_url`][zarr.storage.URLPipeline.from_url] and pass it to + any `StoreLike`-taking function, e.g. + `zarr.open_group(URLPipeline.from_url("s3://bucket/data.zip|zip:|zarr3:"))`. + Instances are immutable and hashable values; nothing is resolved until the + pipeline is opened. Attributes ---------- segments : tuple[PipelineSegment, ...] - Every `|`-delimited sub-URL of the pipeline, in order: the root, any - adapter segments, and an optional trailing format segment. + The sub-URLs in order: the root, any adapter segments, and an optional + trailing format segment (`zarr:`, `zarr2:` or `zarr3:`). Raises ------ TypeError If `segments` is not a tuple of `PipelineSegment`. URLPipelineError - If there are no segments, if a format segment (`zarr:`/`zarr2:`/`zarr3:`) - appears anywhere but last (including as the root), or if a format segment - carries a query or a node path with `.` or `..` components. + If there are no segments or more than 32, if a format segment is + anywhere but last (including the root), or if a format segment carries + a query or a node path with `.` or `..` components. """ segments: tuple[PipelineSegment, ...] @@ -356,57 +247,47 @@ def __post_init__(self) -> None: "URLPipeline.segments must be a tuple of PipelineSegment; " "use URLPipeline.from_url to parse a string" ) - if not self.segments: - raise URLPipelineError("a URL pipeline needs at least one segment") + if not 1 <= len(self.segments) <= _MAX_SEGMENTS: + raise URLPipelineError(f"a URL pipeline needs 1 to {_MAX_SEGMENTS} segments") if self.segments[0].scheme in _FORMAT_SCHEMES: - raise URLPipelineError( - f"the format segment {self.segments[0].raw!r} cannot be the root of " - f"the URL pipeline {str(self)!r}; the root must locate a resource" - ) - for segment in self.segments[1:-1]: - if segment.scheme in _FORMAT_SCHEMES: - raise URLPipelineError( - f"the format segment {segment.raw!r} must be the last segment " - f"of the URL pipeline {str(self)!r}" - ) + raise URLPipelineError(f"a format segment cannot be the root of {self!r}") + if any(segment.scheme in _FORMAT_SCHEMES for segment in self.segments[1:-1]): + raise URLPipelineError(f"a format segment must be the last segment of {self!r}") format_segment = self._format_segment - if format_segment is not None and format_segment.query is not None: - raise URLPipelineError( - f"'{format_segment.scheme}:' segments do not accept a query: {format_segment.raw!r}" - ) if format_segment is not None: - # validate the node path now, before anything is resolved + if format_segment.query is not None: + raise URLPipelineError(f"format segments do not accept a query: {self!r}") try: normalize_path(format_segment.body) - except ValueError as exc: + except ValueError: raise URLPipelineError( - f"invalid node path in the format segment {format_segment.raw!r}: {exc}" + f"the node path of a format segment must not contain '.' or '..': {self!r}" ) from None - def __repr__(self) -> str: - return f"URLPipeline({str(self)!r})" - @classmethod def from_url(cls, url: str) -> URLPipeline: """ - Parse a URL pipeline string. - - A URL without a `|` is a trivial pipeline consisting of its root alone. - Parsing is purely syntactic: nothing is resolved, and no adapter is - loaded, until the pipeline is opened. + Parse a URL pipeline string. A URL without `|` is a pipeline of its + root alone. Parsing loads no adapter. Raises ------ TypeError If `url` is not a `str`. URLPipelineError - If the string cannot be parsed: an empty sub-URL, a root without a URL - scheme (including a bare local or Windows drive path), an invalid - adapter scheme, a fragment, an ASCII control character, or whitespace - at either end of a sub-URL; or if the resulting pipeline is invalid - (see the class docstring). + If the string cannot be parsed: an empty sub-URL, a root without a + URL scheme (including a local or Windows drive path), an invalid + scheme, a fragment, an ASCII control character, or whitespace at + either end of a sub-URL; or if the pipeline is invalid (see the + class docstring). """ - return cls(parse_pipeline(url)) + return cls(_parse_pipeline(url)) + + def __str__(self) -> str: + return "|".join(segment.raw for segment in self.segments) + + def __repr__(self) -> str: + return f"URLPipeline({_redact(str(self))!r})" @property def _format_segment(self) -> PipelineSegment | None: @@ -415,67 +296,6 @@ def _format_segment(self) -> PipelineSegment | None: return last return None - @property - def _store_segments(self) -> tuple[PipelineSegment, ...]: - """ - The segments that address a store: the root and any adapter segments, - i.e. `segments` without a trailing format segment. - """ - if self._format_segment is None: - return self.segments - return self.segments[:-1] - - @property - def _zarr_format(self) -> ZarrFormat | None: - """The format selected by a trailing `zarr2:`/`zarr3:` segment, or None - (also for a trailing `zarr:` segment, which detects the format).""" - format_segment = self._format_segment - if format_segment is None: - return None - return _FORMAT_SCHEMES[format_segment.scheme] - - @property - def _node_path(self) -> str: - """ - The node path given as the body of a trailing format segment, - normalized. Empty when there is no format segment or its body is empty. - """ - format_segment = self._format_segment - if format_segment is None: - return "" - return normalize_path(format_segment.body) - - def _resolve_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat | None: - """ - Combine the format selected by this pipeline with a caller-supplied one. - - Parameters - ---------- - zarr_format : ZarrFormat | None - The format requested by the caller, or None when unspecified. - - Returns - ------- - ZarrFormat | None - The pipeline's format when the caller did not specify one, otherwise - the caller's format. None when neither is set. - - Raises - ------ - ValueError - If the caller's format differs from the one selected by the pipeline. - """ - pipeline_format = self._zarr_format - if pipeline_format is None: - return zarr_format - if zarr_format is not None and zarr_format != pipeline_format: - raise ValueError( - f"zarr_format={zarr_format} conflicts with the 'zarr{pipeline_format}:' " - f"segment of the URL pipeline {str(self)!r}; pass zarr_format=None to use " - "the pipeline's format" - ) - return pipeline_format - async def _resolve_store_path( self, *, @@ -483,57 +303,17 @@ async def _resolve_store_path( storage_options: dict[str, Any] | None = None, ) -> StorePath: """ - Resolve the pipeline into a store and a residual path. - - The residual path joins the path returned by the adapters with the - node path of a trailing format segment (`...|zarr3:path/to/node`). - The format itself is not part of the result; read it from - `_zarr_format`. - - Parameters - ---------- - mode : AccessModeLiteral | None - The caller's access mode. `"r"` requires a read-only store; the - resolver enforces this on whatever the final adapter returns. - storage_options : dict | None - Options forwarded to the root sub-URL's store (and visible to - adapters via the context). - - Raises - ------ - URLPipelineError - If a segment names a scheme with no registered adapter, names an - adapter entry point that fails to import, or the root sub-URL - cannot be resolved. - TypeError - If `storage_options` are passed to a root that does not accept - them (`file:` and `memory:`), as for non-pipeline stores. - OSError - Errors from opening the root resource (e.g. a missing local - directory in mode `"r"`) propagate unchanged, as for non-pipeline - stores. + Resolve the pipeline into a store and a path: the adapters' residual + path joined with the node path of a trailing format segment. """ from zarr.storage._common import StorePath # circular import - resolved = await _resolve(self._store_segments, mode=mode, storage_options=storage_options) - if not self._node_path: - return resolved - # both paths are already normalized: StorePath normalizes the adapter's, - # and __post_init__ validated the format segment's - return StorePath(resolved.store, _join_paths([resolved.path, self._node_path])) - - def __str__(self) -> str: - return "|".join(segment.raw for segment in self.segments) - - -def _root_routes_to_adapter(scheme: str) -> bool: - """ - Whether a root sub-URL with this scheme is dispatched to a registered - root adapter. Schemes zarr resolves natively (`file:`, `memory:`) are - excluded; the registry check inspects entry-point names only — no - adapter code is imported here. - """ - return scheme not in _NATIVE_ROOT_SCHEMES and scheme in _list_url_adapter_schemes() + format_segment = self._format_segment + if format_segment is None: + return await _resolve(self.segments, mode=mode, storage_options=storage_options) + resolved = await _resolve(self.segments[:-1], mode=mode, storage_options=storage_options) + node_path = normalize_path(format_segment.body) + return StorePath(resolved.store, _join_paths([resolved.path, node_path])) async def _resolve( @@ -544,23 +324,19 @@ async def _resolve( ) -> StorePath: from zarr.storage._common import StorePath # circular import - if len(segments) == 1 and not _root_routes_to_adapter(segments[0].scheme): - return StorePath( - await _resolve_root(segments[0], mode=mode, storage_options=storage_options) - ) - *preceding, last = segments - adapter_cls = _get_url_adapter(last.scheme) + if not preceding and ( + last.scheme in _NATIVE_ROOT_SCHEMES or last.scheme not in _list_url_adapter_schemes() + ): + return StorePath(await _resolve_root(last, mode=mode, storage_options=storage_options)) + adapter_cls = _get_url_adapter(last.scheme) context = PipelineContext( - preceding=tuple(preceding), - mode=mode, - storage_options=storage_options, + preceding=tuple(preceding), mode=mode, storage_options=storage_options ) resolved = await adapter_cls.open_pipeline_segment(last, context) if mode == "r" and not resolved.store.read_only: - # The caller required read-only; enforce it rather than trusting - # the adapter to have honored context.read_only. + # enforce read-only rather than trusting the adapter try: read_only_store = resolved.store.with_read_only(True) await read_only_store._ensure_open() @@ -578,6 +354,14 @@ async def _resolve( return resolved +def _reject_dot_segments(path: str, segment: PipelineSegment) -> None: + if any(part in (".", "..") for part in re.split(r"[/\\]", path)): + raise URLPipelineError( + f"'{segment.scheme}:' pipeline root paths must not contain '.' or '..' " + f"components: {_redact(segment.raw)!r}" + ) + + async def _resolve_root( segment: PipelineSegment, *, @@ -585,96 +369,75 @@ async def _resolve_root( storage_options: dict[str, Any] | None, ) -> Store: """ - Resolve the root sub-URL of a pipeline into a store. - - `memory:` and `file:` roots are resolved here with the URL pipeline - spec's semantics (spelling equivalences, mandatory absolute `file:` - paths); every other scheme (e.g. fsspec URLs) delegates to the existing - `StoreLike` machinery unchanged. + Resolve a root sub-URL. `memory:` and `file:` follow the URL pipeline + spec; every other scheme goes to the plain-string machinery (fsspec). """ - if segment.scheme == "memory": - return _resolve_memory_root(segment, mode=mode, storage_options=storage_options) from zarr.storage._common import make_store # circular import + if segment.scheme in _NATIVE_ROOT_SCHEMES and segment.query is not None: + raise URLPipelineError( + f"'{segment.scheme}:' pipeline roots do not accept a query: {_redact(segment.raw)!r}" + ) + + if segment.scheme == "memory": + # one in-memory store; the body is a path within it + # (memory:a ≡ memory:/a ≡ memory://a) + if storage_options is not None: + raise TypeError( + "'storage_options' was provided but unused. " + "'storage_options' is only used when the store is passed as an FSSpec URI string.", + ) + path = segment.body.lstrip("/") + _reject_dot_segments(path, segment) + return ManagedMemoryStore(name="", path=path, read_only=mode == "r") + if segment.scheme == "file": - # Per the spec: file://localhost/p and file:///p are equivalent to - # file:/p; other authorities are unsupported; relative paths are - # forbidden; file: URLs carry no query. - if segment.query is not None: - raise URLPipelineError(f"'file:' pipeline roots do not accept a query: {segment.raw!r}") - body = segment.body + # file://localhost/p ≡ file:///p ≡ file:/p; other authorities and + # relative paths are rejected. On Windows '\' is a separator too, so + # check the '/'-spelled path (this also rejects UNC paths like /\host). + body = segment.body.replace("\\", "/") if sys.platform == "win32" else segment.body if body.startswith("//"): authority, sep, rest = body[2:].partition("/") - if authority not in ("", "localhost"): + if authority.lower() not in ("", "localhost"): raise URLPipelineError( - f"unsupported authority {authority!r} in 'file:' pipeline " - f"root {segment.raw!r}; only an empty authority or " - "'localhost' is allowed" + f"unsupported authority in the 'file:' pipeline root {_redact(segment.raw)!r}; " + "only an empty authority or 'localhost' is allowed" ) body = f"/{rest}" if sep else "" if body.startswith("//"): - # the spec's file-path-absolute grammar forbids an empty first - # path component raise URLPipelineError( - f"'file:' pipeline root paths must not start with '//': {segment.raw!r}" + f"'file:' pipeline root paths must not start with '//': " + f"{_redact(segment.raw)!r}" ) if sys.platform == "win32": - # a file URL spells a Windows drive path as /C:/...; strip the - # leading slash so the local-path machinery sees the drive + # a file URL spells a drive path /C:/...; drop the leading slash if body.startswith("/") and _WINDOWS_DRIVE_RE.match(body[1:]): body = body[1:] - is_absolute = body.startswith("/") or bool(_WINDOWS_DRIVE_RE.match(body)) + drive = PureWindowsPath(body).drive + is_absolute = (body.startswith("/") and not drive) or bool( + _WINDOWS_DRIVE_RE.match(body) + ) + if ":" in body[len(drive) :]: + # elsewhere ':' names an NTFS alternate data stream + raise URLPipelineError( + f"':' is only allowed in the drive of a 'file:' pipeline root on " + f"Windows: {_redact(segment.raw)!r}" + ) else: is_absolute = body.startswith("/") if not is_absolute: raise URLPipelineError( - f"'file:' pipeline roots must carry an absolute path: {segment.raw!r}" + f"'file:' pipeline roots must carry an absolute local path: " + f"{_redact(segment.raw)!r}" ) _reject_dot_segments(body, segment) return await make_store(Path(body), mode=mode, storage_options=storage_options) - # fsspec matches protocols case-sensitively; schemes are case-insensitive + # fsspec matches protocols case-sensitively; URL schemes are not url = segment.scheme + segment.raw[len(segment.scheme) :] try: return await make_store(url, mode=mode, storage_options=storage_options) except ValueError as exc: raise URLPipelineError( - f"could not resolve the pipeline root {segment.raw!r}: {exc}" + f"could not resolve the pipeline root {_redact(segment.raw)!r}: {_redact(str(exc))}" ) from exc - - -def _reject_dot_segments(path: str, segment: PipelineSegment) -> None: - """Reject `.` and `..` path components in a `file:` or `memory:` root.""" - if any(part in (".", "..") for part in re.split(r"[/\\]", path)): - raise URLPipelineError( - f"'{segment.scheme}:' pipeline root paths must not contain '.' or '..' " - f"components: {segment.raw!r}" - ) - - -# The single in-memory store every `memory:` pipeline root addresses. -_PIPELINE_MEMORY_STORE_NAME = "" - - -def _resolve_memory_root( - segment: PipelineSegment, - *, - mode: AccessModeLiteral | None, - storage_options: dict[str, Any] | None, -) -> Store: - """ - Resolve a `memory:` pipeline root per the spec: there is a single - in-memory store, and the body is a path within it. Spellings follow the - spec's equivalences: `memory:` ≡ `memory:/` ≡ `memory://`, and - `memory:a/b` ≡ `memory:/a/b` ≡ `memory://a/b`. - """ - if segment.query is not None: - raise URLPipelineError(f"'memory:' pipeline roots do not accept a query: {segment.raw!r}") - if storage_options is not None: - raise TypeError( - "'storage_options' was provided but unused. " - "'storage_options' is only used when the store is passed as an FSSpec URI string.", - ) - path = segment.body.lstrip("/") - _reject_dot_segments(path, segment) - return ManagedMemoryStore(name=_PIPELINE_MEMORY_STORE_NAME, path=path, read_only=mode == "r") diff --git a/tests/test_url_pipeline/test_open_path.py b/tests/test_url_pipeline/test_open_path.py index 8e294d273f..112b92213f 100644 --- a/tests/test_url_pipeline/test_open_path.py +++ b/tests/test_url_pipeline/test_open_path.py @@ -135,7 +135,7 @@ def test_format_scheme_cannot_be_root(self, url: str) -> None: @pytest.mark.parametrize("body", ["../x", "a/../../x", "./x", "a/./b"]) def test_node_path_dot_segments_rejected_at_parse(self, body: str) -> None: - with pytest.raises(URLPipelineError, match="invalid node path"): + with pytest.raises(URLPipelineError, match="node path of a format segment"): URLPipeline.from_url(f"memory:|zarr3:{body}") async def test_node_path_traversal_creates_nothing(self, tmp_path: Path) -> None: @@ -218,9 +218,10 @@ async def test_dot_segments_rejected(self, body: str) -> None: with pytest.raises(URLPipelineError, match=r"'\.' or '\.\.'"): await make_store_path(URLPipeline.from_url(f"memory:{body}")) - async def test_empty_storage_options_rejected(self) -> None: + @pytest.mark.parametrize("storage_options", [{}, {"anon": True}]) + async def test_storage_options_rejected(self, storage_options: dict[str, object]) -> None: with pytest.raises(TypeError, match="'storage_options' was provided but unused"): - await make_store_path(URLPipeline.from_url("memory:x"), storage_options={}) + await make_store_path(URLPipeline.from_url("memory:x"), storage_options=storage_options) class TestRootSchemeCase: @@ -236,9 +237,7 @@ async def test_uppercase_fsspec_root(self) -> None: store_path = await make_store_path(URLPipeline.from_url("HTTPS://example.com/d")) assert isinstance(store_path.store, FsspecStore) - assert store_path.store.path.startswith("https://") or "example.com" in str( - store_path.store - ) + assert "https" in store_path.store.fs.protocol class TestRegisterURLAdapter: @@ -296,3 +295,114 @@ async def test_group_not_found_message_is_readable(self, tmp_path: Path) -> None with pytest.raises(GroupNotFoundError) as info: zarr.open_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr3:"), mode="r") assert "PipelineSegment(" not in str(info.value) + + +class TestSecretsAreRedacted: + URL = "https://user:pw@h/x?token=SECRET|rec:|zarr2:" + + def test_reprs(self) -> None: + pipeline = URLPipeline.from_url(self.URL) + context = zarr.storage.PipelineContext( + preceding=pipeline.segments[:1], mode="r", storage_options={"secret": "SECRET"} + ) + for text in (repr(pipeline), repr(pipeline.segments[0]), repr(context)): + assert "SECRET" not in text + assert "pw" not in text + assert str(pipeline) == self.URL # str() stays lossless + + @pytest.mark.parametrize( + "url", + [ + "https://h/x?token=SECRET| zip:", + "https://h/x?token=SECRET||zip:", + "https://h/x?token=SECRET#frag", + "https://h/x?token=SECRET|zarr3:|zip:", + "/x?token=SECRET", + "a_b://x?token=SECRET", + ], + ) + def test_parse_errors(self, url: str) -> None: + with pytest.raises(URLPipelineError) as info: + URLPipeline.from_url(url) + assert "SECRET" not in str(info.value) + + async def test_resolution_errors(self, recording: type[RecordingAdapter]) -> None: + with pytest.raises(ValueError, match="conflicts with") as conflict: + zarr.open_group(URLPipeline.from_url(self.URL), mode="w", zarr_format=3) + with pytest.raises(URLPipelineError) as residual: + await make_store(URLPipeline.from_url("memory:?token=SECRET|rec:sub")) + with pytest.raises(URLPipelineError) as root: + await make_store_path(URLPipeline.from_url("vendor.x://[a]/p?token=SECRET")) + for info in (conflict, residual, root): + assert "SECRET" not in str(info.value) + + +class TestEntryPointNames: + @pytest.mark.parametrize("name", ["file", "Memory", "zarr3", "a b", "x\n"]) + def test_unusable_entry_point_names_ignored( + self, name: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + from importlib.metadata import EntryPoint, EntryPoints + + fake = EntryPoints( + [EntryPoint(name=name, value="nowhere:Adapter", group="zarr.url_adapters")] + ) + monkeypatch.setattr(zarr.registry, "get_entry_points", lambda: fake) + zarr.registry._url_adapter_registry.lazy_load_list.clear() + with pytest.warns(ZarrUserWarning, match="ignoring the 'zarr.url_adapters' entry point"): + zarr.registry._collect_entrypoints() + assert zarr.registry._url_adapter_registry.lazy_load_list == [] + + def test_no_adapter_error_does_not_list_installed_schemes( + self, recording: type[RecordingAdapter] + ) -> None: + with pytest.raises(URLPipelineError) as info: + zarr.core.sync.sync(make_store_path(URLPipeline.from_url("memory:|nope:"))) + assert "rec" not in str(info.value) + + @pytest.mark.parametrize("scheme", ["foo\n", "\u212aelvin"]) + def test_register_rejects_scheme_tricks(self, scheme: str) -> None: + with pytest.raises(ValueError, match="not a valid URL scheme"): + register_url_adapter(scheme, RecordingAdapter) + + +class TestLimits: + def test_hand_built_pipeline_segment_limit(self) -> None: + segment = PipelineSegment("w", "", None, "w") + root = PipelineSegment("memory", "", None, "memory:") + URLPipeline((root, *[segment] * 31)) + with pytest.raises(URLPipelineError, match="1 to 32 segments"): + URLPipeline((root, *[segment] * 32)) + + +class TestWindowsFileRoots: + """Windows path rules, checked on any platform by patching `sys.platform`.""" + + @pytest.mark.parametrize( + "url", + [ + r"file:/\attacker\share\data", + r"file:///\attacker\share", + r"file://localhost/\attacker\share", + r"file:\\attacker\share", + ], + ) + async def test_unc_paths_rejected(self, url: str, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys, "platform", "win32") + with pytest.raises(URLPipelineError, match="authority|'//'|absolute local path"): + await make_store_path(URLPipeline.from_url(url), mode="r") + + @pytest.mark.parametrize( + "url", [r"file:C:x", r"file:/C:..\x", "file:C:/a/../b", "file:/C:/a:stream"] + ) + async def test_drive_relative_and_dot_paths_rejected( + self, url: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(sys, "platform", "win32") + with pytest.raises(URLPipelineError, match=r"absolute local path|'\.' or '\.\.'|':'"): + await make_store_path(URLPipeline.from_url(url), mode="r") + + +async def test_file_authority_is_case_insensitive(tmp_path: Path) -> None: + zarr.open_group(URLPipeline.from_url(f"file://LOCALHOST{tmp_path.as_posix()}"), mode="w") + assert (tmp_path / "zarr.json").exists() diff --git a/tests/test_url_pipeline/test_parser.py b/tests/test_url_pipeline/test_parser.py index 0bf76d7764..f60f576f77 100644 --- a/tests/test_url_pipeline/test_parser.py +++ b/tests/test_url_pipeline/test_parser.py @@ -5,41 +5,40 @@ from hypothesis import strategies as st from zarr.errors import URLPipelineError -from zarr.storage._url_pipeline import parse_pipeline +from zarr.storage._url_pipeline import _parse_pipeline def test_single_root_url() -> None: - (segment,) = parse_pipeline("s3://bucket/key") + (segment,) = _parse_pipeline("s3://bucket/key") assert segment.scheme == "s3" assert segment.body == "//bucket/key" assert segment.query is None assert segment.raw == "s3://bucket/key" - assert str(segment) == "s3://bucket/key" @pytest.mark.parametrize("url", ["/local/path", "/local/path|zip:", "relative/path|zip:"]) def test_schemeless_root_rejected(url: str) -> None: """The root sub-URL must carry a URL scheme; a bare path (or anything urlparse would not read as a scheme) is an error.""" with pytest.raises(URLPipelineError, match="URL scheme"): - parse_pipeline(url) + _parse_pipeline(url) def test_adapter_chain() -> None: - segments = parse_pipeline("s3://bucket/data.zip|zip:inner/path|zarr3:") + segments = _parse_pipeline("s3://bucket/data.zip|zip:inner/path|zarr3:") assert [s.scheme for s in segments] == ["s3", "zip", "zarr3"] assert segments[1].body == "inner/path" assert segments[2].body == "" def test_trailing_colon_optional() -> None: - with_colon = parse_pipeline("file:/tmp/x.zip|zip:") - without_colon = parse_pipeline("file:/tmp/x.zip|zip") + with_colon = _parse_pipeline("file:/tmp/x.zip|zip:") + without_colon = _parse_pipeline("file:/tmp/x.zip|zip") assert with_colon[1].scheme == without_colon[1].scheme == "zip" assert with_colon[1].body == without_colon[1].body == "" def test_scheme_case_insensitive() -> None: - segments = parse_pipeline("FILE:/tmp/x.zip|ZIP:Inner/Path") + segments = _parse_pipeline("FILE:/tmp/x.zip|ZIP:Inner/Path") assert segments[0].scheme == "file" assert segments[1].scheme == "zip" # bodies are case-preserved @@ -48,19 +47,19 @@ def test_scheme_case_insensitive() -> None: def test_case_preserved_in_raw() -> None: # e.g. icechunk snapshot IDs are case-significant - segments = parse_pipeline("file:/tmp/repo|icechunk://ABCDEFGH12345678ABCD/x") + segments = _parse_pipeline("file:/tmp/repo|icechunk://ABCDEFGH12345678ABCD/x") assert segments[1].raw == "icechunk://ABCDEFGH12345678ABCD/x" assert segments[1].body == "//ABCDEFGH12345678ABCD/x" def test_vendor_prefixed_scheme() -> None: - segments = parse_pipeline("file:/data|vendor-1.custom+adapter:sub/path") + segments = _parse_pipeline("file:/data|vendor-1.custom+adapter:sub/path") assert segments[1].scheme == "vendor-1.custom+adapter" assert segments[1].body == "sub/path" def test_query_strings() -> None: - segments = parse_pipeline("https://example.com/d.zip?token=abc|zip:x?opt=1") + segments = _parse_pipeline("https://example.com/d.zip?token=abc|zip:x?opt=1") assert segments[0].query == "token=abc" assert segments[0].body == "//example.com/d.zip" assert segments[1].query == "opt=1" @@ -68,27 +67,38 @@ def test_query_strings() -> None: def test_empty_query() -> None: - (segment,) = parse_pipeline("https://example.com/d?") + (segment,) = _parse_pipeline("https://example.com/d?") assert segment.query == "" -@pytest.mark.parametrize("url", [r"C:\data\store", "C:/data/store|zip:", "c:\\x"]) -def test_windows_drive_root_rejected(url: str) -> None: - """A bare Windows drive path is rejected on every platform, not read as scheme `c`.""" +@pytest.mark.parametrize( + "url", [r"C:\data\store", "C:/data/store|zip:", "c:\\x", "c:data", "C:..\\x", "x://host/p"] +) +def test_single_letter_root_scheme_rejected(url: str) -> None: + """A one-letter root scheme reads as a drive path on Windows, so it is rejected everywhere.""" with pytest.raises(URLPipelineError, match="Windows drive path"): - parse_pipeline(url) + _parse_pipeline(url) + + +@pytest.mark.parametrize("scheme", ["\u212aelvin", "s\u0131", "zi\u0440"]) +def test_non_ascii_scheme_rejected(scheme: str) -> None: + """Validation happens before lowercasing, so a look-alike cannot alias an ASCII scheme.""" + with pytest.raises(URLPipelineError, match="invalid URL scheme"): + _parse_pipeline(f"{scheme}://x") + with pytest.raises(URLPipelineError, match="invalid URL scheme"): + _parse_pipeline(f"s3://b|{scheme}:x") -def test_single_letter_scheme_without_slash_is_a_scheme() -> None: - (segment,) = parse_pipeline("c:data") - assert segment.scheme == "c" - assert segment.body == "data" +def test_segment_limit() -> None: + _parse_pipeline("memory:" + "|w" * 31) + with pytest.raises(URLPipelineError, match="more than 32 segments"): + _parse_pipeline("memory:" + "|w" * 32) @pytest.mark.parametrize("url", [None, 42, b"s3://b|zip:", ["s3://b"]]) def test_non_string_rejected(url: object) -> None: with pytest.raises(TypeError, match="must be a str"): - parse_pipeline(url) # type: ignore[arg-type] + _parse_pipeline(url) # type: ignore[arg-type] @pytest.mark.parametrize( @@ -97,18 +107,18 @@ def test_non_string_rejected(url: object) -> None: ) def test_control_characters_rejected(url: str) -> None: with pytest.raises(URLPipelineError, match="control character"): - parse_pipeline(url) + _parse_pipeline(url) @pytest.mark.parametrize("url", [" s3://b", "s3://b |zip:", "s3://b| zip:", "s3://b|zip:x\u00a0"]) def test_surrounding_whitespace_rejected(url: str) -> None: with pytest.raises(URLPipelineError, match="leading or trailing whitespace"): - parse_pipeline(url) + _parse_pipeline(url) def test_interior_whitespace_is_preserved() -> None: """zarr does not percent-decode, so a key with a space can only be spelled with one.""" - segments = parse_pipeline("s3://b/My Data.zip|zip:a b") + segments = _parse_pipeline("s3://b/My Data.zip|zip:a b") assert segments[0].body == "//b/My Data.zip" assert segments[1].body == "a b" @@ -116,37 +126,37 @@ def test_interior_whitespace_is_preserved() -> None: @pytest.mark.parametrize("url", ["a||b:", "|zip:", "file:/tmp|", ""]) def test_empty_sub_url_rejected(url: str) -> None: with pytest.raises(URLPipelineError, match="empty sub-URL"): - parse_pipeline(url) + _parse_pipeline(url) def test_fragment_rejected() -> None: with pytest.raises(URLPipelineError, match="fragment"): - parse_pipeline("file:/tmp/x.zip|zip:inner#frag") + _parse_pipeline("file:/tmp/x.zip|zip:inner#frag") @pytest.mark.parametrize("segment", ["1zip:", "zi p:x", "zip@:x"]) def test_invalid_adapter_scheme_rejected(segment: str) -> None: - with pytest.raises(URLPipelineError, match="invalid adapter scheme"): - parse_pipeline(f"file:/tmp/x|{segment}") + with pytest.raises(URLPipelineError, match="invalid URL scheme"): + _parse_pipeline(f"file:/tmp/x|{segment}") -def test_single_letter_scheme_with_exotic_authority() -> None: +def test_scheme_detection_ignores_authority_syntax() -> None: """Scheme detection works on the raw text, so an authority urlparse would reject is no obstacle.""" - segments = parse_pipeline("x://[authority]/path|zip:") - assert segments[0].scheme == "x" - assert segments[0].raw == "x://[authority]/path" + segments = _parse_pipeline("xy://[authority]/path|zip:") + assert segments[0].scheme == "xy" + assert segments[0].raw == "xy://[authority]/path" def test_round_trip() -> None: url = "s3://bucket/a.zip?v=2|zip:b/inner.zip|zip:c|zarr3:" - segments = parse_pipeline(url) + segments = _parse_pipeline(url) assert "|".join(s.raw for s in segments) == url # --- property-based tests ----------------------------------------------- # # These exercise the parser's *splitting invariants*, not URL pipeline spec -# validity: parse_pipeline is a permissive segment splitter (bodies and +# validity: _parse_pipeline is a permissive segment splitter (bodies and # queries are opaque text to it), and semantic validation is left to the # resolver and the adapters. Rather than enumerating registered schemes, # the strategies sample the full scheme grammar the parser accepts @@ -191,7 +201,7 @@ def test_valid_pipeline_invariants( case: tuple[str, list[tuple[str, str, str | None, str]]], ) -> None: url, expected = case - segments = parse_pipeline(url) + segments = _parse_pipeline(url) # schemes lowercased; bodies and queries preserved verbatim assert [(s.scheme, s.body, s.query, s.raw) for s in segments] == expected # lossless round trip @@ -206,7 +216,7 @@ def test_property_empty_sub_url_rejected( parts = url.split("|") parts.insert(data.draw(st.integers(0, len(parts))), "") with pytest.raises(URLPipelineError, match="empty sub-URL"): - parse_pipeline("|".join(parts)) + _parse_pipeline("|".join(parts)) @given(pipelines(min_depth=2), st.data()) @@ -219,8 +229,8 @@ def test_property_invalid_adapter_scheme_rejected( # scheme grammar forbids index = data.draw(st.integers(1, len(parts) - 1)) parts[index] = data.draw(st.sampled_from(["1", "@", "~"])) + parts[index] - with pytest.raises(URLPipelineError, match="invalid adapter scheme"): - parse_pipeline("|".join(parts)) + with pytest.raises(URLPipelineError, match="invalid URL scheme"): + _parse_pipeline("|".join(parts)) @given(pipelines(), st.data()) @@ -231,4 +241,4 @@ def test_property_fragment_rejected( parts = url.split("|") parts[data.draw(st.integers(0, len(parts) - 1))] += "#frag" with pytest.raises(URLPipelineError, match="fragment"): - parse_pipeline("|".join(parts)) + _parse_pipeline("|".join(parts)) diff --git a/tests/test_url_pipeline/test_pipeline_object.py b/tests/test_url_pipeline/test_pipeline_object.py index 62282233c6..70fd051000 100644 --- a/tests/test_url_pipeline/test_pipeline_object.py +++ b/tests/test_url_pipeline/test_pipeline_object.py @@ -4,64 +4,27 @@ from zarr.errors import URLPipelineError from zarr.storage import PipelineSegment, URLPipeline -from zarr.storage._url_pipeline import parse_pipeline +from zarr.storage._url_pipeline import _parse_pipeline @pytest.mark.parametrize( - ("url", "schemes", "store_schemes", "zarr_format", "path"), + "url", [ - ( - "s3://bucket/data.zip|zip:inner|zarr3:", - ("s3", "zip", "zarr3"), - ("s3", "zip"), - 3, - "", - ), - # the trailing colon of a format segment is optional - ("file:///data/node.zarr|zarr2", ("file", "zarr2"), ("file",), 2, ""), - # the body of a format segment is a node path, normalized - ( - "file:///repo|icechunk:|zarr3:/path/to/array/", - ("file", "icechunk", "zarr3"), - ("file", "icechunk"), - 3, - "path/to/array", - ), - # no format segment - ("memory://x|wrap:a", ("memory", "wrap"), ("memory", "wrap"), None, ""), - # a single URL is a trivial pipeline - ("s3://bucket/key", ("s3",), ("s3",), None, ""), + "s3://bucket/data.zip|zip:inner|zarr3:", + "file:///data/node.zarr|zarr2", + "file:///repo|icechunk:|zarr3:/path/to/array/", + "memory://x|wrap:a", + "s3://bucket/key", ], ) -def test_from_url( - url: str, - schemes: tuple[str, ...], - store_schemes: tuple[str, ...], - zarr_format: int | None, - path: str, -) -> None: - """ - `URLPipeline.from_url` holds every parsed segment of the URL, `str()` - reproduces the URL exactly, and the zarr format, node path and - store-addressing segments are derived from a trailing `zarr2:`/`zarr3:` - segment. - """ +def test_from_url(url: str) -> None: + """`from_url` holds every parsed segment; `str()` reproduces the URL exactly.""" pipeline = URLPipeline.from_url(url) - assert pipeline.segments == parse_pipeline(url) - assert tuple(segment.scheme for segment in pipeline.segments) == schemes - assert tuple(segment.scheme for segment in pipeline._store_segments) == store_schemes - assert pipeline._zarr_format == zarr_format - assert pipeline._node_path == path + assert pipeline.segments == _parse_pipeline(url) assert str(pipeline) == url assert URLPipeline(pipeline.segments) == pipeline -def test_format_segment_must_be_last() -> None: - """A `zarr2:`/`zarr3:` segment anywhere but the end of the pipeline is rejected.""" - with pytest.raises(URLPipelineError, match="must be the last segment"): - URLPipeline.from_url("file:/data|zarr3:|zip:") - - def test_format_segment_rejects_query() -> None: """A `zarr2:`/`zarr3:` segment takes a node path only; a query is an error.""" with pytest.raises(URLPipelineError, match="do not accept a query"): @@ -70,14 +33,14 @@ def test_format_segment_rejects_query() -> None: def test_direct_construction_is_validated() -> None: """Validation lives in the constructor, so a hand-built pipeline is checked too.""" - segments = parse_pipeline("file:/data|zarr3:|zip:") + segments = _parse_pipeline("file:/data|zarr3:|zip:") with pytest.raises(URLPipelineError, match="must be the last segment"): URLPipeline(segments) def test_empty_pipeline_rejected() -> None: """A pipeline needs at least a root segment.""" - with pytest.raises(URLPipelineError, match="at least one segment"): + with pytest.raises(URLPipelineError, match="1 to 32 segments"): URLPipeline(()) @@ -104,26 +67,3 @@ def test_hashable() -> None: assert a == b assert hash(a) == hash(b) assert {a: 1}[b] == 1 - - -@pytest.mark.parametrize( - ("pipeline_format", "requested", "expected"), - [(None, None, None), (None, 3, 3), (2, None, 2), (2, 2, 2)], -) -def test_resolve_zarr_format( - pipeline_format: int | None, requested: int | None, expected: int | None -) -> None: - """ - `resolve_zarr_format` returns the pipeline's format when the caller did not - specify one, and the caller's format otherwise. - """ - suffix = "" if pipeline_format is None else f"|zarr{pipeline_format}:" - pipeline = URLPipeline.from_url(f"memory://x{suffix}") - assert pipeline._resolve_zarr_format(requested) == expected # type: ignore[arg-type] - - -def test_resolve_zarr_format_conflict_raises() -> None: - """A caller-supplied format that differs from the pipeline's segment is an error.""" - pipeline = URLPipeline.from_url("memory://x|zarr2:") - with pytest.raises(ValueError, match="conflicts with the 'zarr2:' segment"): - pipeline._resolve_zarr_format(3) diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index 3f2825412e..563394eb5f 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -346,15 +346,15 @@ async def open_pipeline_segment( await make_store_path("rooty://org/repo") assert calls == [] - def test_native_schemes_do_not_route_to_adapters(self) -> None: - """An installed package cannot intercept `file:`/`memory:` roots, even via the registry.""" - # register_url_adapter refuses these schemes; an entry point could still - # name them, so put them in the registry directly + async def test_native_schemes_do_not_route_to_adapters(self) -> None: + """Even an adapter forced into the registry never handles `file:`/`memory:`.""" zarr.registry._url_adapter_registry.register(RootAdapter, "file") zarr.registry._url_adapter_registry.register(RootAdapter, "memory") - pipeline = URLPipeline.from_url("memory://x") - store = zarr.core.sync.sync(make_store(pipeline)) + store = await make_store(URLPipeline.from_url("memory://x")) assert isinstance(store, ManagedMemoryStore) + for url in ("memory:|file:/etc", "memory:|memory:x"): + with pytest.raises(URLPipelineError, match="cannot be an adapter segment"): + await make_store_path(URLPipeline.from_url(url)) class TestResolve: @@ -575,15 +575,6 @@ async def test_single_url_is_a_trivial_pipeline(self) -> None: assert isinstance(result.store, ManagedMemoryStore) assert result.path == "" - async def test_resolve_method(self, tmp_path: Path) -> None: - """`URLPipeline.resolve` yields the adapter's store and residual path.""" - register_url_adapter("wrap", WrapperAdapter) - result = await URLPipeline.from_url( - f"file:{tmp_path.as_posix()}|wrap:a" - )._resolve_store_path() - assert isinstance(result.store, TracingStore) - assert result.path == "a" - async def test_format_segment_path_joins_residual_path(self, tmp_path: Path) -> None: """The node path of a trailing format segment extends the adapters' residual path.""" register_url_adapter("wrap", WrapperAdapter) @@ -597,8 +588,7 @@ async def test_unknown_adapter_scheme_raises(self) -> None: await URLPipeline.from_url("memory://base|no-such-adapter:")._resolve_store_path() async def test_base_case_value_error_is_wrapped(self) -> None: - # a root that only the fallback scheme detection accepts must not - # leak urlparse's ValueError out of the base case + # an authority urlparse rejects must not leak its ValueError register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="could not resolve the pipeline root"): await URLPipeline.from_url("vendor.x://[authority]/path|wrap:")._resolve_store_path() @@ -628,25 +618,11 @@ async def test_memory_root_path_spellings_equivalent(self, root: str) -> None: assert inner._name == "" assert inner.path == "a/b" - async def test_memory_root_shares_data_across_spellings(self) -> None: - register_url_adapter("wrap", WrapperAdapter) - group = zarr.open_group(URLPipeline.from_url("memory:pipe-share|wrap:"), mode="w") - group.create_array("x", shape=(2,), dtype="i4") - reopened = zarr.open_group(URLPipeline.from_url("memory://pipe-share|wrap:"), mode="r") - assert "x" in reopened - async def test_memory_root_rejects_query(self) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="do not accept a query"): await URLPipeline.from_url("memory:a?opt=1|wrap:")._resolve_store_path() - async def test_memory_root_rejects_storage_options(self) -> None: - register_url_adapter("wrap", WrapperAdapter) - with pytest.raises(TypeError, match="'storage_options' was provided but unused"): - await URLPipeline.from_url("memory:a|wrap:")._resolve_store_path( - storage_options={"anon": True} - ) - async def test_file_root_localhost_authority(self, tmp_path: Path) -> None: # file://localhost/p and file:///p are equivalent to file:/p register_url_adapter("wrap", WrapperAdapter) @@ -672,7 +648,7 @@ async def test_file_root_windows_drive_spellings(self, tmp_path: Path) -> None: @pytest.mark.skipif(sys.platform == "win32", reason="POSIX path semantics") async def test_file_root_drive_letter_is_not_special_on_posix(self) -> None: """Off Windows, `/C:/x` is an ordinary absolute path and `C:/x` is relative.""" - with pytest.raises(URLPipelineError, match="absolute path"): + with pytest.raises(URLPipelineError, match="absolute local path"): await make_store_path(URLPipeline.from_url("file:C:/drive/data"), mode="r") with pytest.raises(FileNotFoundError, match="/C:/drive/data"): await make_store_path(URLPipeline.from_url("file:///C:/drive/data"), mode="r") @@ -685,7 +661,7 @@ async def test_file_root_rejects_other_authority(self) -> None: @pytest.mark.parametrize("root", ["file:relative/path", "file://localhost"]) async def test_file_root_rejects_relative_paths(self, root: str) -> None: register_url_adapter("wrap", WrapperAdapter) - with pytest.raises(URLPipelineError, match="absolute path"): + with pytest.raises(URLPipelineError, match="absolute local path"): await URLPipeline.from_url(f"{root}|wrap:")._resolve_store_path() async def test_file_root_rejects_query(self, tmp_path: Path) -> None: @@ -754,26 +730,6 @@ async def open_pipeline_segment( store = await make_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|slashy:")) assert store is not None - async def test_make_store_closes_store_on_residual_path_error(self) -> None: - closed: list[bool] = [] - - class ClosingStore(MemoryStore): - def close(self) -> None: - closed.append(True) - super().close() - - class LeakProbe(URLPipelineAdapter): - @classmethod - async def open_pipeline_segment( - cls, segment: PipelineSegment, context: PipelineContext - ) -> StorePath: - return StorePath(await ClosingStore.open(), path="residual") - - register_url_adapter("leaky", LeakProbe) - with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): - await make_store(URLPipeline.from_url("memory://base|leaky:")) - assert closed == [True] - async def test_make_store_no_residual_path(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) store = await make_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:")) @@ -802,13 +758,10 @@ async def test_pipeline_store_read_only_mode(self) -> None: assert store_path.read_only async def test_mode_a_on_read_only_store_raises(self) -> None: - """Mode "a" on a read-only store raises, whether the store comes from a pipeline or not.""" + """Mode "a" on a read-only adapter store raises, as for any read-only store.""" register_url_adapter("rooty", RootAdapter) with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"): await make_store_path(URLPipeline.from_url("rooty://org/repo"), mode="a") - store = await MemoryStore.open(read_only=True) - with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"): - await make_store_path(store, mode="a") async def test_explicit_write_mode_reaches_adapter(self) -> None: register_url_adapter("rooty", RootAdapter) @@ -825,31 +778,6 @@ async def test_storage_options_forwarded_to_root(self) -> None: URLPipeline.from_url("file:/tmp/some/path|wrap:"), storage_options={"anon": True} ) - async def test_store_path_has_no_format(self, tmp_path: Path) -> None: - """The pipeline's format is consumed by the open call; it does not linger on the StorePath.""" - store_path = await make_store_path( - URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:") - ) - assert not hasattr(store_path, "zarr_format") - - async def test_zarr_format_merges_into_open(self, tmp_path: Path) -> None: - group = zarr.open_group( - URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:"), mode="w" - ) - assert group.metadata.zarr_format == 2 - - async def test_conflicting_explicit_format_raises(self, tmp_path: Path) -> None: - with pytest.raises(ValueError, match="conflicts with"): - zarr.open_group( - URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:"), mode="w", zarr_format=3 - ) - - async def test_matching_explicit_format_ok(self, tmp_path: Path) -> None: - group = zarr.open_group( - URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:"), mode="w", zarr_format=2 - ) - assert group.metadata.zarr_format == 2 - class TestOpenFunctionsAcceptPipelines: """ diff --git a/tests/test_url_pipeline/test_spec_examples.py b/tests/test_url_pipeline/test_spec_examples.py index 5538074bea..bf62548c34 100644 --- a/tests/test_url_pipeline/test_spec_examples.py +++ b/tests/test_url_pipeline/test_spec_examples.py @@ -13,7 +13,7 @@ into the expected sub-URL schemes, and preserve the text losslessly. To regenerate after a spec update: extract examples per the rule above and -recompute the expected scheme tuples with `parse_pipeline`, reviewing any +recompute the expected scheme tuples with `_parse_pipeline`, reviewing any changes against the spec diff. """ @@ -23,7 +23,7 @@ import pytest -from zarr.storage._url_pipeline import parse_pipeline +from zarr.storage._url_pipeline import _parse_pipeline SPEC_EXAMPLES: list[tuple[str, tuple[str, ...]]] = [ # README.md @@ -245,7 +245,7 @@ def repl(match: re.Match[str]) -> str: @pytest.mark.parametrize(("url", "expected_schemes"), SPEC_EXAMPLES, ids=lambda v: str(v)) def test_spec_example_parses(url: str, expected_schemes: tuple[str, ...]) -> None: - segments = parse_pipeline(url) + segments = _parse_pipeline(url) assert tuple(segment.scheme for segment in segments) == expected_schemes # lossless round-trip of the original text assert "|".join(segment.raw for segment in segments) == url @@ -254,5 +254,5 @@ def test_spec_example_parses(url: str, expected_schemes: tuple[str, ...]) -> Non @pytest.mark.parametrize(("url", "expected_schemes"), SPEC_EXAMPLES, ids=lambda v: str(v)) def test_spec_example_schemes_case_insensitive(url: str, expected_schemes: tuple[str, ...]) -> None: upper = _uppercase_schemes(url) - segments = parse_pipeline(upper) + segments = _parse_pipeline(upper) assert tuple(segment.scheme for segment in segments) == expected_schemes From 5ef0956b8cf278eadf6a97fedccacfea6e9206dd Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 23 Oct 2026 00:32:57 +0200 Subject: [PATCH 18/20] docs(url-pipeline): polish wording, comments and test names Shorter module docstring, changelog and adapter-authoring notes; clearer error-message wording; comments that restated the code trimmed. Tests: names and docstrings that match what they check, sync tests that never await, related tests grouped, a dead registration removed. Co-Authored-By: Claude Opus 5.5 --- changes/4192.feature.md | 16 +++-- docs/user-guide/extending.md | 15 ++--- docs/user-guide/storage.md | 12 ++-- src/zarr/registry.py | 21 ++++--- src/zarr/storage/_common.py | 6 +- src/zarr/storage/_url_pipeline.py | 31 ++++------ tests/test_url_pipeline/test_open_path.py | 59 +++++++++---------- tests/test_url_pipeline/test_parser.py | 16 ++--- .../test_url_pipeline/test_pipeline_object.py | 2 +- tests/test_url_pipeline/test_resolver.py | 28 ++++----- tests/test_url_pipeline/test_spec_examples.py | 17 +++--- 11 files changed, 102 insertions(+), 121 deletions(-) diff --git a/changes/4192.feature.md b/changes/4192.feature.md index b722ca82c6..9ecd24aa17 100644 --- a/changes/4192.feature.md +++ b/changes/4192.feature.md @@ -1,9 +1,7 @@ -Added core support for URL pipelines (https://github.com/jbms/url-pipeline): -`|`-chained URLs such as `s3://bucket/data.zip|zip:|zarr3:` that address zarr data -through nested storage layers. Parse one with `zarr.storage.URLPipeline.from_url` and -pass it to any function that accepts a `StoreLike`; plain strings are never read as -pipelines. A trailing `zarr:`, `zarr2:` or `zarr3:` segment gives the node path and -selects (or, for `zarr:`, detects) the zarr format. Packages such as Icechunk provide -adapters for other schemes by subclassing `zarr.abc.url_pipeline.URLPipelineAdapter` -and registering it under the `zarr.url_adapters` entry-point group, or with -`zarr.registry.register_url_adapter`. See the storage and extending user guides. +Added core support for URL pipelines (https://github.com/jbms/url-pipeline), `|`-chained +URLs such as `s3://bucket/data.zip|zip:|zarr3:`. Parse one with +`zarr.storage.URLPipeline.from_url` and pass it to any function that accepts a `StoreLike`; +plain strings are never read as pipelines. A trailing `zarr:`, `zarr2:` or `zarr3:` segment +gives the node path and selects (or detects) the format. Packages provide adapters for other +schemes by subclassing `zarr.abc.url_pipeline.URLPipelineAdapter` and registering it under the +`zarr.url_adapters` entry-point group or with `zarr.registry.register_url_adapter`. diff --git a/docs/user-guide/extending.md b/docs/user-guide/extending.md index b104199159..bf0eb442c9 100644 --- a/docs/user-guide/extending.md +++ b/docs/user-guide/extending.md @@ -104,7 +104,7 @@ class MyAdapter(URLPipelineAdapter): async def open_pipeline_segment( cls, segment: PipelineSegment, context: PipelineContext ) -> StorePath: - # A *wrapper* adapter opens the resource to its left and wraps it, joining the + # A wrapper adapter opens the resource to its left and wraps it, joining the # preceding residual path with this segment's own path. preceding = await context.resolve_preceding() store = WrapperStore(preceding.store) @@ -123,6 +123,7 @@ from zarr.storage import URLPipeline register_url_adapter("mypackage.myscheme", MyAdapter) url = "memory://x|mypackage.myscheme:a|mypackage.myscheme:b" +# keep a reference: in-memory data lives only while a store refers to it created = zarr.create_group(URLPipeline.from_url(url)) group = zarr.open_group(URLPipeline.from_url(url), mode="r") assert group.path == "a/b" @@ -143,12 +144,12 @@ that scheme over at the root of every pipeline. The schemes zarr handles itself (`file`, `memory`, `zarr`, `zarr2`, `zarr3`) cannot be registered, and entry points with those names are ignored. -Every adapter sees all of the caller's `storage_options`, including credentials meant -for the root: consume only keys namespaced for your adapter, never log the options, and -strip your keys before resolving the rest of the pipeline. The returned store must honor -`context.read_only`. `open_pipeline_segment` runs on zarr's event loop, so it must not -block or call zarr's synchronous API. See [`PipelineContext`][zarr.storage.PipelineContext] -for the full contract. +Every adapter sees all of the caller's `storage_options`, including the root's +credentials: consume only keys namespaced for your adapter, strip them before resolving +the rest of the pipeline, and never log the options. The returned store must honor +`context.read_only`, and `open_pipeline_segment` must not block or call zarr's +synchronous API. See [`PipelineContext`][zarr.storage.PipelineContext] for the full +contract. ## Custom array buffers diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index 21ced15b41..ae2d17e3cb 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -102,7 +102,7 @@ print(group) - a [`URLPipeline`][zarr.storage.URLPipeline], the parsed form of a [URL pipeline](#user-guide-url-pipelines) such as `s3://bucket/data.zip|zip:`, which - is resolved through registered adapters. Strings are never interpreted as pipelines. + is resolved through registered adapters. - a [`Store`][zarr.abc.store.Store] or [`StorePath`][zarr.storage.StorePath] - see explicit store creation below. @@ -128,11 +128,11 @@ group = zarr.open_group(pipeline, mode="w") print(zarr.open_group(pipeline, mode="r").metadata.zarr_format) ``` -A trailing `zarr2:` or `zarr3:` *format segment* selects the zarr format of the call, -and its body (`zarr3:path/to/node`) is the path of the node; `zarr:` gives a node path -and leaves the format to be detected. A conflicting `zarr_format` argument raises -`ValueError`. That includes the default of 3 of `zarr.create_array`, `Group.open`, -`Group.from_store` and `Array.open`: pass the matching format, or use +A trailing `zarr2:` or `zarr3:` *format segment* selects the zarr format, +and its body (`zarr3:path/to/node`) is the node path; `zarr:` gives a node path and +detects the format. A conflicting `zarr_format` argument raises `ValueError`, including +the default `zarr_format=3` of `zarr.create_array`, `Group.open`, `Group.from_store` and +`Array.open`; pass the matching format, or use `zarr.open_group` / `zarr.open_array`, which follow the pipeline. The root sub-URL must carry a scheme, so a local path is spelled as an absolute `file:` diff --git a/src/zarr/registry.py b/src/zarr/registry.py index c1d94e9095..a9773d561f 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -314,7 +314,7 @@ def _collect_entrypoints() -> list[Registry[Any]]: else: warnings.warn( f"ignoring the 'zarr.url_adapters' entry point {entry_point.name!r} " - f"({entry_point.value!r}): it is not a URL scheme an adapter can handle", + f"({entry_point.value!r}): not a valid adapter scheme", category=ZarrUserWarning, stacklevel=2, ) @@ -521,8 +521,8 @@ def get_chunk_key_encoding_class(key: str) -> type[ChunkKeyEncoding]: return _chunk_key_encoding_registry[key] -# A URL scheme (RFC 3986, ASCII only), plus "." for vendor-prefixed schemes -# such as "earthmover.myscheme". Use with fullmatch. +# An RFC 3986 URL scheme (ASCII only), e.g. "s3" or "earthmover.myscheme". +# Use with fullmatch. _URL_SCHEME_RE = re.compile(r"[a-zA-Z][a-zA-Z0-9+.\-]*") # Schemes zarr-python handles itself: the native `file:` and `memory:` roots @@ -590,10 +590,10 @@ def _get_url_adapter(scheme: str) -> type[URLPipelineAdapter]: Loads pending `zarr.url_adapters` entry points for this scheme only, so resolving one scheme never imports other providers' packages. When the - scheme is already registered (e.g. with `register_url_adapter`), a same-named entry - point is discarded with a [`ZarrUserWarning`][zarr.errors.ZarrUserWarning]; - when several entry points share the name, the first one wins and a - warning names the ones ignored. + scheme is already registered (e.g. with `register_url_adapter`), a + same-named entry point is discarded with a + [`ZarrUserWarning`][zarr.errors.ZarrUserWarning]; when several entry + points share the name, the first one wins and a warning names the rest. Raises ------ @@ -606,10 +606,9 @@ def _get_url_adapter(scheme: str) -> type[URLPipelineAdapter]: raise URLPipelineError( f"'{key}:' is handled by zarr-python itself and cannot be an adapter segment" ) - # Take the matching entry points out of the pending list under the lock, - # so concurrent first-time resolutions of different schemes never clobber - # each other's view of the list. The import itself runs *outside* the lock: - # it may take arbitrarily long and may itself resolve URL adapters. + # Claim this scheme's entry points under the lock so concurrent loads of + # different schemes can't clobber the list; import outside it, since an + # import may be slow or itself resolve URL adapters. with _url_adapter_lock: registered = _url_adapter_registry.get(key) pending = [e for e in _url_adapter_registry.lazy_load_list if e.name.lower() == key] diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index 88cbef1a8f..cd9b2f40db 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -326,8 +326,8 @@ async def make_store( - `None` = `MemoryStore` object. - `FSMap` = `FsspecStore` object. - `UPath` = `FsspecStore` object, or `LocalStore` for a local `UPath`. - - `URLPipeline` = the store the [URL pipeline][user-guide-url-pipelines] resolves to, - which must be the store root (`make_store_path` accepts any pipeline). + - `URLPipeline` = the store a [URL pipeline][user-guide-url-pipelines] resolves to; + the pipeline must address the store root. Parameters ---------- @@ -475,7 +475,7 @@ async def make_store_path( TypeError If the StoreLike object is not one of the supported types, or if storage_options is provided but not used. ValueError - If path is provided for a store that does not support it. + If mode is invalid, or path is provided for a store that does not support it. URLPipelineError If `store_like` is a [URL pipeline][user-guide-url-pipelines] that cannot be resolved. diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py index 63cb75fa3f..4d89d2ba2a 100644 --- a/src/zarr/storage/_url_pipeline.py +++ b/src/zarr/storage/_url_pipeline.py @@ -2,9 +2,7 @@ Parsing and resolution of URL pipelines (https://github.com/jbms/url-pipeline). Only [`URLPipeline.from_url`][zarr.storage.URLPipeline.from_url] reads a string -as a pipeline; plain string store specifications keep their meaning. Adapters -registered through the `zarr.url_adapters` entry-point group are imported only -when a pipeline naming their scheme is resolved. No percent-escape is decoded. +as a pipeline. Adapters are imported lazily, and no percent-escape is decoded. """ from __future__ import annotations @@ -100,8 +98,8 @@ class PipelineContext: Context handed to a [`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] describing the pipeline to the left of its segment. - Instances are immutable and compare by identity. Derive a context with - other values with `dataclasses.replace`. + Instances are immutable and compare by identity; use `dataclasses.replace` + to derive one with other values. Attributes ---------- @@ -188,14 +186,13 @@ def _parse_pipeline(url: str) -> tuple[PipelineSegment, ...]: for index, part in enumerate(parts): if part != part.strip(): raise URLPipelineError(f"sub-URL {_redact(part)!r} has leading or trailing whitespace") - body_and_scheme, query = _split_query(part) - scheme, sep, body = body_and_scheme.partition(":") + scheme_and_body, query = _split_query(part) + scheme, sep, body = scheme_and_body.partition(":") if index == 0: if not sep or not scheme or "/" in scheme or "\\" in scheme: raise URLPipelineError( - f"the root sub-URL {_redact(part)!r} has no URL scheme. Spell a local path " - "as an absolute 'file:' URL; a local path containing '|' cannot be " - "addressed by a URL pipeline, so open it with a plain path instead." + f"the root sub-URL {_redact(part)!r} has no URL scheme; spell a local path " + "as an absolute 'file:' URL, or pass a path containing '|' as a plain string" ) _check_scheme(scheme, part) if len(scheme) == 1: @@ -261,7 +258,7 @@ def __post_init__(self) -> None: normalize_path(format_segment.body) except ValueError: raise URLPipelineError( - f"the node path of a format segment must not contain '.' or '..': {self!r}" + f"the node path of a format segment must not contain '.' or '..' components: {self!r}" ) from None @classmethod @@ -278,8 +275,8 @@ def from_url(cls, url: str) -> URLPipeline: If the string cannot be parsed: an empty sub-URL, a root without a URL scheme (including a local or Windows drive path), an invalid scheme, a fragment, an ASCII control character, or whitespace at - either end of a sub-URL; or if the pipeline is invalid (see the - class docstring). + either end of a sub-URL, or the pipeline is invalid (see the class + docstring). """ return cls(_parse_pipeline(url)) @@ -344,8 +341,7 @@ async def _resolve( resolved.store.close() raise URLPipelineError( f"adapter {last.scheme!r} returned a writable store for mode 'r', " - "and the store does not support read-only conversion via " - ".with_read_only()" + "and the store does not support read-only conversion via 'with_read_only()'" ) from exc except Exception: resolved.store.close() @@ -392,9 +388,8 @@ async def _resolve_root( return ManagedMemoryStore(name="", path=path, read_only=mode == "r") if segment.scheme == "file": - # file://localhost/p ≡ file:///p ≡ file:/p; other authorities and - # relative paths are rejected. On Windows '\' is a separator too, so - # check the '/'-spelled path (this also rejects UNC paths like /\host). + # file://localhost/p ≡ file:///p ≡ file:/p. On Windows, check the + # '/'-spelled path, which also rejects UNC paths like /\host. body = segment.body.replace("\\", "/") if sys.platform == "win32" else segment.body if body.startswith("//"): authority, sep, rest = body[2:].partition("/") diff --git a/tests/test_url_pipeline/test_open_path.py b/tests/test_url_pipeline/test_open_path.py index 112b92213f..9c40f3e9c2 100644 --- a/tests/test_url_pipeline/test_open_path.py +++ b/tests/test_url_pipeline/test_open_path.py @@ -77,7 +77,7 @@ async def test_invalid_mode_rejected_for_plain_stores(self) -> None: class TestStoresClosedOnError: - async def test_read_only_store_in_mode_a(self, recording: type[RecordingAdapter]) -> None: + def test_read_only_store_in_mode_a(self, recording: type[RecordingAdapter]) -> None: recording.read_only_store = True with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"): zarr.open_group(URLPipeline.from_url("memory:|rec:"), mode="a") @@ -103,13 +103,13 @@ async def test_make_store_residual_path(self, recording: type[RecordingAdapter]) await make_store(URLPipeline.from_url("memory:|rec:sub")) assert len(ClosingStore.closes) == 1 - async def test_success_does_not_close(self, recording: type[RecordingAdapter]) -> None: + def test_success_does_not_close(self, recording: type[RecordingAdapter]) -> None: zarr.open_group(URLPipeline.from_url("memory:|rec:"), mode="w") assert ClosingStore.closes == [] class TestConsolidateMetadata: - async def test_honors_format_segment(self, tmp_path: Path) -> None: + def test_honors_format_segment(self, tmp_path: Path) -> None: root = tmp_path.as_posix() zarr.create_group(tmp_path, zarr_format=3).create_array("x", shape=(2,), dtype="i4") # a zarr2: pipeline over v3 data finds no v2 group @@ -119,7 +119,7 @@ async def test_honors_format_segment(self, tmp_path: Path) -> None: group = zarr.consolidate_metadata(URLPipeline.from_url(f"file:{root}|zarr3:")) assert group.metadata.consolidated_metadata is not None - async def test_conflicting_format_raises(self, tmp_path: Path) -> None: + def test_conflicting_format_raises(self, tmp_path: Path) -> None: zarr.create_group(tmp_path, zarr_format=2) with pytest.raises(ValueError, match="conflicts with"): zarr.consolidate_metadata( @@ -138,13 +138,13 @@ def test_node_path_dot_segments_rejected_at_parse(self, body: str) -> None: with pytest.raises(URLPipelineError, match="node path of a format segment"): URLPipeline.from_url(f"memory:|zarr3:{body}") - async def test_node_path_traversal_creates_nothing(self, tmp_path: Path) -> None: + def test_node_path_traversal_creates_nothing(self, tmp_path: Path) -> None: target = tmp_path / "newdir" with pytest.raises(URLPipelineError): zarr.open_group(URLPipeline.from_url(f"file:{target.as_posix()}|zarr3:../x"), mode="w") assert not target.exists() - async def test_zarr_segment_detects_format(self, tmp_path: Path) -> None: + def test_zarr_segment_detects_format(self, tmp_path: Path) -> None: root = tmp_path.as_posix() zarr.create_group(URLPipeline.from_url(f"file:{root}|zarr2:g"), zarr_format=2) group = zarr.open_group(URLPipeline.from_url(f"file:{root}|zarr:g"), mode="r") @@ -167,7 +167,7 @@ class TestFileRoots: @pytest.mark.parametrize( "url", ["file:////data", "file:////nonexistent/data", "file://localhost//data"] ) - async def test_double_slash_path_rejected( + def test_double_slash_path_rejected( self, url: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.chdir(tmp_path) @@ -176,16 +176,16 @@ async def test_double_slash_path_rejected( assert list(tmp_path.iterdir()) == [] @pytest.mark.parametrize("suffix", ["/../x", "/./x", "/a/../../x"]) - async def test_dot_segments_rejected(self, suffix: str, tmp_path: Path) -> None: + def test_dot_segments_rejected(self, suffix: str, tmp_path: Path) -> None: with pytest.raises(URLPipelineError, match=r"'\.' or '\.\.'"): zarr.open_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}{suffix}"), mode="w") - async def test_root_opens_absolute_local_path(self, tmp_path: Path) -> None: + def test_root_opens_absolute_local_path(self, tmp_path: Path) -> None: group = zarr.open_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}/g"), mode="w") group.attrs["x"] = 1 assert (tmp_path / "g" / "zarr.json").exists() - async def test_storage_options_rejected(self, tmp_path: Path) -> None: + def test_storage_options_rejected(self, tmp_path: Path) -> None: with pytest.raises(TypeError, match="'storage_options' was provided but unused"): zarr.open_group( URLPipeline.from_url(f"file:{tmp_path.as_posix()}"), @@ -193,10 +193,14 @@ async def test_storage_options_rejected(self, tmp_path: Path) -> None: storage_options={"anon": True}, ) + def test_authority_is_case_insensitive(self, tmp_path: Path) -> None: + zarr.open_group(URLPipeline.from_url(f"file://LOCALHOST{tmp_path.as_posix()}"), mode="w") + assert (tmp_path / "zarr.json").exists() + class TestMemoryRoot: async def test_single_store_body_is_a_path(self) -> None: - """`memory:n/a/b`, `memory:n|...a/b` and `memory:|...n/a/b` address the same node.""" + """The body of a `memory:` root is a path, so it can be split across adapter segments.""" class PathAdapter(URLPipelineAdapter): @classmethod @@ -225,7 +229,7 @@ async def test_storage_options_rejected(self, storage_options: dict[str, object] class TestRootSchemeCase: - async def test_uppercase_native_root(self, tmp_path: Path) -> None: + def test_uppercase_native_root(self, tmp_path: Path) -> None: group = zarr.open_group(URLPipeline.from_url(f"FILE:{tmp_path.as_posix()}"), mode="w") assert isinstance(group, zarr.Group) @@ -255,6 +259,11 @@ def test_vendor_scheme_accepted(self) -> None: register_url_adapter("Vendor.My-Scheme+x", RecordingAdapter) assert zarr.registry._get_url_adapter("vendor.my-scheme+x") is RecordingAdapter + @pytest.mark.parametrize("scheme", ["foo\n", "\u212aelvin"]) + def test_register_rejects_scheme_tricks(self, scheme: str) -> None: + with pytest.raises(ValueError, match="not a valid URL scheme"): + register_url_adapter(scheme, RecordingAdapter) + class TestSegmentValidation: @pytest.mark.parametrize( @@ -291,11 +300,18 @@ def test_no_percent_encoding_advice(self, url: str) -> None: assert "%20" not in str(info.value) @pytest.mark.skipif(sys.platform == "win32", reason="'|' is not allowed in Windows file names") - async def test_group_not_found_message_is_readable(self, tmp_path: Path) -> None: + def test_group_not_found_message_is_readable(self, tmp_path: Path) -> None: with pytest.raises(GroupNotFoundError) as info: zarr.open_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr3:"), mode="r") assert "PipelineSegment(" not in str(info.value) + def test_no_adapter_error_does_not_list_installed_schemes( + self, recording: type[RecordingAdapter] + ) -> None: + with pytest.raises(URLPipelineError) as info: + zarr.core.sync.sync(make_store_path(URLPipeline.from_url("memory:|nope:"))) + assert "rec" not in str(info.value) + class TestSecretsAreRedacted: URL = "https://user:pw@h/x?token=SECRET|rec:|zarr2:" @@ -353,18 +369,6 @@ def test_unusable_entry_point_names_ignored( zarr.registry._collect_entrypoints() assert zarr.registry._url_adapter_registry.lazy_load_list == [] - def test_no_adapter_error_does_not_list_installed_schemes( - self, recording: type[RecordingAdapter] - ) -> None: - with pytest.raises(URLPipelineError) as info: - zarr.core.sync.sync(make_store_path(URLPipeline.from_url("memory:|nope:"))) - assert "rec" not in str(info.value) - - @pytest.mark.parametrize("scheme", ["foo\n", "\u212aelvin"]) - def test_register_rejects_scheme_tricks(self, scheme: str) -> None: - with pytest.raises(ValueError, match="not a valid URL scheme"): - register_url_adapter(scheme, RecordingAdapter) - class TestLimits: def test_hand_built_pipeline_segment_limit(self) -> None: @@ -401,8 +405,3 @@ async def test_drive_relative_and_dot_paths_rejected( monkeypatch.setattr(sys, "platform", "win32") with pytest.raises(URLPipelineError, match=r"absolute local path|'\.' or '\.\.'|':'"): await make_store_path(URLPipeline.from_url(url), mode="r") - - -async def test_file_authority_is_case_insensitive(tmp_path: Path) -> None: - zarr.open_group(URLPipeline.from_url(f"file://LOCALHOST{tmp_path.as_posix()}"), mode="w") - assert (tmp_path / "zarr.json").exists() diff --git a/tests/test_url_pipeline/test_parser.py b/tests/test_url_pipeline/test_parser.py index f60f576f77..195a27d13b 100644 --- a/tests/test_url_pipeline/test_parser.py +++ b/tests/test_url_pipeline/test_parser.py @@ -18,7 +18,7 @@ def test_single_root_url() -> None: @pytest.mark.parametrize("url", ["/local/path", "/local/path|zip:", "relative/path|zip:"]) def test_schemeless_root_rejected(url: str) -> None: - """The root sub-URL must carry a URL scheme; a bare path (or anything urlparse would not read as a scheme) is an error.""" + """The root sub-URL must carry a URL scheme; a bare path is an error.""" with pytest.raises(URLPipelineError, match="URL scheme"): _parse_pipeline(url) @@ -141,7 +141,7 @@ def test_invalid_adapter_scheme_rejected(segment: str) -> None: def test_scheme_detection_ignores_authority_syntax() -> None: - """Scheme detection works on the raw text, so an authority urlparse would reject is no obstacle.""" + """Scheme detection works on the raw text, so a malformed authority does not affect it.""" segments = _parse_pipeline("xy://[authority]/path|zip:") assert segments[0].scheme == "xy" assert segments[0].raw == "xy://[authority]/path" @@ -155,15 +155,9 @@ def test_round_trip() -> None: # --- property-based tests ----------------------------------------------- # -# These exercise the parser's *splitting invariants*, not URL pipeline spec -# validity: _parse_pipeline is a permissive segment splitter (bodies and -# queries are opaque text to it), and semantic validation is left to the -# resolver and the adapters. Rather than enumerating registered schemes, -# the strategies sample the full scheme grammar the parser accepts -# (RFC 3986 plus "." for vendor prefixes), so every valid root/adapter -# scheme is reachable. Bodies and queries draw from printable ASCII minus -# the characters the parser itself splits on ("|", "#", and "?" for -# bodies); the generated text is not required to be a well-formed URI. +# These check the parser's splitting invariants, not spec validity. Schemes are +# drawn from the full scheme grammar; bodies and queries from printable ASCII +# minus the characters the parser splits on ("|", "#", and "?" for bodies). _ADAPTER_SCHEME = st.from_regex(r"[a-zA-Z][a-zA-Z0-9+.\-]{0,15}", fullmatch=True) # two or more characters, so Windows drive-letter handling cannot reclassify diff --git a/tests/test_url_pipeline/test_pipeline_object.py b/tests/test_url_pipeline/test_pipeline_object.py index 70fd051000..8db1d65744 100644 --- a/tests/test_url_pipeline/test_pipeline_object.py +++ b/tests/test_url_pipeline/test_pipeline_object.py @@ -49,7 +49,7 @@ def test_empty_pipeline_rejected() -> None: ["s3://b|zip:", ["s3://b"], ("s3://b",), [PipelineSegment("s3", "//b", None, "s3://b")]], ) def test_direct_construction_rejects_non_segments(segments: object) -> None: - """A string or a list is not a pipeline; `from_url` is the way to parse one.""" + """Only a tuple of `PipelineSegment` is accepted; `from_url` parses strings.""" with pytest.raises(TypeError, match="tuple of PipelineSegment"): URLPipeline(segments) # type: ignore[arg-type] diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index 563394eb5f..c5c2666737 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -35,23 +35,23 @@ from zarr.storage._common import _has_fsspec, make_store, make_store_path from zarr.storage._utils import _join_paths -# fsspec < 2024.12.0 has no AsyncFileSystemWrapper, so a plain memory:// URL -# (a sync filesystem) cannot be opened at all there -_has_async_fsspec_wrapper = _has_fsspec and parse_version(version("fsspec")) >= parse_version( - "2024.12.0" -) - if TYPE_CHECKING: from collections.abc import Callable from pathlib import Path from zarr.core.common import AccessModeLiteral +# fsspec < 2024.12.0 has no AsyncFileSystemWrapper, so a plain memory:// URL +# (a sync filesystem) cannot be opened at all there +_has_async_fsspec_wrapper = _has_fsspec and parse_version(version("fsspec")) >= parse_version( + "2024.12.0" +) + pytestmark = pytest.mark.usefixtures("clean_url_adapter_registry") class TracingStore(WrapperStore[Store]): - """Wrapper that records the context it was created from.""" + """Wrapper that records the context and segment it was created from.""" context: PipelineContext segment: PipelineSegment @@ -91,7 +91,7 @@ async def open_pipeline_segment( class RootAdapter(URLPipelineAdapter): - """A root-scheme adapter (no preceding segments), like al://.""" + """A read-only root-scheme adapter (no preceding segments).""" @classmethod async def open_pipeline_segment( @@ -546,8 +546,6 @@ async def open_pipeline_segment( assert ConsumingWrapper.seen == "s3cr3t" async def test_storage_options_visible_to_adapter(self) -> None: - register_url_adapter("native", NativeAdapter) - class OptionsProbe(NativeAdapter): seen_options: dict[str, object] | None = None @@ -587,8 +585,8 @@ async def test_unknown_adapter_scheme_raises(self) -> None: with pytest.raises(URLPipelineError, match="no URL pipeline adapter is registered"): await URLPipeline.from_url("memory://base|no-such-adapter:")._resolve_store_path() - async def test_base_case_value_error_is_wrapped(self) -> None: - # an authority urlparse rejects must not leak its ValueError + async def test_root_value_error_is_wrapped(self) -> None: + # a ValueError from the root store is re-raised as URLPipelineError register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="could not resolve the pipeline root"): await URLPipeline.from_url("vendor.x://[authority]/path|wrap:")._resolve_store_path() @@ -674,7 +672,7 @@ async def test_file_root_rejects_query(self, tmp_path: Path) -> None: reason="plain memory:// routes to fsspec only with fsspec>=2024.12.0", ) async def test_memory_root_is_distinct_from_fsspec_memory_url(self) -> None: - # documented divergence: a plain memory:// URL is fsspec's in-memory + # known divergence: a plain memory:// URL is fsspec's in-memory # filesystem when fsspec is installed, while a pipeline's memory: root # is zarr's ManagedMemoryStore; the two do not share data register_url_adapter("wrap", WrapperAdapter) @@ -715,7 +713,7 @@ async def test_make_store_rejects_format_segment_node_path(self, tmp_path: Path) with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): await make_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:|zarr3:node")) - async def test_make_store_rejects_slash_root_path(self, tmp_path: Path) -> None: + async def test_make_store_accepts_slash_root_path(self, tmp_path: Path) -> None: # an adapter returning path="/" addresses the store root; the raw # value is normalized before the residual-path rejection class SlashRootAdapter(WrapperAdapter): @@ -838,7 +836,7 @@ async def test_strings_are_not_pipelines( open_func(f"{tmp_path}|wrap:|zarr2:", mode="r") -class TestZarrFormatMergeInCore: +class TestZarrFormatFromPipeline: """ Every function taking a StoreLike honors the format selected by a `URLPipeline`, not only the top-level zarr.api functions. Functions whose diff --git a/tests/test_url_pipeline/test_spec_examples.py b/tests/test_url_pipeline/test_spec_examples.py index bf62548c34..758e20d467 100644 --- a/tests/test_url_pipeline/test_spec_examples.py +++ b/tests/test_url_pipeline/test_spec_examples.py @@ -8,13 +8,10 @@ validates each example (and an uppercased-scheme variant) against its ABNF grammar, so this corpus is grammar-valid by construction. -The parser must accept every example — including schemes zarr-python has no -adapter for, since parsing is independent of adapter availability — split it -into the expected sub-URL schemes, and preserve the text losslessly. - -To regenerate after a spec update: extract examples per the rule above and -recompute the expected scheme tuples with `_parse_pipeline`, reviewing any -changes against the spec diff. +The parser must accept every example, whether or not an adapter exists for its +schemes, split it into the expected schemes, and preserve the text losslessly. To +regenerate, re-extract the examples and recompute the expected schemes with +`_parse_pipeline`, reviewing changes against the spec diff. """ from __future__ import annotations @@ -240,10 +237,10 @@ def _uppercase_schemes(example: str) -> str: def repl(match: re.Match[str]) -> str: return match.group(1) + match.group(2).upper() - return re.sub(r"((?:^|\|)\s*)([a-zA-Z][a-zA-Z0-9+.-]*)", repl, example) + return re.sub(r"(^|\|)([a-zA-Z][a-zA-Z0-9+.-]*)", repl, example) -@pytest.mark.parametrize(("url", "expected_schemes"), SPEC_EXAMPLES, ids=lambda v: str(v)) +@pytest.mark.parametrize(("url", "expected_schemes"), SPEC_EXAMPLES, ids=str) def test_spec_example_parses(url: str, expected_schemes: tuple[str, ...]) -> None: segments = _parse_pipeline(url) assert tuple(segment.scheme for segment in segments) == expected_schemes @@ -251,7 +248,7 @@ def test_spec_example_parses(url: str, expected_schemes: tuple[str, ...]) -> Non assert "|".join(segment.raw for segment in segments) == url -@pytest.mark.parametrize(("url", "expected_schemes"), SPEC_EXAMPLES, ids=lambda v: str(v)) +@pytest.mark.parametrize(("url", "expected_schemes"), SPEC_EXAMPLES, ids=str) def test_spec_example_schemes_case_insensitive(url: str, expected_schemes: tuple[str, ...]) -> None: upper = _uppercase_schemes(url) segments = _parse_pipeline(upper) From bb6e11b56710add68084b98b46450bbaeb686c1c Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 23 Oct 2026 01:14:33 +0200 Subject: [PATCH 19/20] test(url-pipeline): fix two tests on Windows and minimum fsspec - Spell the LOCALHOST authority test's path URL-style, so a Windows drive path gets its leading slash. - Check the fsspec store's URL rather than `fs.protocol`, which older fsspec reports as a plain string. Co-Authored-By: Claude Opus 5.5 --- tests/test_url_pipeline/test_open_path.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_url_pipeline/test_open_path.py b/tests/test_url_pipeline/test_open_path.py index 9c40f3e9c2..e429588f3e 100644 --- a/tests/test_url_pipeline/test_open_path.py +++ b/tests/test_url_pipeline/test_open_path.py @@ -194,7 +194,10 @@ def test_storage_options_rejected(self, tmp_path: Path) -> None: ) def test_authority_is_case_insensitive(self, tmp_path: Path) -> None: - zarr.open_group(URLPipeline.from_url(f"file://LOCALHOST{tmp_path.as_posix()}"), mode="w") + # spell the path URL-style: on Windows C:/... gains a leading slash + posix = tmp_path.as_posix() + url_path = posix if posix.startswith("/") else f"/{posix}" + zarr.open_group(URLPipeline.from_url(f"file://LOCALHOST{url_path}"), mode="w") assert (tmp_path / "zarr.json").exists() @@ -241,7 +244,7 @@ async def test_uppercase_fsspec_root(self) -> None: store_path = await make_store_path(URLPipeline.from_url("HTTPS://example.com/d")) assert isinstance(store_path.store, FsspecStore) - assert "https" in store_path.store.fs.protocol + assert store_path.store.path == "https://example.com/d" class TestRegisterURLAdapter: From 334616e68ca44160b56a48e398afb78fb0389e8c Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 23 Oct 2026 16:20:41 +0200 Subject: [PATCH 20/20] docs(url-pipeline): note current string behavior and explicit store creation Co-Authored-By: Claude Opus 5.5 --- docs/user-guide/storage.md | 10 ++++++---- src/zarr/registry.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index ae2d17e3cb..13149da198 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -116,8 +116,8 @@ later sub-URL names an *adapter* that reinterprets everything to its left (e.g. [custom URL pipeline adapters](extending.md#custom-url-pipeline-adapters). Parse a pipeline with [`URLPipeline.from_url`][zarr.storage.URLPipeline.from_url] and -pass it to any function that accepts a `StoreLike`. This is the only way a string is -read as a pipeline: a plain string store specification keeps its existing meaning, so a +pass it to any function that accepts a `StoreLike`. At present, this is the only way a string is +read as a pipeline: a plain string store specification currently maintains its existing meaning, so a local path containing `|` still names a directory. ```python exec="true" session="storage" source="above" result="ansi" @@ -132,8 +132,7 @@ A trailing `zarr2:` or `zarr3:` *format segment* selects the zarr format, and its body (`zarr3:path/to/node`) is the node path; `zarr:` gives a node path and detects the format. A conflicting `zarr_format` argument raises `ValueError`, including the default `zarr_format=3` of `zarr.create_array`, `Group.open`, `Group.from_store` and -`Array.open`; pass the matching format, or use -`zarr.open_group` / `zarr.open_array`, which follow the pipeline. +`Array.open`. The root sub-URL must carry a scheme, so a local path is spelled as an absolute `file:` URL (`file:/data/x.zarr`, with at most a `localhost` authority). Every `memory:` root @@ -148,6 +147,9 @@ see them. When a pipeline may come from an untrusted source, keep credentials ou `storage_options` unless you trust every installed adapter, and don't accept `memory:` roots in a process that serves several users: the in-memory store is process-wide. +For complex store configurations, explicit store creation, as described below, may be +required. + ## Explicit Store Creation In some cases, it may be helpful to create a store instance directly. Zarr-Python offers diff --git a/src/zarr/registry.py b/src/zarr/registry.py index a9773d561f..9ebfed3af9 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -521,7 +521,7 @@ def get_chunk_key_encoding_class(key: str) -> type[ChunkKeyEncoding]: return _chunk_key_encoding_registry[key] -# An RFC 3986 URL scheme (ASCII only), e.g. "s3" or "earthmover.myscheme". +# An RFC 3986 URL scheme (ASCII only), e.g. "s3" or "foo.myscheme". # Use with fullmatch. _URL_SCHEME_RE = re.compile(r"[a-zA-Z][a-zA-Z0-9+.\-]*")