diff --git a/Cargo.lock b/Cargo.lock index 81c0bb17416..45d447e3cd4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3682,6 +3682,40 @@ dependencies = [ "yansi", ] +[[package]] +name = "goosefs-sdk" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1ea4eee6dcbc31b25ab4fd577adc55b677d2bed3aa3016c44c58fbe1b2298a5" +dependencies = [ + "arc-swap", + "async-trait", + "bytes", + "dashmap", + "fastrand", + "futures", + "hostname", + "io-uring", + "itoa", + "libc", + "lru 0.18.1", + "memmap2", + "moka", + "prost 0.14.4", + "prost-types", + "rand 0.9.5", + "reqwest 0.12.28", + "serde", + "thiserror 2.0.19", + "tokio", + "tokio-stream", + "tonic", + "tonic-prost", + "tracing", + "uuid", + "xxhash-rust", +] + [[package]] name = "gpu-scan-cli" version = "0.1.0" @@ -3833,6 +3867,17 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "hostname" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" +dependencies = [ + "cfg-if", + "libc", + "windows-link", +] + [[package]] name = "http" version = "1.5.0" @@ -3872,6 +3917,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humansize" version = "2.1.3" @@ -3910,6 +3961,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -3931,6 +3983,20 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", ] [[package]] @@ -6073,6 +6139,7 @@ checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d" dependencies = [ "opendal-core", "opendal-service-cos", + "opendal-service-goosefs", "opendal-service-oss", ] @@ -6119,6 +6186,20 @@ dependencies = [ "serde", ] +[[package]] +name = "opendal-service-goosefs" +version = "0.58.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60871e6386f04d831e6a5bdbc032af4a91aeba49963252d0ef456a2cf36a9b78" +dependencies = [ + "bytes", + "goosefs-sdk", + "log", + "opendal-core", + "serde", + "tokio", +] + [[package]] name = "opendal-service-oss" version = "0.58.1" @@ -6591,9 +6672,9 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "044b1fa4f259f4df9ad5078e587b208f5d288a25407575fcddb9face30c7c692" dependencies = [ - "rand 0.9.5", + "rand 0.8.7", "socket2", - "thiserror 2.0.19", + "thiserror 1.0.69", ] [[package]] @@ -6774,7 +6855,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.11.0", "log", "multimap", "petgraph", @@ -6806,7 +6887,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2", "quote", "syn 2.0.119", @@ -7542,6 +7623,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams 0.4.2", "web-sys", + "webpki-roots", ] [[package]] @@ -9045,6 +9127,45 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" +[[package]] +name = "tonic" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-prost" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +dependencies = [ + "bytes", + "prost 0.14.4", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -9053,11 +9174,15 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", + "indexmap 2.14.0", "pin-project-lite", + "slab", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -10764,6 +10889,15 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "which" version = "8.0.5" diff --git a/docs/api/python/store/opendal.rst b/docs/api/python/store/opendal.rst index 699e5c68d62..a1c2ad500de 100644 --- a/docs/api/python/store/opendal.rst +++ b/docs/api/python/store/opendal.rst @@ -1,9 +1,10 @@ -================== -OpenDAL (COS, OSS) -================== +=========================== +OpenDAL (COS, OSS, GooseFS) +=========================== -Vortex can read from and write to Tencent Cloud COS and Alibaba Cloud OSS through -`OpenDAL `_, which provides native service support. +Vortex can read from and write to Tencent Cloud COS, Alibaba Cloud OSS, and Tencent Cloud +GooseFS through `OpenDAL `_, which provides native service +support. These stores are available only when Vortex is built with the ``opendal`` feature (e.g. ``maturin develop --features opendal`` or ``cargo build -p vortex-jni --features opendal``). @@ -13,7 +14,7 @@ These stores are available only when Vortex is built with the ``opendal`` featur * - Scheme - Service - - Endpoint variable + - Endpoint / master variable - Credential variables * - ``cos://`` - Tencent Cloud COS @@ -23,6 +24,10 @@ These stores are available only when Vortex is built with the ``opendal`` featur - Alibaba Cloud OSS - ``OSS_ENDPOINT`` - ``ALIBABA_CLOUD_ACCESS_KEY_ID``, ``ALIBABA_CLOUD_ACCESS_KEY_SECRET`` + * - ``goosefs://`` + - Tencent Cloud GooseFS + - ``GOOSEFS_MASTER_ADDR`` + - (optional) ``auth_type`` / ``auth_username`` properties :class:`vortex.store.CosStore` ============================== @@ -47,6 +52,31 @@ These stores are available only when Vortex is built with the ``opendal`` featur :param disable_config_load: When ``True``, disable OpenDAL's automatic config loading and rely only on the explicit configuration. Defaults to ``False``. +:class:`vortex.store.GoosefsStore` +================================== + +.. py:class:: vortex.store.GoosefsStore(master_addr, *, root=None, block_size=None, chunk_size=None, write_type=None, auth_type=None, auth_username=None) + + A Tencent Cloud GooseFS object store, backed by OpenDAL. Construct it with explicit + configuration and pass it to + :func:`vortex.io.read_url` / :func:`vortex.io.write` via the ``store=`` argument, + exactly like the built-in S3/Azure/GCS stores. + + The class is only available when Vortex is built with the ``opendal`` feature; on + a default build, instantiating it raises :class:`ImportError`. + + :param master_addr: GooseFS master address(es). Single master: + ``"10.0.0.1:9200"``. HA (comma-separated): + ``"10.0.0.1:9200,10.0.0.2:9200,10.0.0.3:9200"``. + :param root: Optional key prefix applied to every operation. + :param block_size: Block size in bytes for new files (default: 64 MiB). + :param chunk_size: Chunk size in bytes for streaming RPCs (default: 1 MiB). + :param write_type: Default write type: ``"must_cache"``, ``"cache_through"``, + ``"through"``, or ``"async_through"``. + :param auth_type: Authentication type: ``"nosasl"`` or ``"simple"`` (default: + ``"simple"``). + :param auth_username: Authentication username (default: current OS user). + Reading from COS ================ @@ -91,3 +121,29 @@ configuration comes from the environment variables OpenDAL's OSS builder reads import vortex as vx a = vx.io.read_url("oss://my-bucket/path/to/dataset.vortex") + +Reading from GooseFS +==================== + +Pass a ``goosefs://`` URL directly. The master address is taken from the URL authority, or +from the ``GOOSEFS_MASTER_ADDR`` environment variable when the authority is empty: + +.. code-block:: python + + import vortex as vx + + a = vx.io.read_url("goosefs://10.0.0.1:9200/path/to/dataset.vortex") + +Or configure explicitly with :class:`~vortex.store.GoosefsStore` and pass it to +:func:`vortex.io.read_url` via ``store=``: + +.. code-block:: python + + from vortex.io import read_url + from vortex.store import GoosefsStore + + store = GoosefsStore(master_addr="10.0.0.1:9200") + + # When `store=` is supplied, the path is a key within the store, so the scheme and + # master address are not part of the path passed to read_url. + a = read_url("path/to/dataset.vortex", store=store) diff --git a/docs/conf.py b/docs/conf.py index b73ed99c763..9220067b735 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -52,10 +52,11 @@ nitpicky = True # ensures all :class:, :obj:, etc. links are valid nitpick_ignore = [ - # `vortex.store.CosStore` is re-exported through the private `vortex.store._cos` module, - # and the `ObjectStore` type alias resolves to the private path. The public class is - # fully documented in `opendal.rst`; the private path is intentionally not. + # `vortex.store.CosStore` / `GoosefsStore` are re-exported through private modules, + # and the `ObjectStore` type alias resolves to those private paths. The public + # classes are fully documented in `opendal.rst`; the private paths are intentionally not. ("py:class", "vortex.store._cos.CosStore"), + ("py:class", "vortex.store._goosefs.GoosefsStore"), ] doctest_global_setup = "import pyarrow; import vortex; import vortex as vx; import random; random.seed(a=0)" diff --git a/vortex-cloud/Cargo.toml b/vortex-cloud/Cargo.toml index a10b9f1868f..d6a35ae2c24 100644 --- a/vortex-cloud/Cargo.toml +++ b/vortex-cloud/Cargo.toml @@ -52,8 +52,15 @@ oss = [ "object_store/cloud", "opendal/services-oss", ] +# Tencent Cloud GooseFS, the `goosefs://` scheme. +goosefs = [ + "dep:opendal", + "dep:object_store_opendal", + "dep:tracing", + "opendal/services-goosefs", +] # Every OpenDAL-backed service. -opendal = ["cos", "oss"] +opendal = ["cos", "oss", "goosefs"] [lints] workspace = true diff --git a/vortex-cloud/src/lib.rs b/vortex-cloud/src/lib.rs index 112b7e4a698..fa8e9eacc1e 100644 --- a/vortex-cloud/src/lib.rs +++ b/vortex-cloud/src/lib.rs @@ -10,8 +10,8 @@ //! out of environment variables case-insensitively, matching how the `object_store` `from_env` //! builders behave. //! * `opendal` supplies stores for cloud services the `object_store` crate does not implement -//! natively — Tencent Cloud COS and Alibaba Cloud OSS — bridged through -//! `object_store_opendal`. +//! natively — Tencent Cloud COS, Alibaba Cloud OSS, and Tencent Cloud GooseFS — bridged +//! through `object_store_opendal`. //! //! Every Vortex language binding resolves URLs through this one crate, so a scheme added here is //! reachable from Python, Java and DuckDB alike. @@ -25,12 +25,13 @@ //! HTTP) it resolves URLs to. //! * `cos` — Tencent Cloud COS, the `cos://` scheme. //! * `oss` — Alibaba Cloud OSS, the `oss://` scheme. +//! * `goosefs` — Tencent Cloud GooseFS, the `goosefs://` scheme. //! * `opendal` — every OpenDAL-backed service above. //! //! The `registry` feature picks up whichever OpenDAL services are enabled, so a consumer that //! turns on `oss` gets `oss://` resolution without touching its own scheme matching. -#[cfg(any(feature = "cos", feature = "oss"))] +#[cfg(any(feature = "cos", feature = "goosefs", feature = "oss"))] pub mod opendal; #[cfg(feature = "registry")] mod registry; diff --git a/vortex-cloud/src/opendal/goosefs.rs b/vortex-cloud/src/opendal/goosefs.rs new file mode 100644 index 00000000000..2bec6cc0d37 --- /dev/null +++ b/vortex-cloud/src/opendal/goosefs.rs @@ -0,0 +1,266 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Tencent Cloud GooseFS, served over OpenDAL's `services::GooseFs`. +//! +//! GooseFS is a distributed caching file system accessed via native gRPC. Unlike COS/OSS it has +//! no bucket or HTTP endpoint — instead the client connects to a GooseFS master at +//! `host:port`. An HA cluster is expressed as a comma-separated list of master addresses. + +use std::sync::Arc; + +use ::opendal::services; +use object_store::ObjectStore; +use object_store_opendal::OpendalStore; +use url::Url; +use vortex_utils::aliases::hash_map::HashMap; + +use crate::opendal::OpenDALStoreError; +use crate::opendal::build_operator; +use crate::opendal::property_or_env; +use crate::opendal::warn_on_unknown_properties; + +/// The URL scheme served by Tencent Cloud GooseFS. +pub const GOOSEFS_SCHEME: &str = "goosefs"; + +/// Property keys recognized for `goosefs://` URLs. Anything else is warned about and dropped. +const KNOWN_PROPERTIES: &[&str] = &[ + "master_addr", + "root", + "block_size", + "chunk_size", + "write_type", + "auth_type", + "auth_username", +]; + +/// Strongly-typed configuration for building an OpenDAL store against Tencent Cloud GooseFS. +/// +/// The fields mirror the keyword arguments of the `GoosefsStore` Python class. Building from a +/// [`GoosefsConfig`] avoids the URL-round-trip that the [`crate::opendal::make_opendal_store`] +/// entry point uses, and is the preferred way to construct a GooseFS store. +#[derive(Debug, Clone, Default)] +pub struct GoosefsConfig { + /// GooseFS master address(es). + /// + /// Single master: `"10.0.0.1:9200"` + /// HA (comma-separated): `"10.0.0.1:9200,10.0.0.2:9200,10.0.0.3:9200"` + /// + /// May be overridden by `properties["master_addr"]` when adapting a URL. Falls back to the + /// `GOOSEFS_MASTER_ADDR` environment variable. + pub master_addr: String, + /// Optional root prefix applied to all operations. + pub root: Option, + /// Block size in bytes for new files (default: 64 MiB). + pub block_size: Option, + /// Chunk size in bytes for streaming RPCs (default: 1 MiB). + pub chunk_size: Option, + /// Default write type: `"must_cache"`, `"cache_through"`, `"through"`, `"async_through"`. + pub write_type: Option, + /// Authentication type: `"nosasl"` or `"simple"` (default: `"simple"`). + pub auth_type: Option, + /// Authentication username (default: current OS user). + pub auth_username: Option, +} + +/// Build an [`object_store::ObjectStore`] for Tencent Cloud GooseFS directly from a +/// [`GoosefsConfig`]. +/// +/// This is the preferred entry point for callers that have a strongly-typed configuration object +/// (such as the `GoosefsStore` pyclass in `vortex-python`). It does not synthesize a URL and so is +/// not fragile against reordering of `master_addr` vs URL authority precedence. +pub fn make_goosefs_store( + config: GoosefsConfig, +) -> Result, OpenDALStoreError> { + if config.master_addr.is_empty() { + return Err(OpenDALStoreError::MissingConfig("master_addr")); + } + + let mut builder = services::GooseFs::default().master_addr(&config.master_addr); + + if let Some(root) = config.root.as_deref() { + builder = builder.root(root); + } + if let Some(block_size) = config.block_size { + builder = builder.block_size(block_size); + } + if let Some(chunk_size) = config.chunk_size { + builder = builder.chunk_size(chunk_size); + } + if let Some(write_type) = config.write_type.as_deref() { + builder = builder.write_type(write_type); + } + if let Some(auth_type) = config.auth_type.as_deref() { + builder = builder.auth_type(auth_type); + } + if let Some(auth_username) = config.auth_username.as_deref() { + builder = builder.auth_username(auth_username); + } + + let operator = build_operator(builder)?; + Ok(Arc::new(OpendalStore::new(operator))) +} + +/// Translate a (`goosefs://` URL, properties) pair into a strongly-typed [`GoosefsConfig`]. +/// +/// `master_addr` is taken from `properties["master_addr"]` first and falls back to the URL +/// authority (`host:port`); if neither is present it falls back to the `GOOSEFS_MASTER_ADDR` +/// environment variable. The remaining fields are taken from `properties` directly. +/// +/// `env_lookup` is the source of truth for environment-variable fallbacks. The production entry +/// point passes the real environment; tests pass a closure that returns from a fixed map, so they +/// do not race against the process environment. +pub(crate) fn url_and_properties_to_config( + url: &Url, + properties: &HashMap, + env_lookup: F, +) -> Result +where + F: Fn(&str) -> Option, +{ + warn_on_unknown_properties(properties, KNOWN_PROPERTIES); + + // master_addr: properties → URL authority → GOOSEFS_MASTER_ADDR env + let master_addr = properties + .get("master_addr") + .cloned() + .or_else(|| { + let host = url.host_str(); + let port = url.port(); + match (host, port) { + (Some(h), Some(p)) => Some(format!("{h}:{p}")), + (Some(h), None) => Some(h.to_string()), + _ => None, + } + }) + .or_else(|| { + property_or_env( + properties, + "master_addr", + "GOOSEFS_MASTER_ADDR", + &env_lookup, + ) + }) + .ok_or(OpenDALStoreError::MissingConfig("master_addr"))?; + + Ok(GoosefsConfig { + master_addr, + root: properties.get("root").cloned(), + block_size: properties.get("block_size").and_then(|s| s.parse().ok()), + chunk_size: properties.get("chunk_size").and_then(|s| s.parse().ok()), + write_type: properties.get("write_type").cloned(), + auth_type: properties.get("auth_type").cloned(), + auth_username: properties.get("auth_username").cloned(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `master_addr` is required. With a fixed env-lookup that returns `None` and a URL without + /// authority, the call must fail with `MissingConfig("master_addr")` regardless of what the + /// process environment happens to contain. + #[test] + fn goosefs_requires_master_addr() { + let url = Url::parse("goosefs:///path/to/file").unwrap(); + let props = HashMap::new(); + let result = url_and_properties_to_config(&url, &props, |_| None).unwrap_err(); + assert!(matches!( + result, + OpenDALStoreError::MissingConfig("master_addr") + )); + } + + /// The URL authority (`host:port`) should be used as `master_addr` when properties do not + /// contain it. + #[test] + fn goosefs_uses_url_authority_as_master_addr() { + let url = Url::parse("goosefs://10.0.0.1:9200/path/to/file").unwrap(); + let props = HashMap::new(); + let config = url_and_properties_to_config(&url, &props, |_| None).expect("config"); + assert_eq!(config.master_addr, "10.0.0.1:9200"); + } + + /// When `properties` does not contain `master_addr` and the URL has no authority, the + /// env-lookup should be consulted. + #[test] + fn goosefs_falls_back_to_env() { + let url = Url::parse("goosefs:///path/to/file").unwrap(); + let env = |key: &str| match key { + "GOOSEFS_MASTER_ADDR" => Some("10.0.0.1:9200".to_string()), + _ => None, + }; + let props = HashMap::new(); + let config = url_and_properties_to_config(&url, &props, env).expect("config"); + assert_eq!(config.master_addr, "10.0.0.1:9200"); + } + + /// An explicit property must win over the URL authority and the environment fallback. + #[test] + fn goosefs_property_overrides_url_and_env() { + let url = Url::parse("goosefs://url-host:9200/path").unwrap(); + let env = |key: &str| match key { + "GOOSEFS_MASTER_ADDR" => Some("from-env:9200".to_string()), + _ => None, + }; + let mut props = HashMap::new(); + props.insert("master_addr".to_string(), "from-prop:9200".to_string()); + + let config = url_and_properties_to_config(&url, &props, env).expect("config"); + assert_eq!(config.master_addr, "from-prop:9200"); + } + + /// The URL authority must win over the environment fallback when no explicit property is set. + #[test] + fn goosefs_url_authority_overrides_env() { + let url = Url::parse("goosefs://url-host:9200/path").unwrap(); + let env = |key: &str| match key { + "GOOSEFS_MASTER_ADDR" => Some("from-env:9200".to_string()), + _ => None, + }; + let props = HashMap::new(); + let config = url_and_properties_to_config(&url, &props, env).expect("config"); + assert_eq!(config.master_addr, "url-host:9200"); + } + + /// With a fixed env-lookup that returns an explicit master address, the strongly-typed + /// `make_goosefs_store` should build a store successfully. + #[test] + fn goosefs_builds_with_explicit_config() { + let config = GoosefsConfig { + master_addr: "127.0.0.1:9200".to_string(), + root: Some("/data".to_string()), + ..GoosefsConfig::default() + }; + let store = make_goosefs_store(config).expect("store should build"); + // Sanity: the returned store is a non-null `Arc`. + assert!(Arc::strong_count(&store) >= 1); + } + + /// The strongly-typed `make_goosefs_store` entry point must reject an empty `master_addr` + /// with a `MissingConfig` error before consulting any environment or builder. + #[test] + fn goosefs_config_rejects_empty_master_addr() { + assert!(matches!( + make_goosefs_store(GoosefsConfig { + master_addr: String::new(), + ..GoosefsConfig::default() + }), + Err(OpenDALStoreError::MissingConfig("master_addr")) + )); + } + + /// HA mode: a comma-separated list of master addresses should be accepted and build + /// successfully. + #[test] + fn goosefs_builds_ha_config() { + let config = GoosefsConfig { + master_addr: "10.0.0.1:9200,10.0.0.2:9200,10.0.0.3:9200".to_string(), + root: Some("/data".to_string()), + ..GoosefsConfig::default() + }; + let store = make_goosefs_store(config).expect("HA store should build"); + assert!(Arc::strong_count(&store) >= 1); + } +} diff --git a/vortex-cloud/src/opendal/mod.rs b/vortex-cloud/src/opendal/mod.rs index 78938e3fce7..857adb78070 100644 --- a/vortex-cloud/src/opendal/mod.rs +++ b/vortex-cloud/src/opendal/mod.rs @@ -2,21 +2,22 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors //! OpenDAL-backed [`object_store::ObjectStore`] implementations for cloud providers that are not -//! natively supported by the `object_store` crate: Tencent Cloud COS and Alibaba Cloud OSS. +//! natively supported by the `object_store` crate: Tencent Cloud COS, Alibaba Cloud OSS, and +//! Tencent Cloud GooseFS. //! //! OpenDAL exposes each service as an `Operator`. We adapt an `Operator` into an -//! `object_store::ObjectStore` via the `object_store_opendal::OpendalStore` bridge, which is built -//! against the same `object_store 0.13.x` version the rest of Vortex uses. This lets Vortex consume -//! these services through its existing `ObjectStoreFileSystem` abstraction. +//! `object_store::ObjectStore` via the `object_store_opendal::OpendalStore` bridge. This lets +//! Vortex consume these services through its existing `ObjectStoreFileSystem` abstraction. //! //! Callers that dispatch on a URL scheme should ask [`supports_scheme`] rather than comparing -//! against [`COS_SCHEME`] / [`OSS_SCHEME`] themselves, so that enabling another service does not -//! require touching every call site. +//! against [`COS_SCHEME`] / [`OSS_SCHEME`] / [`GOOSEFS_SCHEME`] themselves, so that enabling +//! another service does not require touching every call site. //! //! # Cargo features //! //! * `cos` — Tencent Cloud COS, the `cos://` scheme. //! * `oss` — Alibaba Cloud OSS, the `oss://` scheme. +//! * `goosefs` — Tencent Cloud GooseFS, the `goosefs://` scheme. //! //! With a single service enabled the module still compiles: [`supports_scheme`] returns `false` //! for every scheme it does not serve and [`make_opendal_store`] reports @@ -32,15 +33,17 @@ #[cfg(feature = "cos")] mod cos; +#[cfg(feature = "goosefs")] +mod goosefs; #[cfg(feature = "oss")] mod oss; use std::sync::Arc; -#[cfg(any(feature = "cos", feature = "oss"))] +#[cfg(any(feature = "cos", feature = "goosefs", feature = "oss"))] use ::opendal::Operator; use object_store::ObjectStore; -#[cfg(any(feature = "cos", feature = "oss"))] +#[cfg(any(feature = "cos", feature = "goosefs", feature = "oss"))] use tracing::warn; use url::Url; use vortex_utils::aliases::hash_map::HashMap; @@ -51,6 +54,12 @@ pub use crate::opendal::cos::COS_SCHEME; pub use crate::opendal::cos::CosConfig; #[cfg(feature = "cos")] pub use crate::opendal::cos::make_cos_store; +#[cfg(feature = "goosefs")] +pub use crate::opendal::goosefs::GOOSEFS_SCHEME; +#[cfg(feature = "goosefs")] +pub use crate::opendal::goosefs::GoosefsConfig; +#[cfg(feature = "goosefs")] +pub use crate::opendal::goosefs::make_goosefs_store; #[cfg(feature = "oss")] pub use crate::opendal::oss::OSS_SCHEME; #[cfg(feature = "oss")] @@ -99,6 +108,8 @@ impl From for object_store::Error { pub const SUPPORTED_SCHEMES: &[&str] = &[ #[cfg(feature = "cos")] COS_SCHEME, + #[cfg(feature = "goosefs")] + GOOSEFS_SCHEME, #[cfg(feature = "oss")] OSS_SCHEME, ]; @@ -116,11 +127,13 @@ pub fn supports_scheme(scheme: &str) -> bool { SUPPORTED_SCHEMES.contains(&scheme) } -/// Build an [`object_store::ObjectStore`] for an OpenDAL-backed URL (`cos://`, `oss://`). +/// Build an [`object_store::ObjectStore`] for an OpenDAL-backed URL (`cos://`, `oss://`, +/// `goosefs://`). /// /// `properties` are per-request configuration overrides (matching the `HashMap` /// passed through the JNI/Python layers). Missing values fall back to the environment variables -/// the corresponding service reads (e.g. `TENCENTCLOUD_SECRET_ID`, `ALIBABA_CLOUD_ACCESS_KEY_ID`). +/// the corresponding service reads (e.g. `TENCENTCLOUD_SECRET_ID`, `ALIBABA_CLOUD_ACCESS_KEY_ID`, +/// `GOOSEFS_MASTER_ADDR`). /// /// Returns [`OpenDALStoreError::UnsupportedScheme`] if `url` uses a scheme this build does not /// serve; test it up-front with [`supports_scheme`]. @@ -150,6 +163,10 @@ where COS_SCHEME => make_cos_store(cos::url_and_properties_to_config( url, properties, env_lookup, )?), + #[cfg(feature = "goosefs")] + GOOSEFS_SCHEME => make_goosefs_store(goosefs::url_and_properties_to_config( + url, properties, env_lookup, + )?), #[cfg(feature = "oss")] OSS_SCHEME => make_oss_store(oss::url_and_properties_to_config( url, properties, env_lookup, @@ -174,7 +191,7 @@ fn env_var_lookup(key: &str) -> Option { } /// Take `key` from `properties`, falling back to `env_lookup(env_var)`. -#[cfg(any(feature = "cos", feature = "oss"))] +#[cfg(any(feature = "cos", feature = "goosefs", feature = "oss"))] pub(crate) fn property_or_env( properties: &HashMap, key: &str, @@ -188,7 +205,7 @@ where } /// Log a warning for every property key the service does not recognize. -#[cfg(any(feature = "cos", feature = "oss"))] +#[cfg(any(feature = "cos", feature = "goosefs", feature = "oss"))] pub(crate) fn warn_on_unknown_properties(properties: &HashMap, known: &[&str]) { for key in properties.keys() { if !known.contains(&key.as_str()) { @@ -198,7 +215,7 @@ pub(crate) fn warn_on_unknown_properties(properties: &HashMap, k } /// Finish an OpenDAL builder into an [`Operator`], mapping builder errors into our error type. -#[cfg(any(feature = "cos", feature = "oss"))] +#[cfg(any(feature = "cos", feature = "goosefs", feature = "oss"))] pub(crate) fn build_operator(builder: B) -> Result where B: ::opendal::Builder, @@ -226,6 +243,7 @@ mod tests { fn supports_scheme_tracks_enabled_features() { assert!(!supports_scheme("s3")); assert_eq!(supports_scheme("cos"), cfg!(feature = "cos")); + assert_eq!(supports_scheme("goosefs"), cfg!(feature = "goosefs")); assert_eq!(supports_scheme("oss"), cfg!(feature = "oss")); } diff --git a/vortex-cloud/src/registry/mod.rs b/vortex-cloud/src/registry/mod.rs index 4b44e3b5bda..084bbd66521 100644 --- a/vortex-cloud/src/registry/mod.rs +++ b/vortex-cloud/src/registry/mod.rs @@ -9,8 +9,9 @@ //! 1. configuration is resolved out of environment variables case-insensitively, matching how the //! various `Store::from_env` builders behave (see //! ); -//! 2. schemes that `object_store` does not recognize natively — the OpenDAL-backed `cos://` and -//! `oss://` — are served by the crate's `opendal` module under the matching service feature. +//! 2. schemes that `object_store` does not recognize natively — the OpenDAL-backed `cos://`, +//! `oss://`, and `goosefs://` — are served by the crate's `opendal` module under the matching +//! service feature. use std::sync::Arc; @@ -101,7 +102,7 @@ enum EnvSource { impl EnvSource { /// Case-insensitive lookup of a single configuration variable. - #[cfg(any(feature = "cos", feature = "oss"))] + #[cfg(any(feature = "cos", feature = "goosefs", feature = "oss"))] fn lookup(&self, key: &str) -> Option { match self { EnvSource::Process => std::env::var(key).ok(), @@ -218,11 +219,12 @@ impl Registry { /// the one caching rule in [`Registry::resolve`]: a scheme says where its store is rooted, and /// the registry decides how to cache it. fn build_store(&self, to_resolve: &Url) -> object_store::Result<(Arc, Path)> { - // OpenDAL-backed schemes (Tencent COS, Alibaba OSS) are not recognized by `object_store`, - // so build them from OpenDAL's own environment-variable configuration (e.g. - // `TENCENTCLOUD_SECRET_ID`). The operator is rooted at the bucket, which lives in the URL - // authority, so — exactly as for `s3://bucket/path` — the whole URL path is the object key. - #[cfg(any(feature = "cos", feature = "oss"))] + // OpenDAL-backed schemes (Tencent COS, Alibaba OSS, Tencent GooseFS) are not recognized + // by `object_store`, so build them from OpenDAL's own environment-variable configuration + // (e.g. `TENCENTCLOUD_SECRET_ID`, `GOOSEFS_MASTER_ADDR`). The operator is rooted at the + // bucket, which lives in the URL authority, so — exactly as for `s3://bucket/path` — + // the whole URL path is the object key. + #[cfg(any(feature = "cos", feature = "goosefs", feature = "oss"))] if crate::opendal::supports_scheme(to_resolve.scheme()) { let store = crate::opendal::make_opendal_store_with_env(to_resolve, &HashMap::new(), |key| { diff --git a/vortex-cloud/src/registry/tests.rs b/vortex-cloud/src/registry/tests.rs index b15815f76f9..9904c72a7a8 100644 --- a/vortex-cloud/src/registry/tests.rs +++ b/vortex-cloud/src/registry/tests.rs @@ -147,22 +147,32 @@ fn test_registered_store_wins_over_build() -> Result<(), Box Result<(), Box> { for scheme in crate::opendal::SUPPORTED_SCHEMES { let url = Url::parse(&format!("{scheme}://bucket/key.vortex"))?; - let err = registry() - .resolve(&url) - .expect_err("no endpoint is configured, so the build must fail"); - let message = err.to_string(); - assert!( - message.contains("OpenDAL"), - "{scheme} did not reach the OpenDAL builder: {message}" - ); + match registry().resolve(&url) { + // GooseFS derives `master_addr` from the URL authority, so the store builds + // successfully. `parse_url_opts` does not recognize `goosefs://`, so an `Ok` + // here proves the OpenDAL builder ran. + Ok(_) => {} + Err(err) => { + let message = err.to_string(); + assert!( + message.contains("OpenDAL"), + "{scheme} did not reach the OpenDAL builder: {message}" + ); + } + } } Ok(()) } diff --git a/vortex-python/python/vortex/_lib/__init__.pyi b/vortex-python/python/vortex/_lib/__init__.pyi index 09c9a520406..2be6b9a5ab7 100644 --- a/vortex-python/python/vortex/_lib/__init__.pyi +++ b/vortex-python/python/vortex/_lib/__init__.pyi @@ -21,3 +21,25 @@ class CosStore: root: str | None = None, disable_config_load: bool = False, ) -> None: ... + +class GoosefsStore: + """A Tencent Cloud GooseFS object store, backed by OpenDAL. + + Construct it with explicit configuration and pass it to + ``vortex.io.read_url(url, store=goosefs_store)`` / + ``vortex.io.write(arrays, path, store=goosefs_store)``. + + This class is only available when Vortex is built with the ``opendal`` feature. + """ + + def __init__( + self, + master_addr: str, + *, + root: str | None = None, + block_size: int | None = None, + chunk_size: int | None = None, + write_type: str | None = None, + auth_type: str | None = None, + auth_username: str | None = None, + ) -> None: ... diff --git a/vortex-python/python/vortex/store/__init__.py b/vortex-python/python/vortex/store/__init__.py index c9127087bf0..9a9d428d307 100644 --- a/vortex-python/python/vortex/store/__init__.py +++ b/vortex-python/python/vortex/store/__init__.py @@ -18,12 +18,15 @@ from ._client import ClientConfig from ._cos import CosStore from ._gcs import GCSConfig, GCSCredential, GCSCredentialProvider, GCSStore +from ._goosefs import GoosefsStore from ._http import HTTPStore from ._local import LocalStore from ._memory import MemoryStore from ._retry import BackoffConfig, RetryConfig -ObjectStore: TypeAlias = AzureStore | CosStore | GCSStore | HTTPStore | S3Store | LocalStore | MemoryStore +ObjectStore: TypeAlias = ( + AzureStore | CosStore | GoosefsStore | GCSStore | HTTPStore | S3Store | LocalStore | MemoryStore +) """All supported ObjectStore implementations.""" @@ -93,6 +96,9 @@ def from_url( # type: ignore[misc] # docstring in pyi file - ``cos://bucket/path`` -> OpenDAL-backed Tencent Cloud COS store (requires the ``opendal`` feature; configure via environment variables such as ``TENCENTCLOUD_SECRET_ID`` / ``TENCENTCLOUD_SECRET_KEY`` and ``COS_ENDPOINT``) + - ``goosefs://master-addr:port/path`` -> OpenDAL-backed Tencent Cloud GooseFS store + (requires the ``opendal`` feature; configure via environment variable + ``GOOSEFS_MASTER_ADDR`` or pass ``master_addr`` in properties) - ``http://mydomain/path`` -> :class:`~vortex.store.HTTPStore` - ``https://mydomain/path`` -> :class:`~vortex.store.HTTPStore` @@ -145,6 +151,8 @@ def from_url( # type: ignore[misc] # docstring in pyi file "RetryConfig", # COS (OpenDAL-backed) "CosStore", + # GooseFS (OpenDAL-backed) + "GoosefsStore", # GCS "GCSConfig", "GCSCredential", diff --git a/vortex-python/python/vortex/store/_goosefs.py b/vortex-python/python/vortex/store/_goosefs.py new file mode 100644 index 00000000000..2de7c4de9fa --- /dev/null +++ b/vortex-python/python/vortex/store/_goosefs.py @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Tencent Cloud GooseFS object store, backed by OpenDAL. + +This store is only available when Vortex is built with the ``opendal`` feature. +The class is re-exported from the native extension module; if the feature is +not enabled, instantiating :class:`GoosefsStore` raises :class:`ImportError`. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + # `vortex._lib.__init__.pyi` declares `GoosefsStore` so type checkers see a real + # signature; the runtime branch below provides the same symbol when the + # `opendal` feature is enabled, and a placeholder that raises on + # instantiation when it is not. + from vortex._lib import GoosefsStore +else: + try: + from vortex._lib import GoosefsStore as GoosefsStore + except ImportError: + + class GoosefsStore: + """Placeholder; the real implementation requires the ``opendal`` feature.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + raise ImportError( + "GoosefsStore requires Vortex to be built with the 'opendal' feature; " + + "build with `maturin build --features opendal` " + + "or `maturin develop --features opendal`." + ) + + +__all__ = ["GoosefsStore"] diff --git a/vortex-python/src/io.rs b/vortex-python/src/io.rs index b444b6be814..cc77475a588 100644 --- a/vortex-python/src/io.rs +++ b/vortex-python/src/io.rs @@ -45,6 +45,8 @@ use crate::object_store::resolve::ResolvedStore; use crate::object_store::resolve::resolve_store; #[cfg(feature = "opendal")] use crate::opendal_store::CosStore; +#[cfg(feature = "opendal")] +use crate::opendal_store::GoosefsStore; use crate::session::session; pub(crate) fn init(py: Python, parent: &Bound) -> PyResult<()> { @@ -145,13 +147,16 @@ pub fn read_url<'py>( /// A store object accepted by `read_url` / `write`. /// /// This recognizes both the built-in `pyo3-object_store` classes (S3, Azure, GCS, HTTP, -/// Local, Memory) and Vortex's own OpenDAL-backed classes (`CosStore`). +/// Local, Memory) and Vortex's own OpenDAL-backed classes (`CosStore`, `GoosefsStore`). pub(crate) enum AnyVortexStore { /// A store extracted from one of the built-in `pyo3-object_store` classes. Builtin(PyObjectStore), /// Vortex's OpenDAL-backed COS store. #[cfg(feature = "opendal")] Cos(CosStore), + /// Vortex's OpenDAL-backed GooseFS store. + #[cfg(feature = "opendal")] + Goosefs(GoosefsStore), } impl AnyVortexStore { @@ -161,6 +166,8 @@ impl AnyVortexStore { AnyVortexStore::Builtin(s) => s.into_inner(), #[cfg(feature = "opendal")] AnyVortexStore::Cos(s) => s.to_arc(), + #[cfg(feature = "opendal")] + AnyVortexStore::Goosefs(s) => s.to_arc(), } } } @@ -176,8 +183,12 @@ impl<'py> FromPyObject<'_, 'py> for AnyVortexStore { if let Ok(cos) = obj.extract::() { return Ok(AnyVortexStore::Cos(cos)); } + #[cfg(feature = "opendal")] + if let Ok(goosefs) = obj.extract::() { + return Ok(AnyVortexStore::Goosefs(goosefs)); + } Err(PyTypeError::new_err( - "Expected an object store instance (S3/Azure/GCS/HTTP/Local/Memory/COS/OSS store)", + "Expected an object store instance (S3/Azure/GCS/HTTP/Local/Memory/COS/OSS/GooseFS store)", )) } } diff --git a/vortex-python/src/opendal_store.rs b/vortex-python/src/opendal_store.rs index 6cd4b5f5393..b77404d9e46 100644 --- a/vortex-python/src/opendal_store.rs +++ b/vortex-python/src/opendal_store.rs @@ -16,6 +16,7 @@ use std::sync::Arc; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use vortex_cloud::opendal::CosConfig; +use vortex_cloud::opendal::GoosefsConfig; /// A Tencent Cloud COS object store, backed by OpenDAL. /// @@ -68,9 +69,66 @@ impl CosStore { } } +/// A Tencent Cloud GooseFS object store, backed by OpenDAL. +/// +/// Construct it with explicit configuration and pass it to +/// ``vortex.io.read_url(url, store=goosefs_store)`` / +/// ``vortex.io.write(arrays, path, store=goosefs_store)``. +#[pyclass(name = "GoosefsStore", module = "vortex._lib", frozen, from_py_object)] +#[derive(Clone, Debug)] +pub struct GoosefsStore { + store: Arc, +} + +impl GoosefsStore { + /// Clone the underlying object store as an `Arc`. + pub fn to_arc(&self) -> Arc { + Arc::clone(&self.store) + } +} + +#[pymethods] +impl GoosefsStore { + #[new] + #[pyo3(signature = ( + master_addr, + *, + root = None, + block_size = None, + chunk_size = None, + write_type = None, + auth_type = None, + auth_username = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + master_addr: String, + root: Option, + block_size: Option, + chunk_size: Option, + write_type: Option, + auth_type: Option, + auth_username: Option, + ) -> PyResult { + let config = GoosefsConfig { + master_addr, + root, + block_size, + chunk_size, + write_type, + auth_type, + auth_username, + }; + let store = vortex_cloud::opendal::make_goosefs_store(config) + .map_err(|e| PyValueError::new_err(e.to_string()))?; + Ok(Self { store }) + } +} + /// Register the OpenDAL-backed store classes on the `vortex._lib` module. pub(crate) fn init(_py: Python, parent: &Bound) -> PyResult<()> { parent.add_class::()?; + parent.add_class::()?; Ok(()) }