From 5934b60b43d551ecd885b70b9001eb801695b1e4 Mon Sep 17 00:00:00 2001 From: benjamin fuentes Date: Mon, 3 Aug 2026 17:48:23 +0200 Subject: [PATCH 1/4] feat(cosi): add RustFS COSI v1alpha1 driver Extract shared rustfs-admin client, ship a tonic COSI driver with Helm toggle, and document BucketClass parameters for Tenant-backed S3. Co-authored-by: Cursor --- .github/workflows/ci.yml | 6 +- Cargo.lock | 403 +++++- Cargo.toml | 6 +- Dockerfile | 3 +- Makefile | 6 +- crates/cosi-driver/Cargo.toml | 39 + crates/cosi-driver/build.rs | 27 + crates/cosi-driver/proto/cosi.proto | 196 +++ crates/cosi-driver/src/backend.rs | 199 +++ crates/cosi-driver/src/driver.rs | 278 ++++ crates/cosi-driver/src/lib.rs | 30 + crates/cosi-driver/src/main.rs | 123 ++ crates/cosi-driver/src/parameters.rs | 119 ++ crates/cosi-driver/src/policy.rs | 126 ++ crates/rustfs-admin/Cargo.toml | 23 + .../rustfs-admin/src}/admin_ops.rs | 96 +- crates/rustfs-admin/src/client.rs | 470 +++++++ .../rustfs-admin/src}/core_ops.rs | 13 +- crates/rustfs-admin/src/credentials.rs | 23 + crates/rustfs-admin/src/helpers.rs | 185 +++ crates/rustfs-admin/src/lib.rs | 42 + .../rustfs-admin/src}/pool_ops.rs | 4 +- .../sts => crates/rustfs-admin/src}/s3_ops.rs | 47 +- crates/rustfs-admin/src/sanitize.rs | 353 +++++ .../rustfs-admin/src}/sts_ops.rs | 7 +- crates/rustfs-admin/src/tests.rs | 1192 +++++++++++++++++ deploy/rustfs-operator/README.md | 21 + deploy/rustfs-operator/templates/NOTES.txt | 11 + deploy/rustfs-operator/templates/_helpers.tpl | 20 + .../templates/cosi-deployment.yaml | 88 ++ .../rustfs-operator/templates/cosi-rbac.yaml | 59 + .../templates/cosi-serviceaccount.yaml | 21 + deploy/rustfs-operator/values.yaml | 64 + docs/operator-user-guide.md | 52 +- e2e/Cargo.lock | 18 +- e2e/tests/sts_functional.rs | 9 +- examples/README.md | 1 + examples/cosi/awscli-pod.yaml | 20 + examples/cosi/bucketaccess.yaml | 10 + examples/cosi/bucketaccessclass.yaml | 13 + examples/cosi/bucketclaim.yaml | 9 + examples/cosi/bucketclass.yaml | 21 + src/reconcile/pool_lifecycle.rs | 14 +- src/reconcile/provisioning.rs | 17 +- src/sts/helpers.rs | 211 +-- src/sts/rustfs_client.rs | 680 ++-------- src/sts/server.rs | 8 +- src/sts/tests.rs | 1187 +--------------- src/sts/types.rs | 8 +- src/tenant_monitor.rs | 12 +- 50 files changed, 4598 insertions(+), 1992 deletions(-) create mode 100644 crates/cosi-driver/Cargo.toml create mode 100644 crates/cosi-driver/build.rs create mode 100644 crates/cosi-driver/proto/cosi.proto create mode 100644 crates/cosi-driver/src/backend.rs create mode 100644 crates/cosi-driver/src/driver.rs create mode 100644 crates/cosi-driver/src/lib.rs create mode 100644 crates/cosi-driver/src/main.rs create mode 100644 crates/cosi-driver/src/parameters.rs create mode 100644 crates/cosi-driver/src/policy.rs create mode 100644 crates/rustfs-admin/Cargo.toml rename {src/sts => crates/rustfs-admin/src}/admin_ops.rs (72%) create mode 100644 crates/rustfs-admin/src/client.rs rename {src/sts => crates/rustfs-admin/src}/core_ops.rs (93%) create mode 100644 crates/rustfs-admin/src/credentials.rs create mode 100644 crates/rustfs-admin/src/helpers.rs create mode 100644 crates/rustfs-admin/src/lib.rs rename {src/sts => crates/rustfs-admin/src}/pool_ops.rs (97%) rename {src/sts => crates/rustfs-admin/src}/s3_ops.rs (73%) create mode 100644 crates/rustfs-admin/src/sanitize.rs rename {src/sts => crates/rustfs-admin/src}/sts_ops.rs (95%) create mode 100644 crates/rustfs-admin/src/tests.rs create mode 100644 deploy/rustfs-operator/templates/cosi-deployment.yaml create mode 100644 deploy/rustfs-operator/templates/cosi-rbac.yaml create mode 100644 deploy/rustfs-operator/templates/cosi-serviceaccount.yaml create mode 100644 examples/cosi/awscli-pod.yaml create mode 100644 examples/cosi/bucketaccess.yaml create mode 100644 examples/cosi/bucketaccessclass.yaml create mode 100644 examples/cosi/bucketclaim.yaml create mode 100644 examples/cosi/bucketclass.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1680f2c..155da498 100755 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,14 +101,14 @@ jobs: - name: Run tests run: | - cargo nextest run --all --no-tests pass - cargo test --all --doc + cargo nextest run --workspace --no-tests pass + cargo test --workspace --doc - name: Check code formatting run: cargo fmt --all --check - name: Run clippy lints - run: cargo clippy --all-features -- -D warnings + run: cargo clippy --workspace --all-features -- -D warnings - name: Check Rust-native e2e harness run: make e2e-check diff --git a/Cargo.lock b/Cargo.lock index 201012db..6115e525 100755 --- a/Cargo.lock +++ b/Cargo.lock @@ -95,6 +95,12 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + [[package]] name = "arbitrary" version = "1.4.2" @@ -224,7 +230,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tower", + "tower 0.5.2", "tower-layer", "tower-service", "tracing", @@ -467,6 +473,26 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cosi-driver" +version = "0.1.0" +dependencies = [ + "hex", + "k8s-openapi", + "kube", + "prost", + "protoc-bin-vendored", + "rustfs-admin", + "sha2", + "snafu", + "tokio", + "tokio-stream", + "tonic", + "tonic-build", + "tracing", + "tracing-subscriber", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -654,6 +680,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "event-listener" version = "5.4.1" @@ -687,6 +723,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.1.8" @@ -887,13 +929,19 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap", + "indexmap 2.12.0", "slab", "tokio", "tokio-util", "tracing", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.15.5" @@ -1077,7 +1125,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.1", "tokio", "tower-service", "tracing", @@ -1215,6 +1263,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.12.0" @@ -1255,6 +1313,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -1373,7 +1440,7 @@ dependencies = [ "thiserror 2.0.17", "tokio", "tokio-util", - "tower", + "tower 0.5.2", "tower-http", "tracing", ] @@ -1421,7 +1488,7 @@ dependencies = [ "futures", "k8s-openapi", "kube", - "rand", + "rand 0.9.4", "snafu", "tokio", "tokio-util", @@ -1491,6 +1558,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + [[package]] name = "litemap" version = "0.8.1" @@ -1576,6 +1649,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1639,7 +1718,6 @@ dependencies = [ "const-str", "futures", "hex", - "hmac", "hostname", "http", "http-body-util", @@ -1649,8 +1727,8 @@ dependencies = [ "kube", "kube-leader-election", "rcgen", - "reqwest", "ring", + "rustfs-admin", "rustls", "rustls-pemfile", "rustls-webpki", @@ -1667,11 +1745,10 @@ dependencies = [ "tokio-rustls", "tokio-stream", "tokio-util", - "tower", + "tower 0.5.2", "tower-http", "tracing", "tracing-subscriber", - "url", "utoipa", "utoipa-swagger-ui", ] @@ -1773,6 +1850,16 @@ dependencies = [ "sha2", ] +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap 2.12.0", +] + [[package]] name = "pin-project" version = "1.1.10" @@ -1835,6 +1922,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.103" @@ -1844,6 +1941,122 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "protoc-bin-vendored" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa" +dependencies = [ + "protoc-bin-vendored-linux-aarch_64", + "protoc-bin-vendored-linux-ppcle_64", + "protoc-bin-vendored-linux-s390_64", + "protoc-bin-vendored-linux-x86_32", + "protoc-bin-vendored-linux-x86_64", + "protoc-bin-vendored-macos-aarch_64", + "protoc-bin-vendored-macos-x86_64", + "protoc-bin-vendored-win32", +] + +[[package]] +name = "protoc-bin-vendored-linux-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c" + +[[package]] +name = "protoc-bin-vendored-linux-ppcle_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c" + +[[package]] +name = "protoc-bin-vendored-linux-s390_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0" + +[[package]] +name = "protoc-bin-vendored-linux-x86_32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5" + +[[package]] +name = "protoc-bin-vendored-linux-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78" + +[[package]] +name = "protoc-bin-vendored-macos-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092" + +[[package]] +name = "protoc-bin-vendored-macos-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756" + +[[package]] +name = "protoc-bin-vendored-win32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" + [[package]] name = "quinn" version = "0.11.9" @@ -1857,7 +2070,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2", + "socket2 0.6.1", "thiserror 2.0.17", "tokio", "tracing", @@ -1873,7 +2086,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -1894,7 +2107,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.6.1", "tracing", "windows-sys 0.60.2", ] @@ -1914,14 +2127,35 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ - "rand_chacha", - "rand_core", + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", ] [[package]] @@ -1931,7 +2165,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", ] [[package]] @@ -2042,7 +2285,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", - "tower", + "tower 0.5.2", "tower-http", "tower-service", "url", @@ -2106,6 +2349,35 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rustfs-admin" +version = "0.1.0" +dependencies = [ + "axum", + "chrono", + "hex", + "hmac", + "reqwest", + "serde", + "serde_json", + "sha2", + "tokio", + "url", +] + +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.35" @@ -2361,7 +2633,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap", + "indexmap 2.12.0", "itoa", "ryu", "serde", @@ -2374,7 +2646,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f" dependencies = [ - "indexmap", + "indexmap 2.12.0", "itoa", "ryu", "serde", @@ -2470,6 +2742,16 @@ dependencies = [ "syn", ] +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.1" @@ -2550,6 +2832,19 @@ dependencies = [ "syn", ] +[[package]] +name = "tempfile" +version = "3.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -2668,7 +2963,7 @@ dependencies = [ "mio", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.6.1", "tokio-macros", "windows-sys 0.61.2", ] @@ -2722,6 +3017,70 @@ dependencies = [ "tokio", ] +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2 0.5.10", + "tokio", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.7", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.5.2" @@ -2763,7 +3122,7 @@ dependencies = [ "pin-project-lite", "tokio", "tokio-util", - "tower", + "tower 0.5.2", "tower-layer", "tower-service", "tracing", @@ -2947,7 +3306,7 @@ version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fcc29c80c21c31608227e0912b2d7fddba57ad76b606890627ba8ee7964e993" dependencies = [ - "indexmap", + "indexmap 2.12.0", "serde", "serde_json", "utoipa-gen", @@ -3493,7 +3852,7 @@ dependencies = [ "crossbeam-utils", "displaydoc", "flate2", - "indexmap", + "indexmap 2.12.0", "memchr", "thiserror 2.0.17", "zopfli", diff --git a/Cargo.toml b/Cargo.toml index 84c682ec..f269619e 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,15 +31,13 @@ rustls-pemfile = "2.2.0" webpki = { package = "rustls-webpki", version = "0.103" } rcgen = "0.13" sha2 = "0.10" -hmac = "0.12" hex = "0.4" base64 = "0.22" ring = "0.17" -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } -url = "2.5" shadow-rs = "1.5.0" snafu = { version = "0.8.9", features = ["futures"] } kube-leader-election = { path = "crates/leader-election" } +rustfs-admin = { path = "crates/rustfs-admin" } hostname = "0.4" # Console dependencies @@ -63,7 +61,7 @@ shadow-rs = { version = "1.5.0", features = ["build"] } unused_variables = "allow" [workspace] -members = ["crates/leader-election"] +members = ["crates/leader-election", "crates/rustfs-admin", "crates/cosi-driver"] [lints.clippy] unwrap_used = "deny" diff --git a/Dockerfile b/Dockerfile index 97e4b1b8..612cdd96 100755 --- a/Dockerfile +++ b/Dockerfile @@ -53,7 +53,7 @@ WORKDIR /app COPY . . COPY --from=cacher /app/target target COPY --from=cacher /usr/local/cargo /usr/local/cargo -RUN cargo build --release +RUN cargo build --release -p operator -p cosi-driver # Stage 4: Build the static Console frontend FROM ${NODE_BUILD_IMAGE} AS console-web-builder @@ -72,5 +72,6 @@ FROM ${BASE_IMAGE} WORKDIR /app COPY --from=builder /app/target/release/operator . +COPY --from=builder /app/target/release/rustfs-cosi-driver . COPY --from=console-web-builder /app/console-web/out ./console-web ENTRYPOINT ["./operator"] diff --git a/Makefile b/Makefile index e0f4f5eb..8b8c4364 100644 --- a/Makefile +++ b/Makefile @@ -60,11 +60,11 @@ fmt-check: # Run clippy checks. clippy: - cargo clippy --all-features -- -D warnings + cargo clippy --workspace --all-features -- -D warnings # Run Rust tests. test: - cargo test --all + cargo test --workspace # Run frontend ESLint checks. Run pnpm install in console-web first. console-lint: @@ -84,7 +84,7 @@ console-fmt-check: # Build the project. build: - cargo build --release + cargo build --release --workspace # Rust-native e2e harness (live-first, dedicated Kind) E2E_MANIFEST ?= e2e/Cargo.toml diff --git a/crates/cosi-driver/Cargo.toml b/crates/cosi-driver/Cargo.toml new file mode 100644 index 00000000..31afb2ca --- /dev/null +++ b/crates/cosi-driver/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "cosi-driver" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" +description = "RustFS Container Object Storage Interface (COSI) v1alpha1 driver" +publish = false + +[[bin]] +name = "rustfs-cosi-driver" +path = "src/main.rs" + +[dependencies] +hex = "0.4" +k8s-openapi = { version = "0.26.1", features = ["v1_30"] } +kube = { version = "2.0.1", features = ["client", "rustls-tls"] } +prost = "0.13" +rustfs-admin = { path = "../rustfs-admin" } +sha2 = "0.10" +snafu = { version = "0.8.9", features = ["futures"] } +tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs", "signal", "net"] } +tokio-stream = { version = "0.1", features = ["net"] } +tonic = "0.12" +tracing = "0.1.44" +tracing-subscriber = { version = "0.3.22", features = ["env-filter"] } + +[build-dependencies] +protoc-bin-vendored = "3" +tonic-build = "0.12" + +[dev-dependencies] +tokio = { version = "1.49.0", features = ["rt", "macros"] } + +[lints.rust] +unused_variables = "allow" + +[lints.clippy] +unwrap_used = "deny" +expect_used = "deny" diff --git a/crates/cosi-driver/build.rs b/crates/cosi-driver/build.rs new file mode 100644 index 00000000..60df6919 --- /dev/null +++ b/crates/cosi-driver/build.rs @@ -0,0 +1,27 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +fn main() -> Result<(), Box> { + let protoc = protoc_bin_vendored::protoc_bin_path()?; + // SAFETY: build scripts run single-threaded before compilation. + unsafe { + std::env::set_var("PROTOC", protoc); + } + + tonic_build::configure() + .build_server(true) + .build_client(false) + .compile_protos(&["proto/cosi.proto"], &["proto"])?; + Ok(()) +} diff --git a/crates/cosi-driver/proto/cosi.proto b/crates/cosi-driver/proto/cosi.proto new file mode 100644 index 00000000..e14133e5 --- /dev/null +++ b/crates/cosi-driver/proto/cosi.proto @@ -0,0 +1,196 @@ +// Code generated by make; DO NOT EDIT. +syntax = "proto3"; +package cosi.v1alpha1; + + +service Identity { + // This call is meant to retrieve the unique provisioner Identity. + // This identity will have to be set in BucketClaim.DriverName field in order to invoke this specific provisioner. + rpc DriverGetInfo (DriverGetInfoRequest) returns (DriverGetInfoResponse) {} +} + +service Provisioner { + // This call is made to create the bucket in the backend. + // This call is idempotent + // 1. If a bucket that matches both name and parameters already exists, then OK (success) must be returned. + // 2. If a bucket by same name, but different parameters is provided, then the appropriate error code ALREADY_EXISTS must be returned. + rpc DriverCreateBucket (DriverCreateBucketRequest) returns (DriverCreateBucketResponse) {} + // This call is made to delete the bucket in the backend. + // If the bucket has already been deleted, then no error should be returned. + rpc DriverDeleteBucket (DriverDeleteBucketRequest) returns (DriverDeleteBucketResponse) {} + + // This call grants access to an account. The account_name in the request shall be used as a unique identifier to create credentials. + // The account_id returned in the response will be used as the unique identifier for deleting this access when calling DriverRevokeBucketAccess. + rpc DriverGrantBucketAccess (DriverGrantBucketAccessRequest) returns (DriverGrantBucketAccessResponse); + // This call revokes all access to a particular bucket from a principal. + rpc DriverRevokeBucketAccess (DriverRevokeBucketAccessRequest) returns (DriverRevokeBucketAccessResponse); +} + +// S3SignatureVersion is the version of the signing algorithm for all s3 requests +enum S3SignatureVersion { + UnknownSignature = 0; + // S3V2, Signature version v2 + S3V2 = 1; + // S3V4, Signature version v4 + S3V4 = 2; +} + +enum AnonymousBucketAccessMode { + UnknownBucketAccessMode = 0; + // Default, disallow uncredentialed access to the backend storage. + Private = 1; + // Read only, uncredentialed users can call ListBucket and GetObject. + ReadOnly = 2; + // Write only, uncredentialed users can only call PutObject. + WriteOnly = 3; + // Read/Write, uncredentialed users can read objects as well as PutObject. + ReadWrite = 4; +} + +enum AuthenticationType { + UnknownAuthenticationType = 0; + // Default, KEY based authentication. + Key = 1; + // Storageaccount based authentication. + IAM = 2; +} + +message S3 { + // region denotes the geographical region where the S3 server is running + string region = 1; + // signature_version denotes the signature version for signing all s3 requests + S3SignatureVersion signature_version = 2; +} + +message AzureBlob { + // storage_account is the id of the azure storage account + string storage_account = 1; +} + +message GCS { + // private_key_name denotes the name of the private key in the storage backend + string private_key_name = 1; + // project_id denotes the name of the project id in the storage backend + string project_id = 2; + // service_account denotes the name of the service account in the storage backend + string service_account = 3; +} + +message Protocol { + oneof type { + S3 s3 = 1; + AzureBlob azureBlob = 2; + GCS gcs = 3; + } +} + +message CredentialDetails { + // map of the details in the secrets for the protocol string + map secrets = 1; +} + +message DriverGetInfoRequest { + // Intentionally left blank +} + +message DriverGetInfoResponse { + // This field is REQUIRED + // The name MUST follow domain name notation format + // (https://tools.ietf.org/html/rfc1035#section-2.3.1). It SHOULD + // include the plugin's host company name and the plugin name, + // to minimize the possibility of collisions. It MUST be 63 + // characters or less, beginning and ending with an alphanumeric + // character () with dashes (-), dots (.), and + // alphanumerics between. + string name = 1; +} + +message DriverCreateBucketRequest { + // This field is REQUIRED + // name specifies the name of the bucket that should be created. + string name = 1; + + // This field is OPTIONAL + // The caller should treat the values in parameters as opaque. + // The receiver is responsible for parsing and validating the values. + map parameters = 2; +} + +message DriverCreateBucketResponse { + // bucket_id returned here is expected to be the globally unique + // identifier for the bucket in the object storage provider. + string bucket_id = 1; + + // bucket_info returned here stores the data specific to the + // bucket required by the object storage provider to connect to the bucket. + Protocol bucket_info = 2; +} + +message DriverDeleteBucketRequest { + // This field is REQUIRED + // bucket_id is a globally unique identifier for the bucket + // in the object storage provider + string bucket_id = 1; + + // This field is OPTIONAL + // The caller should treat the values in delete_context as opaque. + // The receiver is responsible for parsing and validating the values. + map delete_context = 2; +} + +message DriverDeleteBucketResponse { + // Intentionally left blank +} + +message DriverGrantBucketAccessRequest { + // This field is REQUIRED + // bucket_id is a globally unique identifier for the bucket + // in the object storage provider + string bucket_id = 1; + + // This field is REQUIRED + // name field is used to define the name of the bucket access object. + string name = 2; + + // This field is REQUIRED + // Requested authentication type for the bucket access. + // Supported authentication types are KEY or IAM. + AuthenticationType authentication_type = 3; + + // This field is OPTIONAL + // The caller should treat the values in parameters as opaque. + // The receiver is responsible for parsing and validating the values. + map parameters = 4; +} + +message DriverGrantBucketAccessResponse { + // This field is REQUIRED + // This is the account_id that is being provided access. This will + // be required later to revoke access. + string account_id = 1; + + // This field is REQUIRED + // Credentials supplied for accessing the bucket ex: aws access key id and secret, etc. + map credentials = 2; +} + +message DriverRevokeBucketAccessRequest { + // This field is REQUIRED + // bucket_id is a globally unique identifier for the bucket + // in the object storage provider. + string bucket_id = 1; + + // This field is REQUIRED + // This is the account_id that is having its access revoked. + string account_id = 2; + + // This field is OPTIONAL + // The caller should treat the values in revoke_access_context as opaque. + // The receiver is responsible for parsing and validating the values. + map revoke_access_context = 3; +} + +message DriverRevokeBucketAccessResponse { + // Intentionally left blank +} + diff --git a/crates/cosi-driver/src/backend.rs b/crates/cosi-driver/src/backend.rs new file mode 100644 index 00000000..93aaa6eb --- /dev/null +++ b/crates/cosi-driver/src/backend.rs @@ -0,0 +1,199 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Kubernetes Secret / ConfigMap loading and RustFS admin client construction. + +use k8s_openapi::ByteString; +use k8s_openapi::api::core::v1::{ConfigMap, Secret}; +use kube::{Api, Client}; +use rustfs_admin::{RustfsAdminClient, RustfsClientError, RustfsCredentials}; +use snafu::{ResultExt, Snafu}; +use std::collections::BTreeMap; + +use crate::parameters::BackendParameters; + +fn box_kube(err: kube::Error) -> Box { + Box::new(err) +} + +#[derive(Debug, Snafu)] +pub enum BackendError { + #[snafu(display("failed to create kubernetes client: {source}"))] + KubeClient { source: Box }, + #[snafu(display("failed to read Secret {namespace}/{name}: {source}"))] + SecretLookup { + namespace: String, + name: String, + source: Box, + }, + #[snafu(display("failed to read ConfigMap {namespace}/{name}: {source}"))] + ConfigMapLookup { + namespace: String, + name: String, + source: Box, + }, + #[snafu(display("Secret {namespace}/{name} missing key `{key}`"))] + MissingSecretKey { + namespace: String, + name: String, + key: &'static str, + }, + #[snafu(display("Secret {namespace}/{name} key `{key}` is not valid UTF-8"))] + InvalidSecretKey { + namespace: String, + name: String, + key: &'static str, + }, + #[snafu(display("Secret {namespace}/{name} key `{key}` is empty"))] + EmptySecretKey { + namespace: String, + name: String, + key: &'static str, + }, + #[snafu(display("ConfigMap {namespace}/{name} missing CA data key"))] + MissingCaData { namespace: String, name: String }, + #[snafu(display("failed to build RustFS admin client: {source}"))] + ClientBuild { source: RustfsClientError }, +} + +#[derive(Clone)] +pub struct BackendFactory { + kube: Client, +} + +impl BackendFactory { + pub async fn try_default() -> Result { + let kube = Client::try_default() + .await + .map_err(box_kube) + .context(KubeClientSnafu)?; + Ok(Self { kube }) + } + + #[cfg(test)] + pub fn from_client(kube: Client) -> Self { + Self { kube } + } + + pub async fn admin_client( + &self, + params: &BackendParameters, + ) -> Result { + let credentials = self + .load_credentials(¶ms.secret_namespace, ¶ms.secret_name) + .await?; + + match ( + params.tls_ca_configmap_name.as_deref(), + params.tls_ca_configmap_namespace.as_deref(), + ) { + (Some(name), Some(namespace)) => { + let ca_pem = self.load_ca_pem(namespace, name).await?; + RustfsAdminClient::new_with_base_url_and_ca_pem( + params.endpoint.clone(), + credentials.access_key, + credentials.secret_key, + &ca_pem, + ) + .context(ClientBuildSnafu) + } + _ => Ok(RustfsAdminClient::new_with_base_url( + params.endpoint.clone(), + credentials.access_key, + credentials.secret_key, + )), + } + } + + async fn load_credentials( + &self, + namespace: &str, + name: &str, + ) -> Result { + let api: Api = Api::namespaced(self.kube.clone(), namespace); + let secret = api + .get(name) + .await + .map_err(box_kube) + .context(SecretLookupSnafu { + namespace: namespace.to_string(), + name: name.to_string(), + })?; + let data = secret.data.as_ref(); + Ok(RustfsCredentials { + access_key: secret_value(data, namespace, name, "accesskey")?, + secret_key: secret_value(data, namespace, name, "secretkey")?, + }) + } + + async fn load_ca_pem(&self, namespace: &str, name: &str) -> Result, BackendError> { + let api: Api = Api::namespaced(self.kube.clone(), namespace); + let cm = api + .get(name) + .await + .map_err(box_kube) + .context(ConfigMapLookupSnafu { + namespace: namespace.to_string(), + name: name.to_string(), + })?; + + if let Some(data) = cm.data.as_ref() { + for key in ["ca.crt", "tls.crt", "ca-bundle.crt"] { + if let Some(value) = data.get(key).filter(|v| !v.trim().is_empty()) { + return Ok(value.as_bytes().to_vec()); + } + } + } + if let Some(bin) = cm.binary_data.as_ref() { + for key in ["ca.crt", "tls.crt", "ca-bundle.crt"] { + if let Some(value) = bin.get(key).filter(|v| !v.0.is_empty()) { + return Ok(value.0.clone()); + } + } + } + + Err(BackendError::MissingCaData { + namespace: namespace.to_string(), + name: name.to_string(), + }) + } +} + +fn secret_value( + data: Option<&BTreeMap>, + namespace: &str, + name: &str, + key: &'static str, +) -> Result { + let raw = + data.and_then(|data| data.get(key)) + .ok_or_else(|| BackendError::MissingSecretKey { + namespace: namespace.to_string(), + name: name.to_string(), + key, + })?; + let value = String::from_utf8(raw.0.clone()).map_err(|_| BackendError::InvalidSecretKey { + namespace: namespace.to_string(), + name: name.to_string(), + key, + })?; + if value.is_empty() { + return Err(BackendError::EmptySecretKey { + namespace: namespace.to_string(), + name: name.to_string(), + key, + }); + } + Ok(value) +} diff --git a/crates/cosi-driver/src/driver.rs b/crates/cosi-driver/src/driver.rs new file mode 100644 index 00000000..34c4fafe --- /dev/null +++ b/crates/cosi-driver/src/driver.rs @@ -0,0 +1,278 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! gRPC Identity and Provisioner servers for COSI v1alpha1. + +#![allow(clippy::result_large_err)] + +use std::collections::HashMap; + +use sha2::{Digest, Sha256}; +use tonic::{Request, Response, Status}; +use tracing::info; + +use crate::backend::{BackendError, BackendFactory}; +use crate::parameters::{BackendParameters, ParameterError}; +use crate::policy::{bucket_policy_document, policy_name_for}; +use crate::proto::cosi::v1alpha1::{ + AuthenticationType, CredentialDetails, DriverCreateBucketRequest, DriverCreateBucketResponse, + DriverDeleteBucketRequest, DriverDeleteBucketResponse, DriverGetInfoRequest, + DriverGetInfoResponse, DriverGrantBucketAccessRequest, DriverGrantBucketAccessResponse, + DriverRevokeBucketAccessRequest, DriverRevokeBucketAccessResponse, Protocol, S3, + S3SignatureVersion, identity_server::Identity, provisioner_server::Provisioner, +}; + +pub const DRIVER_NAME: &str = "rustfs.objectstorage.k8s.io"; + +/// Deterministic secret so DriverGrantBucketAccess is idempotent across sidecar retries. +fn credentials_for_account(account_id: &str) -> String { + let digest = Sha256::digest(format!("rustfs-cosi-v1:{account_id}").as_bytes()); + hex::encode(digest) +} + +#[cfg(test)] +mod credential_tests { + use super::credentials_for_account; + + #[test] + fn credentials_are_deterministic_and_long_enough() { + let a = credentials_for_account("ba-test-uid"); + let b = credentials_for_account("ba-test-uid"); + assert_eq!(a, b); + assert!(a.len() >= 8); + assert_ne!(a, credentials_for_account("other-account")); + } +} + +pub struct IdentityService { + pub name: String, +} + +#[tonic::async_trait] +impl Identity for IdentityService { + async fn driver_get_info( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(DriverGetInfoResponse { + name: self.name.clone(), + })) + } +} + +pub struct ProvisionerService { + pub backend: BackendFactory, +} + +#[tonic::async_trait] +impl Provisioner for ProvisionerService { + async fn driver_create_bucket( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let bucket_name = req.name.trim(); + if bucket_name.is_empty() { + return Err(Status::invalid_argument("bucket name is required")); + } + + let params = parse_params(&req.parameters)?; + let client = self + .backend + .admin_client(¶ms) + .await + .map_err(map_backend)?; + + info!(bucket = %bucket_name, endpoint = %params.endpoint, "creating bucket"); + client + .create_bucket(bucket_name, params.region.as_deref(), false) + .await + .map_err(map_admin)?; + + Ok(Response::new(DriverCreateBucketResponse { + bucket_id: bucket_name.to_string(), + bucket_info: Some(Protocol { + r#type: Some(crate::proto::cosi::v1alpha1::protocol::Type::S3(S3 { + region: params.region.unwrap_or_else(|| "us-east-1".to_string()), + signature_version: S3SignatureVersion::S3v4 as i32, + })), + }), + })) + } + + async fn driver_delete_bucket( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let bucket_id = req.bucket_id.trim(); + if bucket_id.is_empty() { + return Err(Status::invalid_argument("bucket_id is required")); + } + + let params = parse_params(&req.delete_context)?; + let client = self + .backend + .admin_client(¶ms) + .await + .map_err(map_backend)?; + + info!(bucket = %bucket_id, "deleting bucket"); + client.delete_bucket(bucket_id).await.map_err(map_admin)?; + + Ok(Response::new(DriverDeleteBucketResponse {})) + } + + async fn driver_grant_bucket_access( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let bucket_id = req.bucket_id.trim(); + let account_name = req.name.trim(); + if bucket_id.is_empty() { + return Err(Status::invalid_argument("bucket_id is required")); + } + if account_name.is_empty() { + return Err(Status::invalid_argument("account name is required")); + } + if req.authentication_type != AuthenticationType::Key as i32 + && req.authentication_type != AuthenticationType::UnknownAuthenticationType as i32 + { + return Err(Status::invalid_argument( + "only KEY authentication is supported", + )); + } + + let params = parse_params(&req.parameters)?; + let client = self + .backend + .admin_client(¶ms) + .await + .map_err(map_backend)?; + + let account_id = account_name.to_string(); + let secret_key = credentials_for_account(&account_id); + let policy_name = policy_name_for(&account_id, bucket_id); + let policy_doc = bucket_policy_document(bucket_id, params.access_policy); + + info!( + bucket = %bucket_id, + account = %account_id, + policy = %policy_name, + "granting bucket access" + ); + + client + .add_canned_policy(&policy_name, &policy_doc) + .await + .map_err(map_admin)?; + + if !client.user_exists(&account_id).await.map_err(map_admin)? { + client + .add_user(&account_id, &secret_key) + .await + .map_err(map_admin)?; + } + client + .set_user_policy(&account_id, std::slice::from_ref(&policy_name)) + .await + .map_err(map_admin)?; + + let mut secrets = HashMap::new(); + secrets.insert("endpoint".to_string(), params.endpoint.clone()); + secrets.insert( + "region".to_string(), + params.region.unwrap_or_else(|| "us-east-1".to_string()), + ); + secrets.insert("accessKeyID".to_string(), account_id.clone()); + secrets.insert("accessSecretKey".to_string(), secret_key); + secrets.insert("bucketName".to_string(), bucket_id.to_string()); + + let mut credentials = HashMap::new(); + credentials.insert("s3".to_string(), CredentialDetails { secrets }); + + Ok(Response::new(DriverGrantBucketAccessResponse { + account_id, + credentials, + })) + } + + async fn driver_revoke_bucket_access( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let bucket_id = req.bucket_id.trim(); + let account_id = req.account_id.trim(); + if account_id.is_empty() { + return Err(Status::invalid_argument("account_id is required")); + } + + let params = parse_params(&req.revoke_access_context)?; + let client = self + .backend + .admin_client(¶ms) + .await + .map_err(map_backend)?; + + let policy_name = if bucket_id.is_empty() { + None + } else { + Some(policy_name_for(account_id, bucket_id)) + }; + + info!(account = %account_id, bucket = %bucket_id, "revoking bucket access"); + client.remove_user(account_id).await.map_err(map_admin)?; + if let Some(policy_name) = policy_name { + client + .remove_canned_policy(&policy_name) + .await + .map_err(map_admin)?; + } + + Ok(Response::new(DriverRevokeBucketAccessResponse {})) + } +} + +fn parse_params(params: &HashMap) -> Result { + BackendParameters::from_map(params).map_err(map_params) +} + +fn map_params(err: ParameterError) -> Status { + Status::invalid_argument(err.to_string()) +} + +fn map_backend(err: BackendError) -> Status { + Status::failed_precondition(err.to_string()) +} + +fn map_admin(err: rustfs_admin::RustfsClientError) -> Status { + match &err { + rustfs_admin::RustfsClientError::UnexpectedStatus { status, .. } + if status.as_u16() == 409 => + { + Status::already_exists(err.to_string()) + } + rustfs_admin::RustfsClientError::InvalidPolicyName + | rustfs_admin::RustfsClientError::InvalidPolicyDocument + | rustfs_admin::RustfsClientError::InvalidCredentialValue { .. } + | rustfs_admin::RustfsClientError::EmptyCredentialValue { .. } + | rustfs_admin::RustfsClientError::MissingCredentialKey { .. } + | rustfs_admin::RustfsClientError::RequestBuildFailed => { + Status::invalid_argument(err.to_string()) + } + _ => Status::internal(err.to_string()), + } +} diff --git a/crates/cosi-driver/src/lib.rs b/crates/cosi-driver/src/lib.rs new file mode 100644 index 00000000..761d3e88 --- /dev/null +++ b/crates/cosi-driver/src/lib.rs @@ -0,0 +1,30 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! RustFS COSI v1alpha1 driver library (Identity + Provisioner gRPC services). + +pub mod backend; +pub mod driver; +pub mod parameters; +pub mod policy; + +pub mod proto { + pub mod cosi { + pub mod v1alpha1 { + tonic::include_proto!("cosi.v1alpha1"); + } + } +} + +pub use driver::{DRIVER_NAME, IdentityService, ProvisionerService}; diff --git a/crates/cosi-driver/src/main.rs b/crates/cosi-driver/src/main.rs new file mode 100644 index 00000000..ae7face5 --- /dev/null +++ b/crates/cosi-driver/src/main.rs @@ -0,0 +1,123 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::path::PathBuf; +use std::time::Duration; + +use cosi_driver::backend::BackendFactory; +use cosi_driver::driver::{DRIVER_NAME, IdentityService, ProvisionerService}; +use cosi_driver::proto::cosi::v1alpha1::{ + identity_server::IdentityServer, provisioner_server::ProvisionerServer, +}; +use tokio::net::UnixListener; +use tokio_stream::wrappers::UnixListenerStream; +use tonic::transport::Server; +use tracing::{info, warn}; +use tracing_subscriber::EnvFilter; + +const DEFAULT_ENDPOINT: &str = "unix:///var/lib/cosi/cosi.sock"; +const SHUTDOWN_GRACE: Duration = Duration::from_secs(5); + +#[tokio::main] +async fn main() -> Result<(), Box> { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env().add_directive("info".parse()?)) + .init(); + + let endpoint = std::env::var("COSI_ENDPOINT").unwrap_or_else(|_| DEFAULT_ENDPOINT.to_string()); + let socket_path = parse_unix_endpoint(&endpoint)?; + + if let Some(parent) = socket_path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + let _ = tokio::fs::remove_file(&socket_path).await; + + let backend = BackendFactory::try_default().await?; + let identity = IdentityService { + name: DRIVER_NAME.to_string(), + }; + let provisioner = ProvisionerService { backend }; + + let uds = UnixListener::bind(&socket_path)?; + let uds_stream = UnixListenerStream::new(uds); + + info!( + driver = DRIVER_NAME, + endpoint = %endpoint, + "starting RustFS COSI driver" + ); + + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>(); + tokio::spawn(async move { + wait_for_shutdown().await; + let _ = shutdown_tx.send(()); + }); + + Server::builder() + .add_service(IdentityServer::new(identity)) + .add_service(ProvisionerServer::new(provisioner)) + .serve_with_incoming_shutdown(uds_stream, async { + let _ = shutdown_rx.await; + info!("shutdown signal received"); + }) + .await?; + + let _ = tokio::fs::remove_file(&socket_path).await; + // Allow in-flight RPCs a brief window before process exit. + tokio::time::sleep(SHUTDOWN_GRACE).await; + info!("RustFS COSI driver stopped"); + Ok(()) +} + +fn parse_unix_endpoint(endpoint: &str) -> Result { + let endpoint = endpoint.trim(); + if let Some(path) = endpoint.strip_prefix("unix://") { + if path.is_empty() { + return Err("COSI_ENDPOINT unix path is empty".into()); + } + return Ok(PathBuf::from(path)); + } + if endpoint.starts_with('/') { + return Ok(PathBuf::from(endpoint)); + } + Err(format!( + "unsupported COSI_ENDPOINT `{endpoint}` (expected unix:///path/to.sock)" + )) +} + +async fn wait_for_shutdown() { + let ctrl_c = async { + if let Err(err) = tokio::signal::ctrl_c().await { + warn!(error = %err, "failed to install Ctrl+C handler"); + } + }; + + #[cfg(unix)] + let terminate = async { + match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) { + Ok(mut stream) => { + stream.recv().await; + } + Err(err) => warn!(error = %err, "failed to install SIGTERM handler"), + } + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + () = ctrl_c => {}, + () = terminate => {}, + } +} diff --git a/crates/cosi-driver/src/parameters.rs b/crates/cosi-driver/src/parameters.rs new file mode 100644 index 00000000..ed7f40dc --- /dev/null +++ b/crates/cosi-driver/src/parameters.rs @@ -0,0 +1,119 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! BucketClass / BucketAccessClass parameter parsing (Rook-style). + +use std::collections::HashMap; + +use snafu::Snafu; + +use crate::policy::AccessPolicy; + +pub const PARAM_SECRET_NAME: &str = "objectStoreUserSecretName"; +pub const PARAM_SECRET_NAMESPACE: &str = "objectStoreUserSecretNamespace"; +pub const PARAM_ENDPOINT: &str = "endpoint"; +pub const PARAM_REGION: &str = "region"; +pub const PARAM_TLS_CA_CM_NAME: &str = "tlsCAConfigMapName"; +pub const PARAM_TLS_CA_CM_NAMESPACE: &str = "tlsCAConfigMapNamespace"; +pub const PARAM_POLICY: &str = "policy"; + +#[derive(Debug, Snafu)] +pub enum ParameterError { + #[snafu(display("missing required parameter `{name}`"))] + Missing { name: &'static str }, + #[snafu(display("parameter `{name}` must not be empty"))] + Empty { name: &'static str }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BackendParameters { + pub secret_name: String, + pub secret_namespace: String, + pub endpoint: String, + pub region: Option, + pub tls_ca_configmap_name: Option, + pub tls_ca_configmap_namespace: Option, + pub access_policy: AccessPolicy, +} + +impl BackendParameters { + pub fn from_map(params: &HashMap) -> Result { + Ok(Self { + secret_name: required(params, PARAM_SECRET_NAME)?, + secret_namespace: required(params, PARAM_SECRET_NAMESPACE)?, + endpoint: required(params, PARAM_ENDPOINT)?, + region: optional(params, PARAM_REGION), + tls_ca_configmap_name: optional(params, PARAM_TLS_CA_CM_NAME), + tls_ca_configmap_namespace: optional(params, PARAM_TLS_CA_CM_NAMESPACE), + access_policy: AccessPolicy::parse(params.get(PARAM_POLICY).map(String::as_str)), + }) + } +} + +fn required( + params: &HashMap, + name: &'static str, +) -> Result { + let value = params + .get(name) + .ok_or(ParameterError::Missing { name })? + .trim(); + if value.is_empty() { + return Err(ParameterError::Empty { name }); + } + Ok(value.to_string()) +} + +fn optional(params: &HashMap, name: &str) -> Option { + params + .get(name) + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_required_parameters() { + let mut map = HashMap::new(); + map.insert(PARAM_SECRET_NAME.to_string(), "creds".into()); + map.insert(PARAM_SECRET_NAMESPACE.to_string(), "ns".into()); + map.insert( + PARAM_ENDPOINT.to_string(), + "http://tenant-io.ns.svc:9000".into(), + ); + map.insert(PARAM_POLICY.to_string(), "readonly".into()); + + let parsed = BackendParameters::from_map(&map).expect("parse"); + assert_eq!(parsed.secret_name, "creds"); + assert_eq!(parsed.access_policy, AccessPolicy::Readonly); + assert_eq!(parsed.endpoint, "http://tenant-io.ns.svc:9000"); + } + + #[test] + fn rejects_missing_endpoint() { + let mut map = HashMap::new(); + map.insert(PARAM_SECRET_NAME.to_string(), "creds".into()); + map.insert(PARAM_SECRET_NAMESPACE.to_string(), "ns".into()); + let err = BackendParameters::from_map(&map).expect_err("missing endpoint"); + assert!(matches!( + err, + ParameterError::Missing { + name: PARAM_ENDPOINT + } + )); + } +} diff --git a/crates/cosi-driver/src/policy.rs b/crates/cosi-driver/src/policy.rs new file mode 100644 index 00000000..d062f999 --- /dev/null +++ b/crates/cosi-driver/src/policy.rs @@ -0,0 +1,126 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! IAM policy documents scoped to a single bucket for COSI BucketAccess grants. + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AccessPolicy { + Readonly, + ReadWrite, +} + +impl AccessPolicy { + pub fn parse(value: Option<&str>) -> Self { + match value.map(str::trim).map(str::to_ascii_lowercase).as_deref() { + Some("readonly") | Some("read-only") | Some("read") => Self::Readonly, + _ => Self::ReadWrite, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::Readonly => "readonly", + Self::ReadWrite => "readwrite", + } + } +} + +/// Build a canned IAM policy JSON document limited to `bucket`. +pub fn bucket_policy_document(bucket: &str, policy: AccessPolicy) -> String { + let actions = match policy { + AccessPolicy::Readonly => { + r#"[ + "s3:GetBucketLocation", + "s3:ListBucket", + "s3:GetObject" + ]"# + } + AccessPolicy::ReadWrite => { + r#"[ + "s3:GetBucketLocation", + "s3:ListBucket", + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ]"# + } + }; + + format!( + r#"{{ + "Version": "2012-10-17", + "Statement": [ + {{ + "Effect": "Allow", + "Action": {actions}, + "Resource": [ + "arn:aws:s3:::{bucket}", + "arn:aws:s3:::{bucket}/*" + ] + }} + ] +}}"# + ) +} + +/// Deterministic canned policy name for a COSI account + bucket pair. +pub fn policy_name_for(account_id: &str, bucket_id: &str) -> String { + // RustFS policy names should stay reasonably short and DNS-safe. + let raw = format!("cosi-{account_id}-{bucket_id}"); + raw.chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' { + ch + } else { + '-' + } + }) + .take(128) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_policy_defaults_to_readwrite() { + assert_eq!(AccessPolicy::parse(None), AccessPolicy::ReadWrite); + assert_eq!(AccessPolicy::parse(Some("")), AccessPolicy::ReadWrite); + assert_eq!( + AccessPolicy::parse(Some("readwrite")), + AccessPolicy::ReadWrite + ); + assert_eq!( + AccessPolicy::parse(Some("readonly")), + AccessPolicy::Readonly + ); + } + + #[test] + fn policy_document_includes_bucket_resources() { + let doc = bucket_policy_document("my-bucket", AccessPolicy::Readonly); + assert!(doc.contains("arn:aws:s3:::my-bucket")); + assert!(doc.contains("s3:GetObject")); + assert!(!doc.contains("s3:PutObject")); + } + + #[test] + fn policy_name_is_sanitized() { + let name = policy_name_for("ba.uid", "bucket/name"); + assert!(!name.contains('.')); + assert!(!name.contains('/')); + assert!(name.starts_with("cosi-")); + } +} diff --git a/crates/rustfs-admin/Cargo.toml b/crates/rustfs-admin/Cargo.toml new file mode 100644 index 00000000..c91ecfe4 --- /dev/null +++ b/crates/rustfs-admin/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "rustfs-admin" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" +description = "Kube-agnostic RustFS admin/S3/STS client" + +[dependencies] +chrono = { version = "0.4", features = ["serde"] } +hex = "0.4" +hmac = "0.12" +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +sha2 = "0.10" +url = "2.5" + +[dev-dependencies] +axum = { version = "0.7", features = ["macros", "json"] } +tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "sync", "time"] } + +[lints.rust] +unused_variables = "allow" diff --git a/src/sts/admin_ops.rs b/crates/rustfs-admin/src/admin_ops.rs similarity index 72% rename from src/sts/admin_ops.rs rename to crates/rustfs-admin/src/admin_ops.rs index 593d1a57..837c6191 100644 --- a/src/sts/admin_ops.rs +++ b/crates/rustfs-admin/src/admin_ops.rs @@ -18,17 +18,19 @@ use std::collections::BTreeMap; -use super::helpers::{ - body_mentions_not_found, build_canonical_query, extract_canned_policy_document, -}; -use super::{ - ADD_CANNED_POLICY_PATH, ADD_USER_PATH, ADMIN_SIGNING_SERVICE, INFO_CANNED_POLICY_PATH, - JSON_CONTENT_TYPE, LIST_CANNED_POLICIES_PATH, RustfsAdminClient, RustfsClientError, - RustfsServerInfo, RustfsServerInfoResponse, SERVER_INFO_PATH, SET_POLICY_PATH, USER_INFO_PATH, -}; use reqwest::StatusCode; use serde_json::Value; +use crate::client::{ + ADD_CANNED_POLICY_PATH, ADD_USER_PATH, ADMIN_SIGNING_SERVICE, INFO_CANNED_POLICY_PATH, + JSON_CONTENT_TYPE, LIST_CANNED_POLICIES_PATH, REMOVE_CANNED_POLICY_PATH, REMOVE_USER_PATH, + RustfsAdminClient, RustfsClientError, RustfsServerInfo, RustfsServerInfoResponse, + SERVER_INFO_PATH, SET_POLICY_PATH, USER_INFO_PATH, +}; +use crate::helpers::{ + body_mentions_not_found, build_canonical_query, extract_canned_policy_document, +}; + impl RustfsAdminClient { // Admin duties: user and policy management APIs. // (Candidly scoped to tenant admin operations.) @@ -120,6 +122,45 @@ impl RustfsAdminClient { Ok(()) } + /// Remove a RustFS canned policy. Succeeds if the policy is already gone. + pub async fn remove_canned_policy(&self, policy_name: &str) -> Result<(), RustfsClientError> { + if policy_name.trim().is_empty() { + return Err(RustfsClientError::InvalidPolicyName); + } + + let query = build_canonical_query(&[("name", policy_name)]); + let path = REMOVE_CANNED_POLICY_PATH; + let url = format!("{}{}?{query}", self.base_url.trim_end_matches('/'), path); + + let signed = self.sign_request("DELETE", path, &query, "", None, ADMIN_SIGNING_SERVICE)?; + let host = self.host()?; + + let response = self + .http_client + .delete(url) + .header("x-amz-date", &signed.amz_date) + .header("x-amz-content-sha256", &signed.payload_hash) + .header("authorization", &signed.authorization) + .header("host", host) + .send() + .await + .map_err(|_| RustfsClientError::RequestFailed)?; + + if response.status().is_success() { + return Ok(()); + } + + let status = response.status(); + let (body, truncated) = RustfsClientError::limited_response_body(response).await; + if status == StatusCode::NOT_FOUND || body_mentions_not_found(&body) { + return Ok(()); + } + + Err(RustfsClientError::unexpected_status_with_limited_body( + status, &body, truncated, + )) + } + pub async fn list_canned_policies( &self, ) -> Result, RustfsClientError> { @@ -216,6 +257,45 @@ impl RustfsAdminClient { .map(|_| ()) } + /// Remove a RustFS user. Succeeds if the user is already gone. + pub async fn remove_user(&self, access_key: &str) -> Result<(), RustfsClientError> { + if access_key.trim().is_empty() { + return Err(RustfsClientError::InvalidCredentialValue { key: "accesskey" }); + } + + let query = build_canonical_query(&[("accessKey", access_key)]); + let path = REMOVE_USER_PATH; + let url = format!("{}{}?{query}", self.base_url.trim_end_matches('/'), path); + + let signed = self.sign_request("DELETE", path, &query, "", None, ADMIN_SIGNING_SERVICE)?; + let host = self.host()?; + + let response = self + .http_client + .delete(url) + .header("x-amz-date", &signed.amz_date) + .header("x-amz-content-sha256", &signed.payload_hash) + .header("authorization", &signed.authorization) + .header("host", host) + .send() + .await + .map_err(|_| RustfsClientError::RequestFailed)?; + + if response.status().is_success() { + return Ok(()); + } + + let status = response.status(); + let (body, truncated) = RustfsClientError::limited_response_body(response).await; + if status == StatusCode::NOT_FOUND || body_mentions_not_found(&body) { + return Ok(()); + } + + Err(RustfsClientError::unexpected_status_with_limited_body( + status, &body, truncated, + )) + } + pub async fn set_user_policy( &self, access_key: &str, diff --git a/crates/rustfs-admin/src/client.rs b/crates/rustfs-admin/src/client.rs new file mode 100644 index 00000000..11ae191b --- /dev/null +++ b/crates/rustfs-admin/src/client.rs @@ -0,0 +1,470 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Client types: credentials, response models, error type and the +//! `RustfsAdminClient` handle used by every ops module. +use std::{collections::BTreeMap, time::Duration}; + +use reqwest::{Certificate, Client as HttpClient, Response, StatusCode}; + +use crate::sanitize::redact_sensitive_pairs; + +pub(crate) const FORM_CONTENT_TYPE: &str = "application/x-www-form-urlencoded"; +pub(crate) const JSON_CONTENT_TYPE: &str = "application/json"; +pub(crate) const ASSUME_ROLE_PATH: &str = "/"; +pub(crate) const ADD_USER_PATH: &str = "/rustfs/admin/v3/add-user"; +pub(crate) const REMOVE_USER_PATH: &str = "/rustfs/admin/v3/remove-user"; +pub(crate) const USER_INFO_PATH: &str = "/rustfs/admin/v3/user-info"; +pub(crate) const SET_POLICY_PATH: &str = "/rustfs/admin/v3/set-policy"; +pub(crate) const LIST_CANNED_POLICIES_PATH: &str = "/rustfs/admin/v3/list-canned-policies"; +pub(crate) const ADD_CANNED_POLICY_PATH: &str = "/rustfs/admin/v3/add-canned-policy"; +pub(crate) const REMOVE_CANNED_POLICY_PATH: &str = "/rustfs/admin/v3/remove-canned-policy"; +pub(crate) const INFO_CANNED_POLICY_PATH: &str = "/rustfs/admin/v3/info-canned-policy"; +pub(crate) const SERVER_INFO_PATH: &str = "/rustfs/admin/v3/info"; +pub(crate) const POOLS_LIST_PATH: &str = "/rustfs/admin/v3/pools/list"; +pub(crate) const POOLS_STATUS_PATH: &str = "/rustfs/admin/v3/pools/status"; +pub(crate) const POOLS_DECOMMISSION_PATH: &str = "/rustfs/admin/v3/pools/decommission"; +pub(crate) const POOLS_CANCEL_PATH: &str = "/rustfs/admin/v3/pools/cancel"; +pub(crate) const ADMIN_SIGNING_SERVICE: &str = "s3"; +pub(crate) const STS_SIGNING_SERVICE: &str = "sts"; +pub(crate) const ADMIN_HTTP_CONNECT_TIMEOUT: Duration = Duration::from_secs(3); +pub(crate) const ADMIN_HTTP_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); +pub(crate) const MAX_UPSTREAM_ERROR_BODY_BYTES: usize = 8 * 1024; +pub(crate) const MAX_UPSTREAM_ERROR_DETAIL_CHARS: usize = 512; + +/// Credentials used to sign requests against a RustFS admin/S3/STS endpoint. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RustfsCredentials { + pub access_key: String, + pub secret_key: String, +} + +#[derive(Debug, Clone, serde::Deserialize, PartialEq)] +pub struct RustfsPoolListItem { + pub id: usize, + #[serde(rename = "cmdline")] + pub cmd_line: String, + #[serde(rename = "lastUpdate")] + pub last_update: String, + #[serde(rename = "totalSize")] + pub total_size: Option, + #[serde(rename = "currentSize")] + pub current_size: Option, + #[serde(rename = "usedSize")] + pub used_size: Option, + pub used: Option, + pub status: String, + #[serde(rename = "decommissionInfo")] + pub decommission: Option, +} + +#[derive(Debug, Clone, serde::Deserialize, PartialEq)] +pub struct RustfsPoolStatus { + pub id: usize, + #[serde(rename = "cmdline")] + pub cmd_line: String, + #[serde(rename = "lastUpdate")] + pub last_update: String, + #[serde(rename = "decommissionInfo")] + pub decommission: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CreateBucketResult { + Created, + AlreadyExists, +} + +#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] +pub struct RustfsPoolDecommissionInfo { + #[serde(rename = "startTime")] + pub start_time: Option, + #[serde(rename = "startSize")] + pub start_size: Option, + #[serde(rename = "totalSize")] + pub total_size: Option, + #[serde(rename = "currentSize")] + pub current_size: Option, + pub complete: Option, + pub failed: Option, + pub canceled: Option, + #[serde(rename = "objectsDecommissioned")] + pub objects_decommissioned: Option, + #[serde(rename = "objectsDecommissionedFailed")] + pub objects_decommissioned_failed: Option, + #[serde(rename = "bytesDecommissioned")] + pub bytes_decommissioned: Option, + #[serde(rename = "bytesDecommissionedFailed")] + pub bytes_decommissioned_failed: Option, +} + +#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] +pub struct RustfsServerInfo { + #[serde(default)] + pub usage: Option, + #[serde(default)] + pub backend: Option, + #[serde(default)] + pub pools: Option>>, +} + +#[derive(Debug, Clone, serde::Deserialize, PartialEq)] +pub(crate) struct RustfsServerInfoResponse { + pub info: RustfsServerInfo, +} + +#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] +pub struct RustfsServerUsage { + #[serde(default)] + pub size: u64, +} + +#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] +pub struct RustfsErasureBackend { + #[serde(default, rename = "onlineDisks")] + pub online_disks: u64, + #[serde(default, rename = "offlineDisks")] + pub offline_disks: u64, + #[serde(default, rename = "standardSCParity", alias = "StandardSCParity")] + pub standard_sc_parity: Option, + #[serde(default, rename = "totalSets")] + pub total_sets: Vec, + #[serde(default, rename = "totalDrivesPerSet", alias = "drivesPerSet")] + pub drives_per_set: Vec, +} + +#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] +pub struct RustfsErasureSetInfo { + #[serde(default, rename = "rawUsage")] + pub raw_usage: u64, + #[serde(default, rename = "rawCapacity")] + pub raw_capacity: u64, + #[serde(default)] + pub usage: u64, + #[serde(default, rename = "objectsCount")] + pub objects_count: u64, + #[serde(default, rename = "healDisks")] + pub heal_disks: u64, +} + +/// Error type for RustFS admin/STS client operations. +/// +/// This also carries the Tenant/kube-related variants used by the operator's +/// kube-aware wrappers (see the `operator` crate's `sts::rustfs_client` +/// module), so both crates can share a single error type end-to-end. +#[derive(Debug)] +pub enum RustfsClientError { + MissingTenantNamespace, + MissingCredsSecret, + MissingCredentialKey { + key: &'static str, + }, + EmptyCredentialValue { + key: &'static str, + }, + InvalidCredentialValue { + key: &'static str, + }, + TenantSecretLookupFailed, + InvalidPolicyName, + InvalidPolicyDocument, + TenantTlsRequired, + TenantTlsNotReady, + TenantTlsClientCertificateRequired, + MissingTenantTlsCaKey { + secret: String, + key: String, + }, + TenantTlsCaSecretLookupFailed { + secret: String, + }, + InvalidTenantTlsCa, + TlsClientBuildFailed, + RequestBuildFailed, + RequestFailed, + UnexpectedStatus { + status: StatusCode, + detail: Option, + }, + ParseResponseFailed, + SigningFailed, +} + +impl std::fmt::Display for RustfsClientError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::MissingTenantNamespace => write!(f, "tenant namespace is missing"), + Self::MissingCredsSecret => write!(f, "tenant credsSecret is missing"), + Self::MissingCredentialKey { key } => write!(f, "secret key missing: {key}"), + Self::EmptyCredentialValue { key } => write!(f, "secret key empty: {key}"), + Self::InvalidCredentialValue { key } => { + write!(f, "secret key is not valid utf8: {key}") + } + Self::TenantSecretLookupFailed => { + write!(f, "failed to load tenant credential secret") + } + Self::InvalidPolicyName => write!(f, "invalid policy name"), + Self::InvalidPolicyDocument => write!(f, "failed to parse canned policy response"), + Self::TenantTlsRequired => write!(f, "STS requires a TLS-enabled tenant"), + Self::TenantTlsNotReady => write!(f, "tenant TLS status is not ready"), + Self::TenantTlsClientCertificateRequired => { + write!(f, "tenant TLS requires a client certificate") + } + Self::MissingTenantTlsCaKey { secret, key } => { + write!(f, "tenant TLS CA secret {secret} missing key {key}") + } + Self::TenantTlsCaSecretLookupFailed { secret } => { + write!(f, "failed to load tenant TLS CA secret {secret}") + } + Self::InvalidTenantTlsCa => write!(f, "tenant TLS CA is not a valid PEM bundle"), + Self::TlsClientBuildFailed => write!(f, "failed to build TLS HTTP client"), + Self::RequestBuildFailed => write!(f, "failed to construct request"), + Self::RequestFailed => write!(f, "request failed"), + Self::UnexpectedStatus { status, detail } => { + write!(f, "upstream returned {status}")?; + if let Some(detail) = detail { + write!(f, ": {detail}")?; + } + Ok(()) + } + Self::ParseResponseFailed => write!(f, "failed to parse AssumeRole response"), + Self::SigningFailed => write!(f, "failed to compute request signature"), + } + } +} + +impl std::error::Error for RustfsClientError {} + +impl RustfsClientError { + pub(crate) async fn unexpected_response(response: Response) -> Self { + let status = response.status(); + let (body, truncated) = read_limited_response_body(response).await; + Self::unexpected_status_with_limited_body(status, &body, truncated) + } + + pub(crate) async fn limited_response_body(response: Response) -> (String, bool) { + read_limited_response_body(response).await + } + + pub(crate) fn unexpected_status_with_limited_body( + status: StatusCode, + body: &str, + body_truncated: bool, + ) -> Self { + Self::UnexpectedStatus { + status, + detail: summarize_upstream_error_body(body, body_truncated), + } + } + + #[cfg(test)] + pub(crate) fn unexpected_status_with_body(status: StatusCode, body: &str) -> Self { + Self::unexpected_status_with_limited_body(status, body, false) + } +} + +async fn read_limited_response_body(mut response: Response) -> (String, bool) { + let mut body = Vec::new(); + let read_limit = MAX_UPSTREAM_ERROR_BODY_BYTES.saturating_add(1); + + loop { + let remaining = read_limit.saturating_sub(body.len()); + if remaining == 0 { + break; + } + + let chunk = match response.chunk().await { + Ok(Some(chunk)) => chunk, + Ok(None) => break, + Err(_) => break, + }; + if chunk.len() > remaining { + body.extend_from_slice(&chunk[..remaining]); + break; + } + body.extend_from_slice(&chunk); + } + + let truncated = body.len() > MAX_UPSTREAM_ERROR_BODY_BYTES; + if truncated { + body.truncate(MAX_UPSTREAM_ERROR_BODY_BYTES); + } + + (String::from_utf8_lossy(&body).into_owned(), truncated) +} + +fn summarize_upstream_error_body(body: &str, body_truncated: bool) -> Option { + let body = body.trim(); + if body.is_empty() { + return None; + } + + if let Some(message) = crate::helpers::extract_xml_tag(body, "Message") { + let message = decode_basic_xml_entities(&message); + let detail = match crate::helpers::extract_xml_tag(body, "Code") { + Some(code) if !code.trim().is_empty() => { + format!("{}: {message}", decode_basic_xml_entities(&code)) + } + _ => message, + }; + return Some(sanitize_error_detail(&detail)); + } + + if let Ok(value) = serde_json::from_str::(body) + && let Some(detail) = summarize_json_error(&value) + { + return Some(sanitize_error_detail(&detail)); + } + + if body_truncated { + return Some(format!( + "response body exceeded {MAX_UPSTREAM_ERROR_BODY_BYTES} bytes" + )); + } + + Some(sanitize_error_detail(body)) +} + +fn summarize_json_error(value: &serde_json::Value) -> Option { + if let Some(message) = value.as_str() { + return Some(message.to_string()); + } + + let object = value.as_object()?; + let message = ["message", "Message", "error", "Error"] + .iter() + .find_map(|key| object.get(*key).and_then(serde_json::Value::as_str))?; + let code = ["code", "Code"] + .iter() + .find_map(|key| object.get(*key).and_then(serde_json::Value::as_str)); + + Some(match code { + Some(code) if !code.trim().is_empty() => format!("{code}: {message}"), + _ => message.to_string(), + }) +} + +fn collapse_whitespace(value: &str) -> String { + value.split_whitespace().collect::>().join(" ") +} + +fn sanitize_error_detail(value: &str) -> String { + let detail = collapse_whitespace(value); + let detail = redact_sensitive_pairs(&detail); + truncate_error_detail(detail) +} + +fn truncate_error_detail(value: String) -> String { + let mut truncated = String::new(); + for (index, ch) in value.chars().enumerate() { + if index >= MAX_UPSTREAM_ERROR_DETAIL_CHARS { + truncated.push_str("..."); + return truncated; + } + truncated.push(ch); + } + truncated +} + +fn decode_basic_xml_entities(value: &str) -> String { + value + .replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'") + .replace("&", "&") +} + +#[derive(Debug)] +pub(crate) struct SignedRequest { + pub(crate) amz_date: String, + pub(crate) payload_hash: String, + pub(crate) authorization: String, +} + +/// RustFS admin/S3/STS client. +pub struct RustfsAdminClient { + pub(crate) base_url: String, + pub(crate) access_key: String, + pub(crate) secret_key: String, + pub(crate) region: String, + pub(crate) http_client: HttpClient, +} + +pub(crate) fn default_http_client() -> HttpClient { + HttpClient::builder() + .connect_timeout(ADMIN_HTTP_CONNECT_TIMEOUT) + .timeout(ADMIN_HTTP_REQUEST_TIMEOUT) + .build() + .unwrap_or_else(|_| HttpClient::new()) +} + +impl RustfsAdminClient { + pub const STS_VERSION: &'static str = "2011-06-15"; + pub const STS_ACTION: &'static str = "AssumeRole"; + + pub fn new_with_base_url( + base_url: impl Into, + access_key: impl Into, + secret_key: impl Into, + ) -> Self { + Self::new_with_base_url_and_http_client( + base_url, + access_key, + secret_key, + default_http_client(), + ) + } + + pub fn new_with_base_url_and_ca_pem( + base_url: impl Into, + access_key: impl Into, + secret_key: impl Into, + ca_pem: &[u8], + ) -> Result { + let certs = Certificate::from_pem_bundle(ca_pem) + .map_err(|_| RustfsClientError::InvalidTenantTlsCa)?; + let mut builder = HttpClient::builder() + .connect_timeout(ADMIN_HTTP_CONNECT_TIMEOUT) + .timeout(ADMIN_HTTP_REQUEST_TIMEOUT); + for cert in certs { + builder = builder.add_root_certificate(cert); + } + let http_client = builder + .build() + .map_err(|_| RustfsClientError::TlsClientBuildFailed)?; + + Ok(Self::new_with_base_url_and_http_client( + base_url, + access_key, + secret_key, + http_client, + )) + } + + pub fn new_with_base_url_and_http_client( + base_url: impl Into, + access_key: impl Into, + secret_key: impl Into, + http_client: HttpClient, + ) -> Self { + Self { + base_url: base_url.into(), + access_key: access_key.into(), + secret_key: secret_key.into(), + region: "us-east-1".to_string(), + http_client, + } + } +} diff --git a/src/sts/core_ops.rs b/crates/rustfs-admin/src/core_ops.rs similarity index 93% rename from src/sts/core_ops.rs rename to crates/rustfs-admin/src/core_ops.rs index ffdb9ae8..d89f4052 100644 --- a/src/sts/core_ops.rs +++ b/crates/rustfs-admin/src/core_ops.rs @@ -16,11 +16,11 @@ use chrono::Utc; use url::Url; -use super::helpers::{derive_signing_key, hmac_sha256_hex, sha256_hex}; -use super::{ADMIN_SIGNING_SERVICE, RustfsAdminClient, RustfsClientError, SignedRequest}; +use crate::client::{ADMIN_SIGNING_SERVICE, RustfsAdminClient, RustfsClientError, SignedRequest}; +use crate::helpers::{derive_signing_key, hmac_sha256_hex, sha256_hex}; impl RustfsAdminClient { - pub(super) async fn send_admin_request( + pub(crate) async fn send_admin_request( &self, method: &str, path: &str, @@ -49,6 +49,7 @@ impl RustfsAdminClient { "GET" => self.http_client.get(url), "POST" => self.http_client.post(url), "PUT" => self.http_client.put(url), + "DELETE" => self.http_client.delete(url), _ => return Err(RustfsClientError::RequestBuildFailed), } .header("x-amz-date", &signed.amz_date) @@ -82,7 +83,7 @@ impl RustfsAdminClient { .map_err(|_| RustfsClientError::RequestFailed) } - pub(super) fn sign_request( + pub(crate) fn sign_request( &self, method: &str, path: &str, @@ -104,7 +105,7 @@ impl RustfsAdminClient { ) } - pub(super) fn sign_request_with_extra_headers( + pub(crate) fn sign_request_with_extra_headers( &self, method: &str, path: &str, @@ -164,7 +165,7 @@ impl RustfsAdminClient { }) } - pub(super) fn host(&self) -> Result { + pub(crate) fn host(&self) -> Result { let parsed = Url::parse(&self.base_url).map_err(|_| RustfsClientError::RequestBuildFailed)?; let mut host = parsed diff --git a/crates/rustfs-admin/src/credentials.rs b/crates/rustfs-admin/src/credentials.rs new file mode 100644 index 00000000..800938a8 --- /dev/null +++ b/crates/rustfs-admin/src/credentials.rs @@ -0,0 +1,23 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Temporary credentials returned by the RustFS STS `AssumeRole` API. + +#[derive(Debug, Clone)] +pub struct StsAssumeRoleCredentials { + pub access_key_id: String, + pub secret_access_key: String, + pub session_token: String, + pub expiration: String, +} diff --git a/crates/rustfs-admin/src/helpers.rs b/crates/rustfs-admin/src/helpers.rs new file mode 100644 index 00000000..1b239c83 --- /dev/null +++ b/crates/rustfs-admin/src/helpers.rs @@ -0,0 +1,185 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Internal helper duties: signature/hash utilities and wire-format parsers. +use hmac::{Hmac, Mac}; +use reqwest::StatusCode; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use url::form_urlencoded; + +use crate::client::RustfsClientError; +use crate::credentials::StsAssumeRoleCredentials; + +/// Encode an `application/x-www-form-urlencoded` request body. +pub(crate) fn build_form_body(params: &[(&str, &str)]) -> String { + let mut pairs: Vec<(String, String)> = params + .iter() + .map(|(k, v)| ((*k).to_string(), (*v).to_string())) + .collect(); + pairs.sort_by(|(k1, v1), (k2, v2)| k1.cmp(k2).then(v1.cmp(v2))); + + let mut serializer = form_urlencoded::Serializer::new(String::new()); + for (key, value) in pairs { + serializer.append_pair(&key, &value); + } + + serializer.finish() +} + +/// Encode and sort query parameters according to the AWS SigV4 rules. +pub(crate) fn build_canonical_query(params: &[(&str, &str)]) -> String { + let mut pairs: Vec<(String, String)> = params + .iter() + .map(|(key, value)| (uri_encode(key), uri_encode(value))) + .collect(); + pairs.sort_unstable(); + + pairs + .into_iter() + .map(|(key, value)| format!("{key}={value}")) + .collect::>() + .join("&") +} + +fn uri_encode(value: &str) -> String { + const HEX: &[u8; 16] = b"0123456789ABCDEF"; + + let mut encoded = String::with_capacity(value.len()); + for byte in value.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + encoded.push(char::from(byte)); + } else { + encoded.push('%'); + encoded.push(char::from(HEX[usize::from(byte >> 4)])); + encoded.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + } + encoded +} + +pub(crate) fn create_bucket_body(region: Option<&str>) -> String { + let Some(region) = region.map(str::trim).filter(|region| !region.is_empty()) else { + return String::new(); + }; + + if region == "us-east-1" { + return String::new(); + } + + format!( + "{}", + escape_xml(region) + ) +} + +pub(crate) fn escape_xml(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} + +pub(crate) fn body_mentions_not_found(body: &str) -> bool { + let body = body.to_ascii_lowercase(); + body.contains("nosuchuser") + || body.contains("no such user") + || body.contains("user not exist") + || body.contains("nosuchpolicy") + || body.contains("no such policy") + || body.contains("objectlockconfigurationnotfound") + || body.contains("not found") +} + +/// Whether the response body indicates the target bucket does not exist. +pub(crate) fn bucket_not_found(body: &str) -> bool { + let body = body.to_ascii_lowercase(); + body.contains("nosuchbucket") || body.contains("no such bucket") || body.contains("not found") +} + +pub(crate) fn bucket_already_exists(status: StatusCode, body: &str) -> bool { + if status == StatusCode::CONFLICT { + let body = body.to_ascii_lowercase(); + return body.contains("bucketalreadyexists") || body.contains("bucketalreadyownedbyyou"); + } + + false +} + +pub(crate) fn extract_canned_policy_document(body: &str) -> Result { + let value = serde_json::from_str::(body) + .map_err(|_| RustfsClientError::InvalidPolicyDocument)?; + let policy = value.get("policy").unwrap_or(&value); + + serde_json::to_string(policy).map_err(|_| RustfsClientError::InvalidPolicyDocument) +} + +pub(crate) fn sha256_hex(payload: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(payload); + hex::encode(hasher.finalize()) +} + +pub(crate) fn hmac_sha256(key: &[u8], message: &str) -> Result, RustfsClientError> { + let mut mac = + Hmac::::new_from_slice(key).map_err(|_| RustfsClientError::SigningFailed)?; + mac.update(message.as_bytes()); + Ok(mac.finalize().into_bytes().to_vec()) +} + +pub(crate) fn hmac_sha256_hex(key: &[u8], message: &str) -> Result { + let bytes = hmac_sha256(key, message)?; + Ok(hex::encode(bytes)) +} + +pub(crate) fn derive_signing_key( + secret_key: &str, + date_stamp: &str, + region: &str, + service: &str, +) -> Result, RustfsClientError> { + let k_secret = format!("AWS4{secret_key}").into_bytes(); + let k_date = hmac_sha256(&k_secret, date_stamp)?; + let k_region = hmac_sha256(&k_date, region)?; + let k_service = hmac_sha256(&k_region, service)?; + hmac_sha256(&k_service, "aws4_request") +} + +pub(crate) fn parse_assume_role_response(body: &str) -> Option { + let access_key_id = extract_xml_tag(body, "AccessKeyId")?; + let secret_access_key = extract_xml_tag(body, "SecretAccessKey")?; + let session_token = extract_xml_tag(body, "SessionToken")?; + let expiration = extract_xml_tag(body, "Expiration")?; + + Some(StsAssumeRoleCredentials { + access_key_id, + secret_access_key, + session_token, + expiration, + }) +} + +pub(crate) fn extract_xml_tag(document: &str, tag: &str) -> Option { + let open = format!("<{tag}>"); + let close = format!(""); + + let open_idx = document.find(&open)?; + let start = open_idx + open.len(); + let rest = &document[start..]; + let end = rest.find(&close)?; + + Some(rest[..end].trim().to_string()) +} diff --git a/crates/rustfs-admin/src/lib.rs b/crates/rustfs-admin/src/lib.rs new file mode 100644 index 00000000..0759a5b3 --- /dev/null +++ b/crates/rustfs-admin/src/lib.rs @@ -0,0 +1,42 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Kube-agnostic RustFS admin/S3/STS client. +//! +//! This crate contains the wire-protocol logic (request signing, HTTP +//! dispatch, response parsing) needed to talk to a RustFS server's admin, +//! S3 and STS APIs. It has no dependency on `kube` or `Tenant` types; +//! kube/Tenant-aware wrappers live in the `operator` crate's +//! `sts::rustfs_client` module. + +mod admin_ops; +mod client; +mod core_ops; +mod credentials; +mod helpers; +mod pool_ops; +mod s3_ops; +mod sanitize; +mod sts_ops; + +pub use client::{ + CreateBucketResult, RustfsAdminClient, RustfsClientError, RustfsCredentials, + RustfsErasureBackend, RustfsErasureSetInfo, RustfsPoolDecommissionInfo, RustfsPoolListItem, + RustfsPoolStatus, RustfsServerInfo, RustfsServerUsage, +}; +pub use credentials::StsAssumeRoleCredentials; + +#[cfg(test)] +#[path = "tests.rs"] +mod tests; diff --git a/src/sts/pool_ops.rs b/crates/rustfs-admin/src/pool_ops.rs similarity index 97% rename from src/sts/pool_ops.rs rename to crates/rustfs-admin/src/pool_ops.rs index a5322ee9..c5e84452 100644 --- a/src/sts/pool_ops.rs +++ b/crates/rustfs-admin/src/pool_ops.rs @@ -15,11 +15,11 @@ //! Pool boundary: //! - list/status and decommission lifecycle operations for tenant pools. -use super::helpers::build_canonical_query; -use super::{ +use crate::client::{ POOLS_CANCEL_PATH, POOLS_DECOMMISSION_PATH, POOLS_LIST_PATH, POOLS_STATUS_PATH, RustfsAdminClient, RustfsClientError, RustfsPoolListItem, RustfsPoolStatus, }; +use crate::helpers::build_canonical_query; impl RustfsAdminClient { // Pool duties: list/status and decommission lifecycle operations. diff --git a/src/sts/s3_ops.rs b/crates/rustfs-admin/src/s3_ops.rs similarity index 73% rename from src/sts/s3_ops.rs rename to crates/rustfs-admin/src/s3_ops.rs index 0df217fc..6fca0a02 100644 --- a/src/sts/s3_ops.rs +++ b/crates/rustfs-admin/src/s3_ops.rs @@ -13,15 +13,18 @@ // limitations under the License. //! S3 boundary: -//! - bucket lifecycle methods (create/lookup features) +//! - bucket lifecycle methods (create/lookup/delete) //! - request semantics for S3-style object storage operations. use reqwest::StatusCode; -use super::helpers::{ - body_mentions_not_found, bucket_already_exists, build_canonical_query, create_bucket_body, +use crate::client::{ + ADMIN_SIGNING_SERVICE, CreateBucketResult, RustfsAdminClient, RustfsClientError, +}; +use crate::helpers::{ + body_mentions_not_found, bucket_already_exists, bucket_not_found, build_canonical_query, + create_bucket_body, }; -use super::{ADMIN_SIGNING_SERVICE, CreateBucketResult, RustfsAdminClient, RustfsClientError}; impl RustfsAdminClient { // S3 duties: bucket operations exposed by the RustFS/S3-compatible endpoint. @@ -91,6 +94,42 @@ impl RustfsAdminClient { )) } + /// Delete a bucket. Succeeds if the bucket is already gone. + pub async fn delete_bucket(&self, bucket: &str) -> Result<(), RustfsClientError> { + if bucket.trim().is_empty() { + return Err(RustfsClientError::RequestBuildFailed); + } + + let path = format!("/{bucket}"); + let signed = self.sign_request("DELETE", &path, "", "", None, ADMIN_SIGNING_SERVICE)?; + let host = self.host()?; + + let response = self + .http_client + .delete(format!("{}{}", self.base_url.trim_end_matches('/'), path)) + .header("x-amz-date", &signed.amz_date) + .header("x-amz-content-sha256", &signed.payload_hash) + .header("authorization", &signed.authorization) + .header("host", host) + .send() + .await + .map_err(|_| RustfsClientError::RequestFailed)?; + + if response.status().is_success() { + return Ok(()); + } + + let status = response.status(); + let (body, truncated) = RustfsClientError::limited_response_body(response).await; + if status == StatusCode::NOT_FOUND || bucket_not_found(&body) { + return Ok(()); + } + + Err(RustfsClientError::unexpected_status_with_limited_body( + status, &body, truncated, + )) + } + pub async fn bucket_object_lock_enabled( &self, bucket: &str, diff --git a/crates/rustfs-admin/src/sanitize.rs b/crates/rustfs-admin/src/sanitize.rs new file mode 100644 index 00000000..c18cd372 --- /dev/null +++ b/crates/rustfs-admin/src/sanitize.rs @@ -0,0 +1,353 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Redact sensitive key/value pairs and XML tags from upstream error bodies +//! before they are surfaced in `RustfsClientError` messages. + +const SENSITIVE_KEYS: [&str; 22] = [ + "token", + "password", + "accesskey", + "access_key", + "access-key", + "accesskeyid", + "access_key_id", + "access-key-id", + "secretkey", + "secret_key", + "secret-key", + "secretaccesskey", + "secret_access_key", + "secret-access-key", + "clientsecret", + "client_secret", + "client-secret", + "sessiontoken", + "session_token", + "session-token", + "credential", + "credentials", +]; + +pub(crate) fn redact_sensitive_pairs(message: &str) -> String { + let message = redact_sensitive_xml_tags(message); + redact_sensitive_key_value_pairs(&message) +} + +fn is_sensitive_key(key: &str) -> bool { + matches!( + normalize_key(key).as_str(), + "token" + | "password" + | "accesskey" + | "accesskeyid" + | "secretkey" + | "secretaccesskey" + | "clientsecret" + | "sessiontoken" + | "credential" + | "credentials" + ) +} + +fn normalize_key(raw: &str) -> String { + raw.trim_matches(|ch: char| !ch.is_ascii_alphanumeric() && ch != '-' && ch != '_') + .chars() + .filter(|ch| ch.is_ascii_alphanumeric()) + .collect::() + .to_ascii_lowercase() +} + +fn redact_sensitive_xml_tags(message: &str) -> String { + let mut output = String::with_capacity(message.len()); + let mut cursor = 0usize; + + while cursor < message.len() { + let Some(ch) = message[cursor..].chars().next() else { + break; + }; + + if ch == '<' + && let Some(replacement) = redact_xml_tag_at(message, cursor) + { + output.push_str(&replacement.redacted); + cursor = replacement.end; + continue; + } + + output.push(ch); + cursor += ch.len_utf8(); + } + + output +} + +struct XmlRedaction { + redacted: String, + end: usize, +} + +fn redact_xml_tag_at(message: &str, cursor: usize) -> Option { + let tag_end = cursor + message[cursor..].find('>')?; + let tag_content = &message[cursor + 1..tag_end]; + if tag_content.starts_with('/') || tag_content.starts_with('?') || tag_content.starts_with('!') + { + return None; + } + let tag_name_end = tag_content + .find(|ch: char| ch.is_whitespace() || ch == '/') + .unwrap_or(tag_content.len()); + let tag_name = &tag_content[..tag_name_end]; + if tag_name.is_empty() || !is_sensitive_key(tag_name) { + return None; + } + + let open_end = tag_end + 1; + let close = format!(""); + let close_start = open_end + message[open_end..].find(&close)?; + let close_end = close_start + close.len(); + + Some(XmlRedaction { + redacted: format!( + "{}{}", + &message[cursor..open_end], + &message[close_start..close_end] + ), + end: close_end, + }) +} + +fn redact_sensitive_key_value_pairs(message: &str) -> String { + let bytes = message.as_bytes(); + let mut output = String::with_capacity(message.len()); + let mut cursor = 0usize; + + while cursor < bytes.len() { + let mut matched = false; + + for key in SENSITIVE_KEYS { + let key_len = key.len(); + + let unquoted_match = matches_key_at(message, cursor, key); + let quoted_match = cursor + key_len + 2 <= bytes.len() + && matches!(bytes[cursor] as char, '"' | '\'') + && bytes[cursor + key_len + 1] == bytes[cursor] + && matches_key_at(message, cursor + 1, key); + + let (key_start, key_end, cursor_after_key) = if unquoted_match { + if cursor > 0 { + let prev = bytes[cursor - 1] as char; + if prev.is_ascii_alphanumeric() || prev == '_' || prev == '-' { + continue; + } + } + (cursor, cursor + key_len, cursor + key_len) + } else if quoted_match { + let key_start = cursor + 1; + (key_start, key_start + key_len, key_start + key_len + 1) + } else { + continue; + }; + + let candidate = &message[key_start..key_end]; + + let sep_index = skip_whitespace(message, cursor_after_key); + if sep_index >= bytes.len() || !matches!(bytes[sep_index] as char, '=' | ':') { + continue; + } + + let value_start = skip_whitespace(message, sep_index + 1); + let value_end = parse_value_end(message, value_start); + if value_end <= value_start || !is_sensitive_key(candidate) { + continue; + } + + output.push_str(&message[cursor..value_start]); + output.push_str(&redacted_value(&message[value_start..value_end])); + cursor = value_end; + matched = true; + break; + } + + if !matched { + let Some(ch) = message[cursor..].chars().next() else { + break; + }; + output.push(ch); + cursor += ch.len_utf8(); + } + } + + output +} + +fn parse_value_end(input: &str, start: usize) -> usize { + if start >= input.len() { + return start; + } + + let mut chars = input[start..].char_indices(); + let Some((_, first)) = chars.next() else { + return start; + }; + if first == '"' || first == '\'' { + let mut previous = first; + for (offset, ch) in chars { + if ch == first && previous != '\\' { + return start + offset + ch.len_utf8(); + } + previous = ch; + } + return input.len(); + } + + for (offset, ch) in input[start..].char_indices() { + if ch.is_whitespace() || matches!(ch, ',' | ';' | '}' | ']' | ')') { + return start + offset; + } + } + input.len() +} + +fn skip_whitespace(input: &str, start: usize) -> usize { + for (offset, ch) in input[start..].char_indices() { + if !ch.is_whitespace() { + return start + offset; + } + } + input.len() +} + +fn matches_key_at(message: &str, start: usize, key: &str) -> bool { + let end = start + key.len(); + end <= message.len() + && message.is_char_boundary(start) + && message.is_char_boundary(end) + && message[start..end].eq_ignore_ascii_case(key) +} + +fn redacted_value(original: &str) -> String { + if original.len() >= 2 { + let bytes = original.as_bytes(); + let first = bytes[0]; + let last = bytes[bytes.len() - 1]; + if (first == b'"' && last == b'"') || (first == b'\'' && last == b'\'') { + let quote = first as char; + return format!("{quote}{quote}"); + } + } + "".to_string() +} + +#[cfg(test)] +mod tests { + use super::redact_sensitive_pairs; + + #[test] + fn preserves_required_key_names() { + let message = "Vault backend requires kmsSecret referencing a Secret with key vault-token"; + + assert_eq!(redact_sensitive_pairs(message), message); + } + + #[test] + fn redacts_colon_and_json_secret_values() { + let message = + "kms config token: tok_123 password: p@ss accesskey: AKIA_TEST secretkey: SK_TEST"; + + let sanitized = redact_sensitive_pairs(message); + + assert!(sanitized.contains("token")); + assert!(sanitized.contains("password")); + assert!(sanitized.contains("accesskey")); + assert!(sanitized.contains("secretkey")); + assert!(!sanitized.contains("tok_123")); + assert!(!sanitized.contains("p@ss")); + assert!(!sanitized.contains("AKIA_TEST")); + assert!(!sanitized.contains("SK_TEST")); + } + + #[test] + fn redacts_key_name_variants_and_xml_tags() { + let message = + r#"clientSecret: oidc-secret {"access_key":"AKIA_JSON"} SK_XML"#; + + let sanitized = redact_sensitive_pairs(message); + + assert!(sanitized.contains("clientSecret: ")); + assert!(sanitized.contains(r#""access_key":"""#)); + assert!(sanitized.contains("")); + assert!(!sanitized.contains("oidc-secret")); + assert!(!sanitized.contains("AKIA_JSON")); + assert!(!sanitized.contains("SK_XML")); + } + + #[test] + fn redacts_sts_credential_field_names() { + let message = r#"AccessKeyId: AKIA_TEXT SecretAccessKey: SK_TEXT {"access_key_id":"AKIA_JSON","secret-access-key":"SK_JSON"} AKIA_XML SK_XML"#; + + let sanitized = redact_sensitive_pairs(message); + + assert!(sanitized.contains("AccessKeyId: ")); + assert!(sanitized.contains("SecretAccessKey: ")); + assert!(sanitized.contains(r#""access_key_id":"""#)); + assert!(sanitized.contains(r#""secret-access-key":"""#)); + assert!(sanitized.contains("")); + assert!(sanitized.contains("")); + assert!(!sanitized.contains("AKIA_TEXT")); + assert!(!sanitized.contains("SK_TEXT")); + assert!(!sanitized.contains("AKIA_JSON")); + assert!(!sanitized.contains("SK_JSON")); + assert!(!sanitized.contains("AKIA_XML")); + assert!(!sanitized.contains("SK_XML")); + } + + #[test] + fn handles_unicode_without_panicking() { + let message = "错误🔐 token: tok_123 用户=测试 secretkey: SK_TEST 完成"; + + let sanitized = redact_sensitive_pairs(message); + + assert!(sanitized.contains("错误🔐")); + assert!(sanitized.contains("用户=测试")); + assert!(sanitized.contains("完成")); + assert!(sanitized.contains("token: ")); + assert!(sanitized.contains("secretkey: ")); + assert!(!sanitized.contains("tok_123")); + assert!(!sanitized.contains("SK_TEST")); + } + + #[test] + fn redacts_unicode_quoted_values() { + let message = "{\"说明\":\"🔐\",\"secretkey\":\"秘密值\"}"; + + let sanitized = redact_sensitive_pairs(message); + + assert!(sanitized.contains("\"说明\":\"🔐\"")); + assert!(sanitized.contains("\"secretkey\":\"\"")); + assert!(!sanitized.contains("秘密值")); + } + + #[test] + fn redacts_after_unicode_whitespace() { + let message = "token:\u{3000}tok_123 secretkey:\u{2003}SK_TEST"; + + let sanitized = redact_sensitive_pairs(message); + + assert!(sanitized.contains("token:\u{3000}")); + assert!(sanitized.contains("secretkey:\u{2003}")); + assert!(!sanitized.contains("tok_123")); + assert!(!sanitized.contains("SK_TEST")); + } +} diff --git a/src/sts/sts_ops.rs b/crates/rustfs-admin/src/sts_ops.rs similarity index 95% rename from src/sts/sts_ops.rs rename to crates/rustfs-admin/src/sts_ops.rs index ba45bccb..2eff08c4 100644 --- a/src/sts/sts_ops.rs +++ b/crates/rustfs-admin/src/sts_ops.rs @@ -14,11 +14,12 @@ //! STS boundary: //! - temporary credentials and AssumeRole request composition/response parsing. -use super::helpers::{build_form_body, parse_assume_role_response}; -use super::{ + +use crate::client::{ ASSUME_ROLE_PATH, FORM_CONTENT_TYPE, RustfsAdminClient, RustfsClientError, STS_SIGNING_SERVICE, }; -use crate::sts::types::StsAssumeRoleCredentials; +use crate::credentials::StsAssumeRoleCredentials; +use crate::helpers::{build_form_body, parse_assume_role_response}; impl RustfsAdminClient { // STS duties: temporary credentials and AssumeRole API call path. diff --git a/crates/rustfs-admin/src/tests.rs b/crates/rustfs-admin/src/tests.rs new file mode 100644 index 00000000..520ea7aa --- /dev/null +++ b/crates/rustfs-admin/src/tests.rs @@ -0,0 +1,1192 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Unit/integration tests for RustfsAdminClient split operation modules. + +use axum::{ + Router, + body::Body, + extract::State, + http::{Request, StatusCode}, + routing::{get, post, put}, +}; +use serde_json::Value; +use std::sync::Arc; +use tokio::sync::Mutex; + +use crate::client::{ + ADD_USER_PATH, ADMIN_SIGNING_SERVICE, CreateBucketResult, FORM_CONTENT_TYPE, JSON_CONTENT_TYPE, + LIST_CANNED_POLICIES_PATH, MAX_UPSTREAM_ERROR_BODY_BYTES, POOLS_DECOMMISSION_PATH, + POOLS_LIST_PATH, POOLS_STATUS_PATH, RustfsAdminClient, RustfsClientError, SERVER_INFO_PATH, + SET_POLICY_PATH, STS_SIGNING_SERVICE, USER_INFO_PATH, +}; +use crate::helpers::{ + build_canonical_query, build_form_body, derive_signing_key, extract_canned_policy_document, + hmac_sha256_hex, parse_assume_role_response, sha256_hex, +}; + +const TEST_ACCESS_KEY: &str = "access"; +const TEST_SECRET_KEY: &str = "secret"; +const TEST_REGION: &str = "us-east-1"; + +#[test] +fn canonical_query_uses_sigv4_uri_encoding_and_encoded_sort_order() { + let query = build_canonical_query(&[ + ("z", "a b~c/雪"), + ("a~", "second"), + ("a ", "first"), + ("amp", "&="), + ("dup", "z"), + ("dup", "a"), + ("empty", ""), + ("雪", "key"), + ]); + + assert_eq!( + query, + "%E9%9B%AA=key&a%20=first&=%26%3D&a~=second&dup=a&dup=z&empty=&z=a%20b~c%2F%E9%9B%AA" + ); +} + +#[test] +fn form_body_keeps_html_form_encoding() { + assert_eq!( + build_form_body(&[("Policy", "a b~c/雪")]), + "Policy=a+b%7Ec%2F%E9%9B%AA" + ); +} + +#[test] +fn duplicate_query_values_match_independent_sigv4_verification() { + let query = + build_canonical_query(&[("dup", "z z"), ("dup", "a+a"), ("dup", "雪"), ("empty", "")]); + assert_eq!(query, "dup=%E9%9B%AA&dup=a%2Ba&dup=z%20z&empty="); + + let client = RustfsAdminClient::new_with_base_url( + "https://rustfs.example.test:9000", + TEST_ACCESS_KEY, + TEST_SECRET_KEY, + ); + let signed = client + .sign_request("GET", "/synthetic", &query, "", None, ADMIN_SIGNING_SERVICE) + .unwrap(); + let request = CapturedRequest { + method: "GET".to_string(), + path: "/synthetic".to_string(), + query, + body: String::new(), + host: "rustfs.example.test:9000".to_string(), + content_type: String::new(), + amz_date: signed.amz_date, + payload_hash: signed.payload_hash, + authorization: signed.authorization, + }; + + assert_sigv4_matches_wire(&request, ADMIN_SIGNING_SERVICE); +} + +fn assert_oversized_upstream_body_hidden(err: RustfsClientError) { + assert_eq!( + err.to_string(), + format!( + "upstream returned 502 Bad Gateway: response body exceeded {MAX_UPSTREAM_ERROR_BODY_BYTES} bytes" + ) + ); +} + +#[test] +fn parse_assume_role_xml_success_and_failure() { + let body_ok = "AKISECTOKEN2026-01-01T00:00:00Z"; + let parsed = + parse_assume_role_response(body_ok).expect("valid assume role response should parse"); + + assert_eq!(parsed.access_key_id, "AKI"); + assert_eq!(parsed.secret_access_key, "SEC"); + assert_eq!(parsed.session_token, "TOKEN"); + assert_eq!(parsed.expiration, "2026-01-01T00:00:00Z"); + + assert!(parse_assume_role_response("").is_none()); +} + +#[test] +fn unexpected_status_includes_upstream_xml_error_summary() { + let err = RustfsClientError::unexpected_status_with_body( + StatusCode::BAD_REQUEST, + r#"InvalidRequestinvalid resource: unknown "*"abc"#, + ); + + let message = err.to_string(); + assert_eq!( + message, + r#"upstream returned 400 Bad Request: InvalidRequest: invalid resource: unknown "*""# + ); + assert!(!message.contains("")); +} + +#[test] +fn unexpected_status_includes_upstream_json_error_summary() { + let err = RustfsClientError::unexpected_status_with_body( + StatusCode::BAD_REQUEST, + r#"{"code":"InvalidRequest","message":"policy Resource must use ARN form"}"#, + ); + + assert_eq!( + err.to_string(), + "upstream returned 400 Bad Request: InvalidRequest: policy Resource must use ARN form" + ); +} + +#[test] +fn unexpected_status_redacts_sensitive_upstream_error_summary() { + let err = RustfsClientError::unexpected_status_with_body( + StatusCode::BAD_REQUEST, + r#"{"code":"InvalidRequest","message":"secretkey: SK_TEST clientSecret: oidc-secret SecretAccessKey: SK_STS AccessKeyId: AKIA_STS SK_XML AKIA_XML"}"#, + ); + + let message = err.to_string(); + assert!(message.contains("secretkey: ")); + assert!(message.contains("clientSecret: ")); + assert!(message.contains("SecretAccessKey: ")); + assert!(message.contains("AccessKeyId: ")); + assert!(message.contains("")); + assert!(message.contains("")); + assert!(!message.contains("SK_TEST")); + assert!(!message.contains("oidc-secret")); + assert!(!message.contains("SK_STS")); + assert!(!message.contains("AKIA_STS")); + assert!(!message.contains("SK_XML")); + assert!(!message.contains("AKIA_XML")); +} + +#[test] +fn unexpected_status_hides_truncated_unstructured_response_body() { + let retained_body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES); + let err = RustfsClientError::unexpected_status_with_limited_body( + StatusCode::BAD_GATEWAY, + &retained_body, + true, + ); + + assert_eq!( + err.to_string(), + format!( + "upstream returned 502 Bad Gateway: response body exceeded {MAX_UPSTREAM_ERROR_BODY_BYTES} bytes" + ) + ); +} + +#[tokio::test] +async fn unexpected_response_preserves_exact_limit_unstructured_response_body() { + let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES); + let router = Router::new().route( + ADD_USER_PATH, + put(move || { + let body = body.clone(); + async move { (StatusCode::BAD_GATEWAY, body) } + }), + ); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let err = client + .add_user("app-user", "secret123") + .await + .expect_err("exact limit body should still report the retained body"); + + let message = err.to_string(); + assert!(message.contains("upstream returned 502 Bad Gateway")); + assert!(!message.contains("response body exceeded")); + + server.abort(); +} + +#[tokio::test] +async fn unexpected_response_hides_over_limit_unstructured_response_body() { + let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES + 1); + let router = Router::new().route( + ADD_USER_PATH, + put(move || { + let body = body.clone(); + async move { (StatusCode::BAD_GATEWAY, body) } + }), + ); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let err = client + .add_user("app-user", "secret123") + .await + .expect_err("oversized body should be hidden"); + + assert_oversized_upstream_body_hidden(err); + + server.abort(); +} + +#[derive(Clone, Default)] +struct Capture { + method: Arc>, + path: Arc>, + query: Arc>, + body: Arc>, + host: Arc>, + content_type: Arc>, + amz_date: Arc>, + payload_hash: Arc>, + authorization: Arc>, + object_lock_header: Arc>, +} + +#[derive(Debug)] +struct CapturedRequest { + method: String, + path: String, + query: String, + body: String, + host: String, + content_type: String, + amz_date: String, + payload_hash: String, + authorization: String, +} + +impl Capture { + async fn request(&self) -> CapturedRequest { + CapturedRequest { + method: self.method.lock().await.clone(), + path: self.path.lock().await.clone(), + query: self.query.lock().await.clone(), + body: self.body.lock().await.clone(), + host: self.host.lock().await.clone(), + content_type: self.content_type.lock().await.clone(), + amz_date: self.amz_date.lock().await.clone(), + payload_hash: self.payload_hash.lock().await.clone(), + authorization: self.authorization.lock().await.clone(), + } + } +} + +fn request_header(req: &Request, name: &str) -> String { + req.headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string() +} + +async fn capture_signed_request(capture: &Capture, req: Request) { + let method = req.method().as_str().to_string(); + let path = req.uri().path().to_string(); + let query = req.uri().query().unwrap_or("").to_string(); + let host = request_header(&req, "host"); + let content_type = request_header(&req, "content-type"); + let amz_date = request_header(&req, "x-amz-date"); + let payload_hash = request_header(&req, "x-amz-content-sha256"); + let authorization = request_header(&req, "authorization"); + let body = axum::body::to_bytes(req.into_body(), usize::MAX) + .await + .unwrap(); + let body = String::from_utf8(body.to_vec()).unwrap(); + + *capture.method.lock().await = method; + *capture.path.lock().await = path; + *capture.query.lock().await = query; + *capture.body.lock().await = body; + *capture.host.lock().await = host; + *capture.content_type.lock().await = content_type; + *capture.amz_date.lock().await = amz_date; + *capture.payload_hash.lock().await = payload_hash; + *capture.authorization.lock().await = authorization; +} + +fn assert_sigv4_matches_wire(request: &CapturedRequest, service: &str) { + let calculated_payload_hash = sha256_hex(request.body.as_bytes()); + assert_eq!(request.payload_hash, calculated_payload_hash); + + let signed_header_names = if request.content_type.is_empty() { + "host;x-amz-content-sha256;x-amz-date" + } else { + "content-type;host;x-amz-content-sha256;x-amz-date" + }; + let mut canonical_headers = String::new(); + if !request.content_type.is_empty() { + canonical_headers.push_str("content-type:"); + canonical_headers.push_str(request.content_type.trim()); + canonical_headers.push('\n'); + } + canonical_headers.push_str("host:"); + canonical_headers.push_str(request.host.trim()); + canonical_headers.push_str("\nx-amz-content-sha256:"); + canonical_headers.push_str(request.payload_hash.trim()); + canonical_headers.push_str("\nx-amz-date:"); + canonical_headers.push_str(request.amz_date.trim()); + canonical_headers.push('\n'); + + let canonical_request = format!( + "{}\n{}\n{}\n{}\n{}\n{}", + request.method, + request.path, + request.query, + canonical_headers, + signed_header_names, + request.payload_hash + ); + let date_stamp = request + .amz_date + .get(..8) + .expect("x-amz-date must start with YYYYMMDD"); + let credential_scope = format!("{date_stamp}/{TEST_REGION}/{service}/aws4_request"); + let string_to_sign = format!( + "AWS4-HMAC-SHA256\n{}\n{}\n{}", + request.amz_date, + credential_scope, + sha256_hex(canonical_request.as_bytes()) + ); + let signing_key = + derive_signing_key(TEST_SECRET_KEY, date_stamp, TEST_REGION, service).unwrap(); + let signature = hmac_sha256_hex(&signing_key, &string_to_sign).unwrap(); + let expected_authorization = format!( + "AWS4-HMAC-SHA256 Credential={TEST_ACCESS_KEY}/{credential_scope}, SignedHeaders={signed_header_names}, Signature={signature}" + ); + + assert_eq!(request.authorization, expected_authorization); +} + +#[tokio::test] +async fn assume_role_request_targets_root_path_and_action_is_assume_role() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new().route( + "/", + post( + move |State(c): State, req: Request| async move { + capture_signed_request(&c, req).await; + + let response = + "AKISECTOKEN2026-01-01T00:00:00Z"; + (StatusCode::OK, response) + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + + let creds = client + .assume_role(Some(r#"{"Statement":[{"Resource":"a b~+/雪"}]}"#), 3600) + .await + .unwrap(); + assert_eq!(creds.access_key_id, "AKI"); + + let request = capture.request().await; + assert_eq!(request.path, "/"); + assert_eq!( + request.body, + "Action=AssumeRole&DurationSeconds=3600&Policy=%7B%22Statement%22%3A%5B%7B%22Resource%22%3A%22a+b%7E%2B%2F%E9%9B%AA%22%7D%5D%7D&Version=2011-06-15" + ); + assert!(request.query.is_empty()); + assert_eq!(request.content_type, FORM_CONTENT_TYPE); + assert_sigv4_matches_wire(&request, STS_SIGNING_SERVICE); + + server.abort(); +} + +#[tokio::test] +async fn info_canned_policy_uses_expected_path_and_query() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + "/rustfs/admin/v3/info-canned-policy", + get( + move |State(c): State, req: Request| async move { + let path = req.uri().path().to_string(); + let query = req.uri().query().unwrap_or("").to_string(); + let authorization = req + .headers() + .get("authorization") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + + *c.path.lock().await = path; + *c.query.lock().await = query; + *c.authorization.lock().await = authorization; + + ( + StatusCode::OK, + "{\"policy_name\":\"tenant-policy\",\"policy\":{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"allow\",\"Effect\":\"Allow\"}]}}", + ) + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + + let policy = client.get_canned_policy("tenant-policy").await.unwrap(); + let policy_value = serde_json::from_str::(&policy).unwrap(); + assert_eq!(policy_value["Version"], "2012-10-17"); + assert_eq!(policy_value["Statement"][0]["Sid"], "allow"); + + assert_eq!( + &*capture.path.lock().await, + "/rustfs/admin/v3/info-canned-policy" + ); + assert!(capture.query.lock().await.contains("name=tenant-policy")); + assert!( + capture + .authorization + .lock() + .await + .contains("/s3/aws4_request") + ); + + server.abort(); +} + +#[tokio::test] +async fn list_canned_policies_extracts_policy_document_and_canonicalizes_json() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + LIST_CANNED_POLICIES_PATH, + get( + move |State(c): State, req: Request| async move { + let path = req.uri().path().to_string(); + let query = req.uri().query().unwrap_or("").to_string(); + + *c.path.lock().await = path; + *c.query.lock().await = query; + + ( + StatusCode::OK, + serde_json::json!({ + "tenant-policy": { + "policy_name":"tenant-policy", + "policy":{ + "Statement": [{ + "Resource": "arn:aws:s3:::tenant", + "Effect": "Allow", + "Action": "s3:GetObject" + }], + "Version":"2012-10-17" + } + }, + "inline-policy": { + "Version": "2012-10-17", + "Statement": [{ + "Sid": "inline", + "Action": "s3:ListBucket", + "Effect": "Allow", + "Resource": ["arn:aws:s3:::tenant*"] + }] + } + }) + .to_string(), + ) + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let policies = client.list_canned_policies().await.unwrap(); + + let tenant_policy = serde_json::from_str::(&policies["tenant-policy"]).unwrap(); + assert_eq!(tenant_policy["Version"], "2012-10-17"); + assert_eq!(tenant_policy["Statement"][0]["Action"], "s3:GetObject"); + + let inline_policy = serde_json::from_str::(&policies["inline-policy"]).unwrap(); + assert_eq!(inline_policy["Version"], "2012-10-17"); + assert_eq!(inline_policy["Statement"][0]["Sid"], "inline"); + assert_eq!(&*capture.path.lock().await, LIST_CANNED_POLICIES_PATH); + assert!(capture.query.lock().await.is_empty()); + + server.abort(); +} + +#[tokio::test] +async fn add_canned_policy_uses_expected_path_query_body_and_admin_signing() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + "/rustfs/admin/v3/add-canned-policy", + put( + move |State(c): State, req: Request| async move { + let path = req.uri().path().to_string(); + let query = req.uri().query().unwrap_or("").to_string(); + let authorization = req + .headers() + .get("authorization") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + let body_bytes = axum::body::to_bytes(req.into_body(), usize::MAX) + .await + .unwrap(); + let body = String::from_utf8(body_bytes.to_vec()).unwrap(); + + *c.path.lock().await = path; + *c.query.lock().await = query; + *c.authorization.lock().await = authorization; + *c.body.lock().await = body; + + StatusCode::OK + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let policy = r#"{"Version":"2012-10-17","Statement":[]}"#; + + client + .add_canned_policy("tenant-policy", policy) + .await + .unwrap(); + + assert_eq!( + &*capture.path.lock().await, + "/rustfs/admin/v3/add-canned-policy" + ); + assert!(capture.query.lock().await.contains("name=tenant-policy")); + assert_eq!(&*capture.body.lock().await, policy); + assert!( + capture + .authorization + .lock() + .await + .contains("/s3/aws4_request") + ); + + server.abort(); +} + +#[tokio::test] +async fn add_canned_policy_reports_upstream_policy_parse_error() { + let router = Router::new().route( + "/rustfs/admin/v3/add-canned-policy", + put(|| async { + ( + StatusCode::BAD_REQUEST, + r#"InvalidRequestinvalid resource: unknown "*""#, + ) + }), + ); + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let policy = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}"#; + let err = client + .add_canned_policy("tenant-policy", policy) + .await + .expect_err("invalid RustFS policy should include upstream parse details"); + + let message = err.to_string(); + assert!(message.contains("upstream returned 400 Bad Request")); + assert!(message.contains(r#"InvalidRequest: invalid resource: unknown "*""#)); + assert!(!message.contains("")); + + server.abort(); +} + +#[tokio::test] +async fn remove_canned_policy_is_idempotent_when_already_gone() { + let router = Router::new().route( + "/rustfs/admin/v3/remove-canned-policy", + axum::routing::delete(|| async { (StatusCode::NOT_FOUND, "NoSuchPolicy") }), + ); + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + client + .remove_canned_policy("tenant-policy") + .await + .expect("removing an already-gone policy should be treated as success"); + + server.abort(); +} + +#[tokio::test] +async fn server_info_uses_expected_path_and_parses_wrapped_health_fields() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + SERVER_INFO_PATH, + get( + move |State(c): State, req: Request| async move { + let path = req.uri().path().to_string(); + let authorization = req + .headers() + .get("authorization") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + + *c.path.lock().await = path; + *c.authorization.lock().await = authorization; + + ( + StatusCode::OK, + serde_json::json!({ + "info": { + "usage": {"size": 42}, + "backend": { + "onlineDisks": 3, + "offlineDisks": 1, + "standardSCParity": 2, + "totalSets": [1], + "totalDrivesPerSet": [4] + }, + "pools": { + "0": { + "0": { + "rawUsage": 100, + "rawCapacity": 400, + "usage": 50, + "objectsCount": 2, + "healDisks": 1 + } + } + } + }, + "admin_discovery": { + "runtimeCapabilities": "/rustfs/admin/v4/runtime/capabilities", + "clusterSnapshot": "/rustfs/admin/v4/cluster/snapshot", + "extensionsCatalog": "/rustfs/admin/v4/extensions/catalog" + }, + }) + .to_string(), + ) + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let info = client.server_info().await.unwrap(); + + let backend = info.backend.unwrap(); + assert_eq!(backend.online_disks, 3); + assert_eq!(backend.offline_disks, 1); + assert_eq!(backend.standard_sc_parity, Some(2)); + assert_eq!(info.usage.unwrap().size, 42); + assert_eq!(info.pools.unwrap()["0"]["0"].raw_capacity, 400); + assert_eq!(&*capture.path.lock().await, SERVER_INFO_PATH); + assert!( + capture + .authorization + .lock() + .await + .contains("/s3/aws4_request") + ); + + server.abort(); +} + +#[tokio::test] +async fn list_pools_parses_current_rustfs_pool_shape() { + let router = Router::new().route( + POOLS_LIST_PATH, + get(|| async { + ( + StatusCode::OK, + r#"[{"id":1,"cmdline":"http://tenant-pool-a-{0...3}.tenant-hl.ns.svc.cluster.local:9000/data/rustfs{0...3}","lastUpdate":"2026-05-20T00:00:00Z","totalSize":100,"currentSize":50,"usedSize":25,"used":25.0,"status":"running","decommissionInfo":{"startTime":"2026-05-20T00:00:00Z","complete":false,"failed":false,"canceled":false,"objectsDecommissioned":7,"objectsDecommissionedFailed":1,"bytesDecommissioned":9,"bytesDecommissionedFailed":2}}]"#, + ) + }), + ); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + + let pools = client.list_pools().await.unwrap(); + + assert_eq!(pools[0].id, 1); + assert_eq!(pools[0].status, "running"); + assert_eq!( + pools[0] + .decommission + .as_ref() + .and_then(|info| info.objects_decommissioned), + Some(7) + ); + + server.abort(); +} + +#[tokio::test] +async fn pool_decommission_start_uses_by_id_query_and_admin_signing() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + POOLS_DECOMMISSION_PATH, + post( + move |State(c): State, req: Request| async move { + *c.path.lock().await = req.uri().path().to_string(); + *c.query.lock().await = req.uri().query().unwrap_or("").to_string(); + *c.authorization.lock().await = req + .headers() + .get("authorization") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + + StatusCode::OK + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + + client.start_pool_decommission_by_id("1").await.unwrap(); + + assert_eq!(&*capture.path.lock().await, POOLS_DECOMMISSION_PATH); + assert_eq!(&*capture.query.lock().await, "by-id=true&pool=1"); + assert!( + capture + .authorization + .lock() + .await + .contains("/s3/aws4_request") + ); + + server.abort(); +} + +#[tokio::test] +async fn pool_status_uses_by_id_query_and_parses_decommission_info() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + POOLS_STATUS_PATH, + get( + move |State(c): State, req: Request| async move { + *c.path.lock().await = req.uri().path().to_string(); + *c.query.lock().await = req.uri().query().unwrap_or("").to_string(); + + ( + StatusCode::OK, + r#"{"id":1,"cmdline":"http://tenant-pool-a-{0...3}.tenant-hl.ns.svc.cluster.local:9000/data/rustfs{0...3}","lastUpdate":"2026-05-20T00:00:00Z","decommissionInfo":{"startTime":"2026-05-20T00:00:00Z","complete":true,"failed":false,"canceled":false,"objectsDecommissioned":10,"objectsDecommissionedFailed":0,"bytesDecommissioned":20,"bytesDecommissionedFailed":0}}"#, + ) + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + + let status = client.pool_status_by_id("1").await.unwrap(); + + assert_eq!(status.id, 1); + assert_eq!(&*capture.path.lock().await, POOLS_STATUS_PATH); + assert_eq!(&*capture.query.lock().await, "by-id=true&pool=1"); + assert_eq!( + status.decommission.and_then(|info| info.complete), + Some(true) + ); + + server.abort(); +} + +#[tokio::test] +async fn add_user_uses_expected_path_query_and_body() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + ADD_USER_PATH, + put( + move |State(c): State, req: Request| async move { + capture_signed_request(&c, req).await; + StatusCode::OK + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + client.add_user("app user~+/雪", "secret123").await.unwrap(); + + let request = capture.request().await; + assert_eq!(request.path, ADD_USER_PATH); + assert_eq!(request.query, "accessKey=app%20user~%2B%2F%E9%9B%AA"); + assert_eq!( + request.body, + r#"{"secretKey":"secret123","status":"enabled"}"# + ); + assert_eq!(request.content_type, JSON_CONTENT_TYPE); + assert_sigv4_matches_wire(&request, ADMIN_SIGNING_SERVICE); + + server.abort(); +} + +#[tokio::test] +async fn remove_user_is_idempotent_when_already_gone() { + let router = Router::new().route( + "/rustfs/admin/v3/remove-user", + axum::routing::delete(|| async { (StatusCode::NOT_FOUND, "NoSuchUser") }), + ); + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + client + .remove_user("app-user") + .await + .expect("removing an already-gone user should be treated as success"); + + server.abort(); +} + +#[tokio::test] +async fn user_exists_limits_unexpected_error_response_body() { + let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES + 1); + let router = Router::new().route( + USER_INFO_PATH, + get(move || { + let body = body.clone(); + async move { (StatusCode::BAD_GATEWAY, body) } + }), + ); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let err = client + .user_exists("app-user") + .await + .expect_err("unexpected user lookup error should hide oversized body"); + + assert_oversized_upstream_body_hidden(err); + + server.abort(); +} + +#[tokio::test] +async fn set_user_policy_uses_single_authoritative_mapping_call() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + SET_POLICY_PATH, + put( + move |State(c): State, req: Request| async move { + *c.path.lock().await = req.uri().path().to_string(); + *c.query.lock().await = req.uri().query().unwrap_or("").to_string(); + StatusCode::OK + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + client + .set_user_policy( + "app-user", + &["app-readwrite".to_string(), "diagnostics".to_string()], + ) + .await + .unwrap(); + + assert_eq!(&*capture.path.lock().await, SET_POLICY_PATH); + assert_eq!( + &*capture.query.lock().await, + "isGroup=false&policyName=app-readwrite%2Cdiagnostics&userOrGroup=app-user" + ); + + server.abort(); +} + +#[tokio::test] +async fn set_user_policy_rejects_empty_policy_list() { + let client = RustfsAdminClient::new_with_base_url("http://127.0.0.1:1", "access", "secret"); + + let err = client + .set_user_policy("app-user", &[]) + .await + .expect_err("empty policy list should be rejected before request"); + + assert!(matches!(err, RustfsClientError::InvalidPolicyName)); +} + +#[tokio::test] +async fn bucket_object_lock_enabled_parses_enabled_response() { + let router = Router::new().route( + "/app-data", + get(|req: Request| async move { + assert_eq!(req.uri().query().unwrap_or(""), "object-lock="); + ( + StatusCode::OK, + "Enabled", + ) + }), + ); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + + assert!(client.bucket_object_lock_enabled("app-data").await.unwrap()); + + server.abort(); +} + +#[tokio::test] +async fn bucket_object_lock_enabled_limits_unexpected_error_response_body() { + let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES + 1); + let router = Router::new().route( + "/app-data", + get(move |req: Request| { + let body = body.clone(); + async move { + assert_eq!(req.uri().query().unwrap_or(""), "object-lock="); + (StatusCode::BAD_GATEWAY, body) + } + }), + ); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let err = client + .bucket_object_lock_enabled("app-data") + .await + .expect_err("unexpected object-lock error should hide oversized body"); + + assert_oversized_upstream_body_hidden(err); + + server.abort(); +} + +#[tokio::test] +async fn create_bucket_sends_object_lock_header_and_region_body() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + "/app-data", + put( + move |State(c): State, req: Request| async move { + *c.path.lock().await = req.uri().path().to_string(); + *c.object_lock_header.lock().await = req + .headers() + .get("x-amz-bucket-object-lock-enabled") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + let body_bytes = axum::body::to_bytes(req.into_body(), usize::MAX) + .await + .unwrap(); + *c.body.lock().await = String::from_utf8(body_bytes.to_vec()).unwrap(); + StatusCode::OK + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let result = client + .create_bucket("app-data", Some("us-west-2"), true) + .await + .unwrap(); + + assert_eq!(result, CreateBucketResult::Created); + assert_eq!(&*capture.path.lock().await, "/app-data"); + assert_eq!(&*capture.object_lock_header.lock().await, "true"); + assert!( + capture + .body + .lock() + .await + .contains("us-west-2") + ); + + server.abort(); +} + +#[tokio::test] +async fn create_bucket_limits_unexpected_error_response_body() { + let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES + 1); + let router = Router::new().route( + "/app-data", + put(move || { + let body = body.clone(); + async move { (StatusCode::BAD_GATEWAY, body) } + }), + ); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let err = client + .create_bucket("app-data", None, false) + .await + .expect_err("unexpected bucket create error should hide oversized body"); + + assert_oversized_upstream_body_hidden(err); + + server.abort(); +} + +#[tokio::test] +async fn delete_bucket_is_idempotent_when_already_gone() { + let router = Router::new().route( + "/app-data", + axum::routing::delete(|| async { (StatusCode::NOT_FOUND, "NoSuchBucket") }), + ); + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + client + .delete_bucket("app-data") + .await + .expect("removing an already-gone bucket should be treated as success"); + + server.abort(); +} + +#[test] +fn extract_canned_policy_document_accepts_raw_policy_document() { + let raw_policy = + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"raw\",\"Effect\":\"Allow\"}]}"; + + let policy = extract_canned_policy_document(raw_policy).unwrap(); + + let policy_value = serde_json::from_str::(&policy).unwrap(); + assert_eq!(policy_value["Version"], "2012-10-17"); + assert_eq!(policy_value["Statement"][0]["Sid"], "raw"); +} diff --git a/deploy/rustfs-operator/README.md b/deploy/rustfs-operator/README.md index b8eb3860..7c6dfb39 100755 --- a/deploy/rustfs-operator/README.md +++ b/deploy/rustfs-operator/README.md @@ -76,6 +76,27 @@ manifests remain consistent. | `sts.service.type` | Kubernetes Service type for STS | `ClusterIP` | | `sts.service.port` | Kubernetes Service port for STS | `4223` | +### COSI Driver Configuration (experimental) + +Install the COSI controller first (`release-0.2`), then enable the chart-managed driver: + +```bash +kubectl apply -k 'github.com/kubernetes-sigs/container-object-storage-interface//?ref=release-0.2' +helm upgrade --install rustfs-operator deploy/rustfs-operator/ --set cosi.enabled=true +``` + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `cosi.enabled` | Deploy the RustFS COSI driver + sidecar | `false` | +| `cosi.driverName` | Driver name used in BucketClass / BucketAccessClass | `rustfs.objectstorage.k8s.io` | +| `cosi.replicas` | Driver Deployment replicas | `1` | +| `cosi.image.repository` | Driver image (empty = operator image) | `""` | +| `cosi.image.tag` | Driver image tag (empty = operator tag) | `""` | +| `cosi.sidecar.image.repository` | Official COSI provisioner sidecar image | `gcr.io/k8s-staging-sig-storage/objectstorage-sidecar` | +| `cosi.sidecar.image.tag` | Sidecar image tag | `v20230130-v0.1.0-24-gc0cf995` | + +Example manifests: `examples/cosi/`. See the operator user guide section on COSI. + The RustFS operator STS endpoint intentionally uses an explicit Tenant route: ```text diff --git a/deploy/rustfs-operator/templates/NOTES.txt b/deploy/rustfs-operator/templates/NOTES.txt index 91c3351f..d810a244 100755 --- a/deploy/rustfs-operator/templates/NOTES.txt +++ b/deploy/rustfs-operator/templates/NOTES.txt @@ -48,4 +48,15 @@ To open the Operator Console locally: {{ end }} {{ end }} + +{{- if .Values.cosi.enabled }} +COSI driver is enabled (experimental, v1alpha1). Driver name: {{ .Values.cosi.driverName }} + +Ensure the COSI controller is installed: + + kubectl apply -k 'github.com/kubernetes-sigs/container-object-storage-interface//?ref=release-0.2' + +Example manifests: examples/cosi/ + +{{- end }} For more information, visit: {{ .Chart.Home }} diff --git a/deploy/rustfs-operator/templates/_helpers.tpl b/deploy/rustfs-operator/templates/_helpers.tpl index 6903f303..4d0e475c 100755 --- a/deploy/rustfs-operator/templates/_helpers.tpl +++ b/deploy/rustfs-operator/templates/_helpers.tpl @@ -86,3 +86,23 @@ Create the name of the console service account to use {{- default "default" .Values.console.serviceAccount.name }} {{- end }} {{- end }} + +{{/* +COSI driver service account name +*/}} +{{- define "rustfs-operator.cosiServiceAccountName" -}} +{{- if .Values.cosi.serviceAccount.create }} +{{- default (printf "%s-cosi" (include "rustfs-operator.fullname" .)) .Values.cosi.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.cosi.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +COSI driver image (falls back to operator image) +*/}} +{{- define "rustfs-operator.cosiImage" -}} +{{- $repo := default .Values.operator.image.repository .Values.cosi.image.repository -}} +{{- $tag := default .Values.operator.image.tag .Values.cosi.image.tag -}} +{{- printf "%s:%s" $repo $tag -}} +{{- end }} diff --git a/deploy/rustfs-operator/templates/cosi-deployment.yaml b/deploy/rustfs-operator/templates/cosi-deployment.yaml new file mode 100644 index 00000000..ea4c16f7 --- /dev/null +++ b/deploy/rustfs-operator/templates/cosi-deployment.yaml @@ -0,0 +1,88 @@ +{{- if .Values.cosi.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "rustfs-operator.fullname" . }}-cosi + namespace: {{ include "rustfs-operator.namespace" . }} + labels: + {{- include "rustfs-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: cosi + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.cosi.replicas }} + selector: + matchLabels: + {{- include "rustfs-operator.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: cosi + template: + metadata: + labels: + {{- include "rustfs-operator.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: cosi + spec: + serviceAccountName: {{ include "rustfs-operator.cosiServiceAccountName" . }} + {{- with (.Values.cosi.imagePullSecrets | default .Values.operator.imagePullSecrets) }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.cosi.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: socket + emptyDir: {} + containers: + - name: rustfs-cosi-driver + image: {{ include "rustfs-operator.cosiImage" . }} + imagePullPolicy: {{ .Values.cosi.image.pullPolicy }} + command: ["/app/rustfs-cosi-driver"] + env: + - name: COSI_ENDPOINT + value: unix:///var/lib/cosi/cosi.sock + - name: RUST_LOG + value: info + volumeMounts: + - name: socket + mountPath: /var/lib/cosi + {{- with .Values.cosi.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.cosi.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: objectstorage-provisioner + image: "{{ .Values.cosi.sidecar.image.repository }}:{{ .Values.cosi.sidecar.image.tag }}" + imagePullPolicy: {{ .Values.cosi.sidecar.image.pullPolicy }} + args: + - "--v=4" + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: socket + mountPath: /var/lib/cosi + {{- with .Values.cosi.sidecar.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.cosi.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.cosi.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.cosi.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/deploy/rustfs-operator/templates/cosi-rbac.yaml b/deploy/rustfs-operator/templates/cosi-rbac.yaml new file mode 100644 index 00000000..e7abda07 --- /dev/null +++ b/deploy/rustfs-operator/templates/cosi-rbac.yaml @@ -0,0 +1,59 @@ +{{- if and .Values.cosi.enabled .Values.cosi.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "rustfs-operator.fullname" . }}-cosi + labels: + {{- include "rustfs-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: cosi + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + # Driver reads Tenant admin Secrets and optional TLS CA ConfigMaps referenced by BucketClass parameters. + - apiGroups: [""] + resources: ["secrets", "configmaps"] + verbs: ["get", "list", "watch"] + # Sidecar reconciles COSI API objects (v1alpha1). + - apiGroups: ["objectstorage.k8s.io"] + resources: + - buckets + - bucketaccesses + - bucketclaims + - bucketaccessclasses + - bucketclasses + verbs: ["get", "list", "watch", "update", "patch", "create", "delete"] + - apiGroups: ["objectstorage.k8s.io"] + resources: + - buckets/status + - bucketaccesses/status + - bucketclaims/status + verbs: ["get", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "rustfs-operator.fullname" . }}-cosi + labels: + {{- include "rustfs-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: cosi + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "rustfs-operator.fullname" . }}-cosi +subjects: + - kind: ServiceAccount + name: {{ include "rustfs-operator.cosiServiceAccountName" . }} + namespace: {{ include "rustfs-operator.namespace" . }} +{{- end }} diff --git a/deploy/rustfs-operator/templates/cosi-serviceaccount.yaml b/deploy/rustfs-operator/templates/cosi-serviceaccount.yaml new file mode 100644 index 00000000..ab743972 --- /dev/null +++ b/deploy/rustfs-operator/templates/cosi-serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if .Values.cosi.enabled }} +{{- if .Values.cosi.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "rustfs-operator.cosiServiceAccountName" . }} + namespace: {{ include "rustfs-operator.namespace" . }} + labels: + {{- include "rustfs-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: cosi + {{- if or .Values.cosi.serviceAccount.annotations .Values.commonAnnotations }} + annotations: + {{- with .Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.cosi.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} diff --git a/deploy/rustfs-operator/values.yaml b/deploy/rustfs-operator/values.yaml index 6dfc20e8..0edda480 100755 --- a/deploy/rustfs-operator/values.yaml +++ b/deploy/rustfs-operator/values.yaml @@ -307,3 +307,67 @@ console: # - secretName: console-tls # hosts: # - console.example.com + +# COSI (Container Object Storage Interface) driver — experimental, v1alpha1. +# Requires the COSI controller/CRDs installed separately: +# kubectl apply -k 'github.com/kubernetes-sigs/container-object-storage-interface//?ref=release-0.2' +cosi: + enabled: false + + driverName: rustfs.objectstorage.k8s.io + + replicas: 1 + + # Driver binary image (defaults to the operator image which embeds rustfs-cosi-driver). + image: + repository: "" # defaults to operator.image.repository + tag: "" # defaults to operator.image.tag + pullPolicy: IfNotPresent + + imagePullSecrets: [] + + # Official COSI provisioner sidecar (UNIX socket to the driver). + sidecar: + image: + repository: gcr.io/k8s-staging-sig-storage/objectstorage-sidecar + tag: v20230130-v0.1.0-24-gc0cf995 + pullPolicy: IfNotPresent + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 256Mi + + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + + podSecurityContext: + fsGroup: 65534 + + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 65534 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + + nodeSelector: {} + tolerations: [] + affinity: {} + + serviceAccount: + create: true + annotations: {} + name: "" + + rbac: + create: true diff --git a/docs/operator-user-guide.md b/docs/operator-user-guide.md index 734d314e..d31b89b4 100644 --- a/docs/operator-user-guide.md +++ b/docs/operator-user-guide.md @@ -930,7 +930,49 @@ Current STS constraints: - Caller-supplied `Policy` request parameters are rejected; issued credentials use the matched `PolicyBinding` policies. - Tenants requiring client certificates for upstream Tenant calls are rejected by Operator STS. -## 10. Monitoring and Status +## 10. COSI (Experimental) + +The operator ships an optional [Container Object Storage Interface](https://github.com/kubernetes-sigs/container-object-storage-interface) (COSI) **v1alpha1** driver (`rustfs.objectstorage.k8s.io`). Applications request buckets with `BucketClaim` / `BucketAccess` instead of embedding bucket lists on the Tenant CR. Tenant bootstrap provisioning (`spec.buckets`) and COSI can coexist. + +### 10.1 Prerequisites + +1. Install the COSI controller and CRDs (`release-0.2`): + +```bash +kubectl apply -k 'github.com/kubernetes-sigs/container-object-storage-interface//?ref=release-0.2' +``` + +2. Enable the driver in the Helm chart: + +```bash +helm upgrade --install rustfs-operator ./deploy/rustfs-operator \ + --set cosi.enabled=true +``` + +The chart deploys a pod with two containers: `rustfs-cosi-driver` and the official `objectstorage-sidecar`, sharing `unix:///var/lib/cosi/cosi.sock`. + +### 10.2 Point a BucketClass at a Tenant + +`BucketClass` / `BucketAccessClass` parameters (Rook-style) identify the Tenant admin credentials and S3 endpoint: + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `objectStoreUserSecretName` | yes | Secret with `accesskey` / `secretkey` (Tenant `spec.credsSecret`) | +| `objectStoreUserSecretNamespace` | yes | Namespace of that Secret | +| `endpoint` | yes | S3 URL, e.g. `http://{tenant}-io.{ns}.svc:9000` | +| `region` | no | Defaults to `us-east-1` | +| `policy` | no | On `BucketAccessClass`: `readonly` or `readwrite` (default) | +| `tlsCAConfigMapName` / `tlsCAConfigMapNamespace` | no | PEM CA for HTTPS Tenants | + +See `examples/cosi/` for full manifests. + +### 10.3 Limitations + +- Experimental; protocol **S3** and authentication **KEY** only. +- Does not replace Operator STS / `PolicyBinding` for temporary credentials. +- Driver name is fixed: `rustfs.objectstorage.k8s.io`. + +## 11. Monitoring and Status Check Tenant status: @@ -988,7 +1030,7 @@ operator: enabled: true ``` -## 11. Operations +## 12. Operations ### Change RustFS Image @@ -1026,7 +1068,7 @@ kubectl create secret generic rustfs-admin-creds \ kubectl rollout restart statefulset -n -l rustfs.tenant= ``` -## 12. Troubleshooting +## 13. Troubleshooting ### Tenant is Blocked @@ -1084,7 +1126,7 @@ kubectl logs -n rustfs-system \ For the RustFS Tenant Console, use the Tenant admin credentials from `spec.credsSecret` or configured RustFS environment variables. -## 13. Best Practices +## 14. Best Practices - Use `spec.credsSecret` or an external secret manager for production credentials. - Enable Kubernetes Secret encryption at rest. @@ -1097,7 +1139,7 @@ For the RustFS Tenant Console, use the Tenant admin credentials from `spec.creds - Keep Tenant examples under version control, but never commit raw Secret values. - Check `status.conditions` before debugging lower-level StatefulSets. -## 14. Related Documentation +## 15. Related Documentation - [Project README](../README.md) - [Deployment guide](../deploy/README.md) diff --git a/e2e/Cargo.lock b/e2e/Cargo.lock index b338af3d..eedfc07c 100644 --- a/e2e/Cargo.lock +++ b/e2e/Cargo.lock @@ -2526,7 +2526,6 @@ dependencies = [ "const-str", "futures", "hex", - "hmac 0.12.1", "hostname", "http 1.4.0", "http-body-util", @@ -2536,8 +2535,8 @@ dependencies = [ "kube", "kube-leader-election", "rcgen", - "reqwest", "ring", + "rustfs-admin", "rustls 0.23.40", "rustls-pemfile", "rustls-webpki 0.103.13", @@ -2558,7 +2557,6 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", - "url", "utoipa", "utoipa-swagger-ui", ] @@ -3110,6 +3108,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfs-admin" +version = "0.1.0" +dependencies = [ + "chrono", + "hex", + "hmac 0.12.1", + "reqwest", + "serde", + "serde_json", + "sha2 0.10.9", + "url", +] + [[package]] name = "rustfs-operator-e2e" version = "0.1.0" diff --git a/e2e/tests/sts_functional.rs b/e2e/tests/sts_functional.rs index 3855e967..16d1af75 100644 --- a/e2e/tests/sts_functional.rs +++ b/e2e/tests/sts_functional.rs @@ -21,7 +21,10 @@ use k8s_openapi::api::core::v1 as corev1; use kube::Api; use operator::{ console::state::AppState, - sts::{rustfs_client::RustfsAdminClient, server::routes}, + sts::{ + rustfs_client::{RustfsAdminClient, load_tenant_credentials, load_tenant_tls_ca}, + server::routes, + }, types::v1alpha1::tenant::Tenant, }; use rustfs_operator_e2e::framework::{ @@ -273,7 +276,7 @@ async fn ensure_rustfs_canned_policy( let rustfs_url = local_https_base_url(&rustfs_host, &rustfs_port_forward_spec); let mut rustfs_port_forward = PortForwardSpec::start_tenant_io(config).context("start RustFS tenant IO port-forward")?; - let tenant_ca = RustfsAdminClient::load_tenant_tls_ca(kube_client, tenant) + let tenant_ca = load_tenant_tls_ca(kube_client, tenant) .await .context("load TLS Tenant CA")? .context("TLS Tenant should publish a CA Secret reference")?; @@ -284,7 +287,7 @@ async fn ensure_rustfs_canned_policy( )?; wait_for_port_forward(&mut rustfs_port_forward, &rustfs_url, &rustfs_probe_client).await?; - let credentials = RustfsAdminClient::load_tenant_credentials(kube_client, tenant) + let credentials = load_tenant_credentials(kube_client, tenant) .await .context("load RustFS tenant credentials")?; let rustfs_admin = RustfsAdminClient::new_with_base_url_and_http_client( diff --git a/examples/README.md b/examples/README.md index 156b096e..4d1f1ec5 100755 --- a/examples/README.md +++ b/examples/README.md @@ -12,6 +12,7 @@ This directory contains example Tenant configurations for the RustFS Kubernetes | [simple-tenant.yaml](./simple-tenant.yaml) | Documentation Reference | ⭐⭐ Moderate | Configurable | Learning all options | | [secret-credentials-tenant.yaml](./secret-credentials-tenant.yaml) | Secret-based Credentials | ⭐ Simple | Configurable | **Production credential security** | | [provisioning-tenant.yaml](./provisioning-tenant.yaml) | Policy/User/Bucket Provisioning | ⭐⭐ Moderate | 40Gi | Tenant bootstrap automation | +| [cosi/](./cosi/) | COSI BucketClaim / BucketAccess | ⭐⭐ Moderate | — | Dynamic buckets via COSI v1alpha1 | | [multi-cert-tls-tenant.yaml](./multi-cert-tls-tenant.yaml) | Public/Internal TLS | ⭐⭐ Moderate | 40Gi | Separate SNI certificates | | [multi-pool-tenant.yaml](./multi-pool-tenant.yaml) | Multiple Pools | ⭐⭐ Moderate | ~160Gi | Multi-pool setups | | [production-ha-tenant.yaml](./production-ha-tenant.yaml) | Production HA | ⭐⭐⭐ Advanced | 6.4TB | HA with zone distribution | diff --git a/examples/cosi/awscli-pod.yaml b/examples/cosi/awscli-pod.yaml new file mode 100644 index 00000000..828ca39a --- /dev/null +++ b/examples/cosi/awscli-pod.yaml @@ -0,0 +1,20 @@ +# Sample app that mounts the COSI BucketAccess Secret and prints BucketInfo. +apiVersion: v1 +kind: Pod +metadata: + name: cosi-awscli + namespace: default +spec: + containers: + - name: awscli + image: amazon/aws-cli:2.15.0 + command: ["sleep", "3600"] + volumeMounts: + - name: cosi-secrets + mountPath: /data/cosi + readOnly: true + volumes: + - name: cosi-secrets + secret: + secretName: sample-bucket-access-secret + restartPolicy: Never diff --git a/examples/cosi/bucketaccess.yaml b/examples/cosi/bucketaccess.yaml new file mode 100644 index 00000000..941f99cb --- /dev/null +++ b/examples/cosi/bucketaccess.yaml @@ -0,0 +1,10 @@ +apiVersion: objectstorage.k8s.io/v1alpha1 +kind: BucketAccess +metadata: + name: sample-bucket-access + namespace: default +spec: + bucketClaimName: sample-bucket-claim + bucketAccessClassName: rustfs-bucket-access-class + credentialsSecretName: sample-bucket-access-secret + protocol: s3 diff --git a/examples/cosi/bucketaccessclass.yaml b/examples/cosi/bucketaccessclass.yaml new file mode 100644 index 00000000..28af551b --- /dev/null +++ b/examples/cosi/bucketaccessclass.yaml @@ -0,0 +1,13 @@ +apiVersion: objectstorage.k8s.io/v1alpha1 +kind: BucketAccessClass +metadata: + name: rustfs-bucket-access-class +driverName: rustfs.objectstorage.k8s.io +authenticationType: KEY +parameters: + objectStoreUserSecretName: provisioning-admin-creds + objectStoreUserSecretNamespace: default + endpoint: http://provisioning-demo-io.default.svc:9000 + region: us-east-1 + # readonly | readwrite (default) + policy: readwrite diff --git a/examples/cosi/bucketclaim.yaml b/examples/cosi/bucketclaim.yaml new file mode 100644 index 00000000..bc7b7014 --- /dev/null +++ b/examples/cosi/bucketclaim.yaml @@ -0,0 +1,9 @@ +apiVersion: objectstorage.k8s.io/v1alpha1 +kind: BucketClaim +metadata: + name: sample-bucket-claim + namespace: default +spec: + bucketClassName: rustfs-bucketclass + protocols: + - s3 diff --git a/examples/cosi/bucketclass.yaml b/examples/cosi/bucketclass.yaml new file mode 100644 index 00000000..d15f3329 --- /dev/null +++ b/examples/cosi/bucketclass.yaml @@ -0,0 +1,21 @@ +# Example BucketClass for a RustFS Tenant. +# +# Prerequisites: +# 1. COSI controller: kubectl apply -k 'github.com/kubernetes-sigs/container-object-storage-interface//?ref=release-0.2' +# 2. Operator chart with cosi.enabled=true +# 3. A ready Tenant whose admin Secret matches objectStoreUserSecretName/Namespace +# 4. endpoint points at the Tenant S3 Service (port 9000) +# +# Replace secret name/namespace and endpoint for your Tenant. + +apiVersion: objectstorage.k8s.io/v1alpha1 +kind: BucketClass +metadata: + name: rustfs-bucketclass +driverName: rustfs.objectstorage.k8s.io +deletionPolicy: Delete +parameters: + objectStoreUserSecretName: provisioning-admin-creds + objectStoreUserSecretNamespace: default + endpoint: http://provisioning-demo-io.default.svc:9000 + region: us-east-1 diff --git a/src/reconcile/pool_lifecycle.rs b/src/reconcile/pool_lifecycle.rs index c723dafd..f0edb516 100644 --- a/src/reconcile/pool_lifecycle.rs +++ b/src/reconcile/pool_lifecycle.rs @@ -25,7 +25,7 @@ use super::{Error, context}; use crate::context::Context; use crate::sts::rustfs_client::{ RustfsAdminClient, RustfsClientError, RustfsPoolDecommissionInfo, RustfsPoolListItem, - RustfsPoolStatus, + RustfsPoolStatus, client_from_tenant, client_from_tls_tenant_for_sts, load_tenant_credentials, }; use crate::types::v1alpha1::pool::Pool; use crate::types::v1alpha1::pool_lifecycle::{DecommissionAction, DecommissionRequest}; @@ -453,17 +453,11 @@ async fn rustfs_admin_client( ctx: &Context, tenant: &Tenant, ) -> Result { - let credentials = RustfsAdminClient::load_tenant_credentials(&ctx.client, tenant).await?; + let credentials = load_tenant_credentials(&ctx.client, tenant).await?; if tenant.spec.tls.as_ref().is_some_and(|tls| tls.is_enabled()) { - RustfsAdminClient::from_tls_tenant_for_sts( - &ctx.client, - tenant, - credentials, - ctx.cluster_domain(), - ) - .await + client_from_tls_tenant_for_sts(&ctx.client, tenant, credentials, ctx.cluster_domain()).await } else { - RustfsAdminClient::from_tenant(tenant, credentials) + client_from_tenant(tenant, credentials) } } diff --git a/src/reconcile/provisioning.rs b/src/reconcile/provisioning.rs index 6fdf633f..f761061e 100644 --- a/src/reconcile/provisioning.rs +++ b/src/reconcile/provisioning.rs @@ -13,7 +13,10 @@ // limitations under the License. use crate::context::{self, Context}; -use crate::sts::rustfs_client::{CreateBucketResult, RustfsAdminClient, RustfsClientError}; +use crate::sts::rustfs_client::{ + CreateBucketResult, RustfsAdminClient, RustfsClientError, client_from_tenant, + client_from_tls_tenant_for_sts, load_tenant_credentials, +}; use crate::types::v1alpha1::provisioning::{ ProvisioningBucket, ProvisioningPolicy, ProvisioningUser, duplicate_user_credentials_secret_names, @@ -496,17 +499,11 @@ async fn rustfs_admin_client( ctx: &Context, tenant: &Tenant, ) -> Result { - let credentials = RustfsAdminClient::load_tenant_credentials(&ctx.client, tenant).await?; + let credentials = load_tenant_credentials(&ctx.client, tenant).await?; if tenant.spec.tls.as_ref().is_some_and(|tls| tls.is_enabled()) { - RustfsAdminClient::from_tls_tenant_for_sts( - &ctx.client, - tenant, - credentials, - ctx.cluster_domain(), - ) - .await + client_from_tls_tenant_for_sts(&ctx.client, tenant, credentials, ctx.cluster_domain()).await } else { - RustfsAdminClient::from_tenant(tenant, credentials) + client_from_tenant(tenant, credentials) } } diff --git a/src/sts/helpers.rs b/src/sts/helpers.rs index b0975bde..8e772baa 100644 --- a/src/sts/helpers.rs +++ b/src/sts/helpers.rs @@ -12,20 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Internal helper duties: shared credential parsing, signature/hash utilities, and parsers. +//! Internal helper duties: Tenant/kube credential and TLS status parsing. +//! Wire-protocol helpers (signing, hashing, response parsing) live in the +//! kube-agnostic `rustfs-admin` crate. use std::collections::BTreeMap; -use hmac::{Hmac, Mac}; use k8s_openapi::ByteString; -use reqwest::StatusCode; -use serde_json::Value; -use sha2::{Digest, Sha256}; -use url::form_urlencoded; use crate::Tenant; -use crate::sts::types::StsAssumeRoleCredentials; - -use super::{RustfsClientError, RustfsCredentials}; +use crate::sts::rustfs_client::{RustfsClientError, RustfsCredentials}; pub(super) fn extract_credentials( data: Option<&BTreeMap>, @@ -69,158 +64,82 @@ pub(super) fn get_secret_value( Ok(value) } -/// Encode an `application/x-www-form-urlencoded` request body. -pub(super) fn build_form_body(params: &[(&str, &str)]) -> String { - let mut pairs: Vec<(String, String)> = params - .iter() - .map(|(k, v)| ((*k).to_string(), (*v).to_string())) - .collect(); - pairs.sort_by(|(k1, v1), (k2, v2)| k1.cmp(k2).then(v1.cmp(v2))); - - let mut serializer = form_urlencoded::Serializer::new(String::new()); - for (key, value) in pairs { - serializer.append_pair(&key, &value); - } +#[cfg(test)] +mod tests { + use k8s_openapi::{ByteString, api::core::v1 as corev1}; + use std::collections::BTreeMap; - serializer.finish() -} + use super::extract_credentials; + use crate::sts::rustfs_client::RustfsClientError; -/// Encode and sort query parameters according to the AWS SigV4 rules. -pub(super) fn build_canonical_query(params: &[(&str, &str)]) -> String { - let mut pairs: Vec<(String, String)> = params - .iter() - .map(|(key, value)| (uri_encode(key), uri_encode(value))) - .collect(); - pairs.sort_unstable(); - - pairs - .into_iter() - .map(|(key, value)| format!("{key}={value}")) - .collect::>() - .join("&") -} + fn secret_with_fields(fields: Vec<(&str, &[u8])>) -> corev1::Secret { + let mut data = BTreeMap::new(); + for (key, value) in fields { + data.insert(key.to_string(), ByteString(value.to_vec())); + } -fn uri_encode(value: &str) -> String { - const HEX: &[u8; 16] = b"0123456789ABCDEF"; - - let mut encoded = String::with_capacity(value.len()); - for byte in value.bytes() { - if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { - encoded.push(char::from(byte)); - } else { - encoded.push('%'); - encoded.push(char::from(HEX[usize::from(byte >> 4)])); - encoded.push(char::from(HEX[usize::from(byte & 0x0f)])); + corev1::Secret { + data: Some(data), + ..Default::default() } } - encoded -} -pub(super) fn create_bucket_body(region: Option<&str>) -> String { - let Some(region) = region.map(str::trim).filter(|region| !region.is_empty()) else { - return String::new(); - }; + #[test] + fn extract_credentials_reports_missing_access_key() { + let secret = secret_with_fields(vec![("secretkey", b"sekret")]); - if region == "us-east-1" { - return String::new(); + let err = + extract_credentials(secret.data.as_ref()).expect_err("expected missing access key"); + assert!(matches!( + err, + RustfsClientError::MissingCredentialKey { key: "accesskey" } + )); } - format!( - "{}", - escape_xml(region) - ) -} - -pub(super) fn escape_xml(value: &str) -> String { - value - .replace('&', "&") - .replace('<', "<") - .replace('>', ">") - .replace('"', """) - .replace('\'', "'") -} - -pub(super) fn body_mentions_not_found(body: &str) -> bool { - let body = body.to_ascii_lowercase(); - body.contains("nosuchuser") - || body.contains("no such user") - || body.contains("user not exist") - || body.contains("nosuchpolicy") - || body.contains("no such policy") - || body.contains("objectlockconfigurationnotfound") - || body.contains("not found") -} + #[test] + fn extract_credentials_reports_non_utf8_access_key() { + let secret = + secret_with_fields(vec![("accesskey", &[0xff, 0xfe]), ("secretkey", b"sekret")]); -pub(super) fn bucket_already_exists(status: StatusCode, body: &str) -> bool { - if status == StatusCode::CONFLICT { - let body = body.to_ascii_lowercase(); - return body.contains("bucketalreadyexists") || body.contains("bucketalreadyownedbyyou"); + let err = extract_credentials(secret.data.as_ref()).expect_err("expected invalid utf8"); + assert!(matches!( + err, + RustfsClientError::InvalidCredentialValue { key: "accesskey" } + )); } - false -} - -pub(super) fn extract_canned_policy_document(body: &str) -> Result { - let value = serde_json::from_str::(body) - .map_err(|_| RustfsClientError::InvalidPolicyDocument)?; - let policy = value.get("policy").unwrap_or(&value); + #[test] + fn extract_credentials_reports_missing_secret_key() { + let secret = secret_with_fields(vec![("accesskey", b"access")]); - serde_json::to_string(policy).map_err(|_| RustfsClientError::InvalidPolicyDocument) -} - -pub(super) fn sha256_hex(payload: &[u8]) -> String { - let mut hasher = Sha256::new(); - hasher.update(payload); - hex::encode(hasher.finalize()) -} - -pub(super) fn hmac_sha256(key: &[u8], message: &str) -> Result, RustfsClientError> { - let mut mac = - Hmac::::new_from_slice(key).map_err(|_| RustfsClientError::SigningFailed)?; - mac.update(message.as_bytes()); - Ok(mac.finalize().into_bytes().to_vec()) -} - -pub(super) fn hmac_sha256_hex(key: &[u8], message: &str) -> Result { - let bytes = hmac_sha256(key, message)?; - Ok(hex::encode(bytes)) -} - -pub(super) fn derive_signing_key( - secret_key: &str, - date_stamp: &str, - region: &str, - service: &str, -) -> Result, RustfsClientError> { - let k_secret = format!("AWS4{secret_key}").into_bytes(); - let k_date = hmac_sha256(&k_secret, date_stamp)?; - let k_region = hmac_sha256(&k_date, region)?; - let k_service = hmac_sha256(&k_region, service)?; - hmac_sha256(&k_service, "aws4_request") -} + let err = + extract_credentials(secret.data.as_ref()).expect_err("expected missing secret key"); + assert!(matches!( + err, + RustfsClientError::MissingCredentialKey { key: "secretkey" } + )); + } -pub(super) fn parse_assume_role_response(body: &str) -> Option { - let access_key_id = extract_xml_tag(body, "AccessKeyId")?; - let secret_access_key = extract_xml_tag(body, "SecretAccessKey")?; - let session_token = extract_xml_tag(body, "SessionToken")?; - let expiration = extract_xml_tag(body, "Expiration")?; - - Some(StsAssumeRoleCredentials { - access_key_id, - secret_access_key, - session_token, - expiration, - }) -} + #[test] + fn extract_credentials_reports_non_utf8_secret_key() { + let secret = + secret_with_fields(vec![("accesskey", b"access"), ("secretkey", &[0xff, 0xfe])]); -pub(super) fn extract_xml_tag(document: &str, tag: &str) -> Option { - let open = format!("<{tag}>"); - let close = format!(""); + let err = extract_credentials(secret.data.as_ref()).expect_err("expected invalid utf8"); + assert!(matches!( + err, + RustfsClientError::InvalidCredentialValue { key: "secretkey" } + )); + } - let open_idx = document.find(&open)?; - let start = open_idx + open.len(); - let rest = &document[start..]; - let end = rest.find(&close)?; + #[test] + fn extract_credentials_reports_empty_secret_key() { + let secret = secret_with_fields(vec![("accesskey", b"abc"), ("secretkey", b"")]); - Some(rest[..end].trim().to_string()) + let err = extract_credentials(secret.data.as_ref()).expect_err("expected empty secret key"); + assert!(matches!( + err, + RustfsClientError::EmptyCredentialValue { key: "secretkey" } + )); + } } diff --git a/src/sts/rustfs_client.rs b/src/sts/rustfs_client.rs index 865c27f3..c1507c82 100644 --- a/src/sts/rustfs_client.rs +++ b/src/sts/rustfs_client.rs @@ -12,420 +12,33 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::{collections::BTreeMap, time::Duration}; +//! Kube/Tenant wrappers around the kube-agnostic RustFS admin/S3/STS client. +//! +//! The wire-protocol client implementation (request signing, HTTP dispatch, +//! response parsing) lives in the `rustfs-admin` crate and is re-exported +//! here. This module only adds the Tenant/kube-specific constructors that +//! need access to `kube::Client` and the `Tenant` CRD type. use k8s_openapi::api::core::v1 as corev1; use kube::{Api, Client}; -use reqwest::{Certificate, Client as HttpClient, Response, StatusCode}; use crate::Tenant; use crate::cluster_dns; -use crate::utils::sanitize::redact_sensitive_pairs; -/// admin_ops: tenant admin operations (user/policy APIs). -#[path = "admin_ops.rs"] -mod admin_ops; -/// core_ops: shared request signing/dispatch internals. -#[path = "core_ops.rs"] -mod core_ops; -/// helpers: credential parsing, signing/hash utilities and parsers. +/// helpers: Tenant/kube credential and TLS status parsing. #[path = "helpers.rs"] mod helpers; -/// pool_ops: pool lifecycle and status operations. -#[path = "pool_ops.rs"] -mod pool_ops; -/// s3_ops: bucket/object-lock operations for S3-compatible endpoints. -#[path = "s3_ops.rs"] -mod s3_ops; -/// sts_ops: temporary credential flows, AssumeRole request/response. -#[path = "sts_ops.rs"] -mod sts_ops; -const FORM_CONTENT_TYPE: &str = "application/x-www-form-urlencoded"; -const JSON_CONTENT_TYPE: &str = "application/json"; -const ASSUME_ROLE_PATH: &str = "/"; -const ADD_USER_PATH: &str = "/rustfs/admin/v3/add-user"; -const USER_INFO_PATH: &str = "/rustfs/admin/v3/user-info"; -const SET_POLICY_PATH: &str = "/rustfs/admin/v3/set-policy"; -const LIST_CANNED_POLICIES_PATH: &str = "/rustfs/admin/v3/list-canned-policies"; -const ADD_CANNED_POLICY_PATH: &str = "/rustfs/admin/v3/add-canned-policy"; -const INFO_CANNED_POLICY_PATH: &str = "/rustfs/admin/v3/info-canned-policy"; -const SERVER_INFO_PATH: &str = "/rustfs/admin/v3/info"; -const POOLS_LIST_PATH: &str = "/rustfs/admin/v3/pools/list"; -const POOLS_STATUS_PATH: &str = "/rustfs/admin/v3/pools/status"; -const POOLS_DECOMMISSION_PATH: &str = "/rustfs/admin/v3/pools/decommission"; -const POOLS_CANCEL_PATH: &str = "/rustfs/admin/v3/pools/cancel"; -const ADMIN_SIGNING_SERVICE: &str = "s3"; -const STS_SIGNING_SERVICE: &str = "sts"; -const ADMIN_HTTP_CONNECT_TIMEOUT: Duration = Duration::from_secs(3); -const ADMIN_HTTP_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); -const MAX_UPSTREAM_ERROR_BODY_BYTES: usize = 8 * 1024; -const MAX_UPSTREAM_ERROR_DETAIL_CHARS: usize = 512; +pub use rustfs_admin::{ + CreateBucketResult, RustfsAdminClient, RustfsClientError, RustfsCredentials, + RustfsErasureBackend, RustfsErasureSetInfo, RustfsPoolDecommissionInfo, RustfsPoolListItem, + RustfsPoolStatus, RustfsServerInfo, RustfsServerUsage, StsAssumeRoleCredentials, +}; -/// Credentials read from Tenant `.spec.credsSecret`. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RustfsCredentials { - pub access_key: String, - pub secret_key: String, -} - -#[derive(Debug, Clone, serde::Deserialize, PartialEq)] -pub struct RustfsPoolListItem { - pub id: usize, - #[serde(rename = "cmdline")] - pub cmd_line: String, - #[serde(rename = "lastUpdate")] - pub last_update: String, - #[serde(rename = "totalSize")] - pub total_size: Option, - #[serde(rename = "currentSize")] - pub current_size: Option, - #[serde(rename = "usedSize")] - pub used_size: Option, - pub used: Option, - pub status: String, - #[serde(rename = "decommissionInfo")] - pub decommission: Option, -} - -#[derive(Debug, Clone, serde::Deserialize, PartialEq)] -pub struct RustfsPoolStatus { - pub id: usize, - #[serde(rename = "cmdline")] - pub cmd_line: String, - #[serde(rename = "lastUpdate")] - pub last_update: String, - #[serde(rename = "decommissionInfo")] - pub decommission: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum CreateBucketResult { - Created, - AlreadyExists, -} - -#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] -pub struct RustfsPoolDecommissionInfo { - #[serde(rename = "startTime")] - pub start_time: Option, - #[serde(rename = "startSize")] - pub start_size: Option, - #[serde(rename = "totalSize")] - pub total_size: Option, - #[serde(rename = "currentSize")] - pub current_size: Option, - pub complete: Option, - pub failed: Option, - pub canceled: Option, - #[serde(rename = "objectsDecommissioned")] - pub objects_decommissioned: Option, - #[serde(rename = "objectsDecommissionedFailed")] - pub objects_decommissioned_failed: Option, - #[serde(rename = "bytesDecommissioned")] - pub bytes_decommissioned: Option, - #[serde(rename = "bytesDecommissionedFailed")] - pub bytes_decommissioned_failed: Option, -} - -#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] -pub struct RustfsServerInfo { - #[serde(default)] - pub usage: Option, - #[serde(default)] - pub backend: Option, - #[serde(default)] - pub pools: Option>>, -} - -#[derive(Debug, Clone, serde::Deserialize, PartialEq)] -pub(super) struct RustfsServerInfoResponse { - pub info: RustfsServerInfo, -} - -#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] -pub struct RustfsServerUsage { - #[serde(default)] - pub size: u64, -} - -#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] -pub struct RustfsErasureBackend { - #[serde(default, rename = "onlineDisks")] - pub online_disks: u64, - #[serde(default, rename = "offlineDisks")] - pub offline_disks: u64, - #[serde(default, rename = "standardSCParity", alias = "StandardSCParity")] - pub standard_sc_parity: Option, - #[serde(default, rename = "totalSets")] - pub total_sets: Vec, - #[serde(default, rename = "totalDrivesPerSet", alias = "drivesPerSet")] - pub drives_per_set: Vec, -} - -#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] -pub struct RustfsErasureSetInfo { - #[serde(default, rename = "rawUsage")] - pub raw_usage: u64, - #[serde(default, rename = "rawCapacity")] - pub raw_capacity: u64, - #[serde(default)] - pub usage: u64, - #[serde(default, rename = "objectsCount")] - pub objects_count: u64, - #[serde(default, rename = "healDisks")] - pub heal_disks: u64, -} - -/// Error type for RustFS admin/STS client operations. -#[derive(Debug)] -pub enum RustfsClientError { - MissingTenantNamespace, - MissingCredsSecret, - MissingCredentialKey { - key: &'static str, - }, - EmptyCredentialValue { - key: &'static str, - }, - InvalidCredentialValue { - key: &'static str, - }, - TenantSecretLookupFailed, - InvalidPolicyName, - InvalidPolicyDocument, - TenantTlsRequired, - TenantTlsNotReady, - TenantTlsClientCertificateRequired, - MissingTenantTlsCaKey { - secret: String, - key: String, - }, - TenantTlsCaSecretLookupFailed { - secret: String, - }, - InvalidTenantTlsCa, - TlsClientBuildFailed, - RequestBuildFailed, - RequestFailed, - UnexpectedStatus { - status: StatusCode, - detail: Option, - }, - ParseResponseFailed, - SigningFailed, -} - -impl std::fmt::Display for RustfsClientError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::MissingTenantNamespace => write!(f, "tenant namespace is missing"), - Self::MissingCredsSecret => write!(f, "tenant credsSecret is missing"), - Self::MissingCredentialKey { key } => write!(f, "secret key missing: {key}"), - Self::EmptyCredentialValue { key } => write!(f, "secret key empty: {key}"), - Self::InvalidCredentialValue { key } => { - write!(f, "secret key is not valid utf8: {key}") - } - Self::TenantSecretLookupFailed => { - write!(f, "failed to load tenant credential secret") - } - Self::InvalidPolicyName => write!(f, "invalid policy name"), - Self::InvalidPolicyDocument => write!(f, "failed to parse canned policy response"), - Self::TenantTlsRequired => write!(f, "STS requires a TLS-enabled tenant"), - Self::TenantTlsNotReady => write!(f, "tenant TLS status is not ready"), - Self::TenantTlsClientCertificateRequired => { - write!(f, "tenant TLS requires a client certificate") - } - Self::MissingTenantTlsCaKey { secret, key } => { - write!(f, "tenant TLS CA secret {secret} missing key {key}") - } - Self::TenantTlsCaSecretLookupFailed { secret } => { - write!(f, "failed to load tenant TLS CA secret {secret}") - } - Self::InvalidTenantTlsCa => write!(f, "tenant TLS CA is not a valid PEM bundle"), - Self::TlsClientBuildFailed => write!(f, "failed to build TLS HTTP client"), - Self::RequestBuildFailed => write!(f, "failed to construct request"), - Self::RequestFailed => write!(f, "request failed"), - Self::UnexpectedStatus { status, detail } => { - write!(f, "upstream returned {status}")?; - if let Some(detail) = detail { - write!(f, ": {detail}")?; - } - Ok(()) - } - Self::ParseResponseFailed => write!(f, "failed to parse AssumeRole response"), - Self::SigningFailed => write!(f, "failed to compute request signature"), - } - } -} - -impl std::error::Error for RustfsClientError {} - -impl RustfsClientError { - pub(super) async fn unexpected_response(response: Response) -> Self { - let status = response.status(); - let (body, truncated) = read_limited_response_body(response).await; - Self::unexpected_status_with_limited_body(status, &body, truncated) - } - - pub(super) async fn limited_response_body(response: Response) -> (String, bool) { - read_limited_response_body(response).await - } - - fn unexpected_status_with_limited_body( - status: StatusCode, - body: &str, - body_truncated: bool, - ) -> Self { - Self::UnexpectedStatus { - status, - detail: summarize_upstream_error_body(body, body_truncated), - } - } - - #[cfg(test)] - pub(super) fn unexpected_status_with_body(status: StatusCode, body: &str) -> Self { - Self::unexpected_status_with_limited_body(status, body, false) - } -} - -async fn read_limited_response_body(mut response: Response) -> (String, bool) { - let mut body = Vec::new(); - let read_limit = MAX_UPSTREAM_ERROR_BODY_BYTES.saturating_add(1); - - loop { - let remaining = read_limit.saturating_sub(body.len()); - if remaining == 0 { - break; - } - - let chunk = match response.chunk().await { - Ok(Some(chunk)) => chunk, - Ok(None) => break, - Err(_) => break, - }; - if chunk.len() > remaining { - body.extend_from_slice(&chunk[..remaining]); - break; - } - body.extend_from_slice(&chunk); - } - - let truncated = body.len() > MAX_UPSTREAM_ERROR_BODY_BYTES; - if truncated { - body.truncate(MAX_UPSTREAM_ERROR_BODY_BYTES); - } - - (String::from_utf8_lossy(&body).into_owned(), truncated) -} - -fn summarize_upstream_error_body(body: &str, body_truncated: bool) -> Option { - let body = body.trim(); - if body.is_empty() { - return None; - } - - if let Some(message) = helpers::extract_xml_tag(body, "Message") { - let message = decode_basic_xml_entities(&message); - let detail = match helpers::extract_xml_tag(body, "Code") { - Some(code) if !code.trim().is_empty() => { - format!("{}: {message}", decode_basic_xml_entities(&code)) - } - _ => message, - }; - return Some(sanitize_error_detail(&detail)); - } - - if let Ok(value) = serde_json::from_str::(body) - && let Some(detail) = summarize_json_error(&value) - { - return Some(sanitize_error_detail(&detail)); - } - - if body_truncated { - return Some(format!( - "response body exceeded {MAX_UPSTREAM_ERROR_BODY_BYTES} bytes" - )); - } - - Some(sanitize_error_detail(body)) -} - -fn summarize_json_error(value: &serde_json::Value) -> Option { - if let Some(message) = value.as_str() { - return Some(message.to_string()); - } - - let object = value.as_object()?; - let message = ["message", "Message", "error", "Error"] - .iter() - .find_map(|key| object.get(*key).and_then(serde_json::Value::as_str))?; - let code = ["code", "Code"] - .iter() - .find_map(|key| object.get(*key).and_then(serde_json::Value::as_str)); - - Some(match code { - Some(code) if !code.trim().is_empty() => format!("{code}: {message}"), - _ => message.to_string(), - }) -} - -fn collapse_whitespace(value: &str) -> String { - value.split_whitespace().collect::>().join(" ") -} - -fn sanitize_error_detail(value: &str) -> String { - let detail = collapse_whitespace(value); - let detail = redact_sensitive_pairs(&detail); - truncate_error_detail(detail) -} - -fn truncate_error_detail(value: String) -> String { - let mut truncated = String::new(); - for (index, ch) in value.chars().enumerate() { - if index >= MAX_UPSTREAM_ERROR_DETAIL_CHARS { - truncated.push_str("..."); - return truncated; - } - truncated.push(ch); - } - truncated -} - -fn decode_basic_xml_entities(value: &str) -> String { - value - .replace("<", "<") - .replace(">", ">") - .replace(""", "\"") - .replace("'", "'") - .replace("&", "&") -} - -#[derive(Debug)] -struct SignedRequest { - amz_date: String, - payload_hash: String, - authorization: String, -} - -/// RustFS admin/STS client. -pub struct RustfsAdminClient { - base_url: String, - access_key: String, - secret_key: String, - region: String, - http_client: HttpClient, -} - -fn default_http_client() -> HttpClient { - HttpClient::builder() - .connect_timeout(ADMIN_HTTP_CONNECT_TIMEOUT) - .timeout(ADMIN_HTTP_REQUEST_TIMEOUT) - .build() - .unwrap_or_else(|_| HttpClient::new()) -} - -fn tls_tenant_base_url(tenant: &Tenant, cluster_domain: &str) -> Result { +pub(super) fn tls_tenant_base_url( + tenant: &Tenant, + cluster_domain: &str, +) -> Result { let namespace = tenant .namespace() .map_err(|_| RustfsClientError::MissingTenantNamespace)?; @@ -434,179 +47,124 @@ fn tls_tenant_base_url(tenant: &Tenant, cluster_domain: &str) -> Result, - access_key: impl Into, - secret_key: impl Into, - ) -> Self { - Self::new_with_base_url_and_http_client( +/// Build a RustFS admin client using the tenant's in-cluster (plain HTTP) service address. +pub fn client_from_tenant( + tenant: &Tenant, + credentials: RustfsCredentials, +) -> Result { + let namespace = tenant + .namespace() + .map_err(|_| RustfsClientError::MissingTenantNamespace)?; + let service_name = tenant + .new_io_service() + .metadata + .name + .unwrap_or_else(|| format!("{}-io", tenant.name())); + + Ok(RustfsAdminClient::new_with_base_url( + format!("http://{service_name}.{namespace}.svc:9000"), + credentials.access_key, + credentials.secret_key, + )) +} + +/// Build a RustFS admin client against the tenant's TLS-enabled headless service, +/// trusting the tenant's CA if one is published. Requires TLS to be enabled. +pub async fn client_from_tls_tenant_for_sts( + kube_client: &Client, + tenant: &Tenant, + credentials: RustfsCredentials, + cluster_domain: &str, +) -> Result { + if !helpers::tenant_tls_enabled(tenant) { + return Err(RustfsClientError::TenantTlsRequired); + } + if helpers::tenant_tls_client_certificate_required(tenant) { + return Err(RustfsClientError::TenantTlsClientCertificateRequired); + } + + let base_url = tls_tenant_base_url(tenant, cluster_domain)?; + + match load_tenant_tls_ca(kube_client, tenant).await? { + Some(ca_pem) => RustfsAdminClient::new_with_base_url_and_ca_pem( base_url, - access_key, - secret_key, - default_http_client(), - ) - } - - pub fn new_with_base_url_and_ca_pem( - base_url: impl Into, - access_key: impl Into, - secret_key: impl Into, - ca_pem: &[u8], - ) -> Result { - let certs = Certificate::from_pem_bundle(ca_pem) - .map_err(|_| RustfsClientError::InvalidTenantTlsCa)?; - let mut builder = HttpClient::builder() - .connect_timeout(ADMIN_HTTP_CONNECT_TIMEOUT) - .timeout(ADMIN_HTTP_REQUEST_TIMEOUT); - for cert in certs { - builder = builder.add_root_certificate(cert); - } - let http_client = builder - .build() - .map_err(|_| RustfsClientError::TlsClientBuildFailed)?; - - Ok(Self::new_with_base_url_and_http_client( + credentials.access_key, + credentials.secret_key, + &ca_pem, + ), + None => Ok(RustfsAdminClient::new_with_base_url( base_url, - access_key, - secret_key, - http_client, - )) - } - - pub fn new_with_base_url_and_http_client( - base_url: impl Into, - access_key: impl Into, - secret_key: impl Into, - http_client: HttpClient, - ) -> Self { - Self { - base_url: base_url.into(), - access_key: access_key.into(), - secret_key: secret_key.into(), - region: "us-east-1".to_string(), - http_client, - } - } - - pub fn from_tenant( - tenant: &Tenant, - credentials: RustfsCredentials, - ) -> Result { - let namespace = tenant - .namespace() - .map_err(|_| RustfsClientError::MissingTenantNamespace)?; - let service_name = tenant - .new_io_service() - .metadata - .name - .unwrap_or_else(|| format!("{}-io", tenant.name())); - - Ok(Self::new_with_base_url( - format!("http://{service_name}.{namespace}.svc:9000"), credentials.access_key, credentials.secret_key, - )) + )), } +} - pub async fn from_tls_tenant_for_sts( - kube_client: &Client, - tenant: &Tenant, - credentials: RustfsCredentials, - cluster_domain: &str, - ) -> Result { - if !helpers::tenant_tls_enabled(tenant) { - return Err(RustfsClientError::TenantTlsRequired); - } - if helpers::tenant_tls_client_certificate_required(tenant) { - return Err(RustfsClientError::TenantTlsClientCertificateRequired); - } - - let base_url = tls_tenant_base_url(tenant, cluster_domain)?; - - match Self::load_tenant_tls_ca(kube_client, tenant).await? { - Some(ca_pem) => Self::new_with_base_url_and_ca_pem( - base_url, - credentials.access_key, - credentials.secret_key, - &ca_pem, - ), - None => Ok(Self::new_with_base_url( - base_url, - credentials.access_key, - credentials.secret_key, - )), - } +/// Load the tenant's TLS CA bundle, if the tenant publishes one. +pub async fn load_tenant_tls_ca( + kube_client: &Client, + tenant: &Tenant, +) -> Result>, RustfsClientError> { + if !helpers::tenant_tls_enabled(tenant) { + return Ok(None); } - pub async fn load_tenant_tls_ca( - kube_client: &Client, - tenant: &Tenant, - ) -> Result>, RustfsClientError> { - if !helpers::tenant_tls_enabled(tenant) { - return Ok(None); - } - - let tls_status = tenant - .status - .as_ref() - .and_then(|status| status.certificates.tls.as_ref()) - .filter(|tls| tls.ready) - .ok_or(RustfsClientError::TenantTlsNotReady)?; + let tls_status = tenant + .status + .as_ref() + .and_then(|status| status.certificates.tls.as_ref()) + .filter(|tls| tls.ready) + .ok_or(RustfsClientError::TenantTlsNotReady)?; - let Some(ca_ref) = tls_status.ca_secret_ref.as_ref() else { - return Ok(None); - }; + let Some(ca_ref) = tls_status.ca_secret_ref.as_ref() else { + return Ok(None); + }; - let namespace = tenant - .namespace() - .map_err(|_| RustfsClientError::MissingTenantNamespace)?; - let api: Api = Api::namespaced(kube_client.clone(), &namespace); - let secret = api.get(&ca_ref.name).await.map_err(|_| { - RustfsClientError::TenantTlsCaSecretLookupFailed { - secret: ca_ref.name.clone(), - } + let namespace = tenant + .namespace() + .map_err(|_| RustfsClientError::MissingTenantNamespace)?; + let api: Api = Api::namespaced(kube_client.clone(), &namespace); + let secret = api.get(&ca_ref.name).await.map_err(|_| { + RustfsClientError::TenantTlsCaSecretLookupFailed { + secret: ca_ref.name.clone(), + } + })?; + let key = ca_ref.key.as_deref().unwrap_or("ca.crt"); + let ca_pem = secret + .data + .as_ref() + .and_then(|data| data.get(key)) + .map(|bytes| bytes.0.clone()) + .filter(|bytes| !bytes.is_empty()) + .ok_or_else(|| RustfsClientError::MissingTenantTlsCaKey { + secret: ca_ref.name.clone(), + key: key.to_string(), })?; - let key = ca_ref.key.as_deref().unwrap_or("ca.crt"); - let ca_pem = secret - .data - .as_ref() - .and_then(|data| data.get(key)) - .map(|bytes| bytes.0.clone()) - .filter(|bytes| !bytes.is_empty()) - .ok_or_else(|| RustfsClientError::MissingTenantTlsCaKey { - secret: ca_ref.name.clone(), - key: key.to_string(), - })?; - Ok(Some(ca_pem)) - } + Ok(Some(ca_pem)) +} - /// Read Tenant credential Secret and return access/secret key pair. - pub async fn load_tenant_credentials( - kube_client: &Client, - tenant: &Tenant, - ) -> Result { - let reference = tenant - .spec - .creds_secret - .as_ref() - .ok_or(RustfsClientError::MissingCredsSecret)?; +/// Read the Tenant credential Secret and return an access/secret key pair. +pub async fn load_tenant_credentials( + kube_client: &Client, + tenant: &Tenant, +) -> Result { + let reference = tenant + .spec + .creds_secret + .as_ref() + .ok_or(RustfsClientError::MissingCredsSecret)?; - let namespace = tenant - .namespace() - .map_err(|_| RustfsClientError::MissingTenantNamespace)?; - let api: Api = Api::namespaced(kube_client.clone(), &namespace); - let secret = api - .get(&reference.name) - .await - .map_err(|_| RustfsClientError::TenantSecretLookupFailed)?; + let namespace = tenant + .namespace() + .map_err(|_| RustfsClientError::MissingTenantNamespace)?; + let api: Api = Api::namespaced(kube_client.clone(), &namespace); + let secret = api + .get(&reference.name) + .await + .map_err(|_| RustfsClientError::TenantSecretLookupFailed)?; - helpers::extract_credentials(secret.data.as_ref()) - } + helpers::extract_credentials(secret.data.as_ref()) } #[cfg(test)] diff --git a/src/sts/server.rs b/src/sts/server.rs index 7914ca71..fd04ec63 100644 --- a/src/sts/server.rs +++ b/src/sts/server.rs @@ -32,7 +32,9 @@ use crate::http_admission::{ use crate::metrics::UnauthenticatedRequestOutcome; use crate::sts::binding; use crate::sts::error::{StsError, StsErrorType, render_sts_error_xml_with_type}; -use crate::sts::rustfs_client::{RustfsAdminClient, RustfsClientError}; +use crate::sts::rustfs_client::{ + RustfsAdminClient, RustfsClientError, client_from_tls_tenant_for_sts, load_tenant_credentials, +}; use crate::sts::session_policy; use crate::sts::token_review::{self, TokenReviewError}; use crate::sts::types::{ @@ -642,11 +644,11 @@ async fn create_rustfs_admin_client( }); } - let credentials = RustfsAdminClient::load_tenant_credentials(client, tenant) + let credentials = load_tenant_credentials(client, tenant) .await .map_err(|_| StsError::InternalError)?; - RustfsAdminClient::from_tls_tenant_for_sts(client, tenant, credentials, cluster_domain) + client_from_tls_tenant_for_sts(client, tenant, credentials, cluster_domain) .await .map_err(map_rustfs_client_creation_error) } diff --git a/src/sts/tests.rs b/src/sts/tests.rs index 502a92aa..52064a0e 100644 --- a/src/sts/tests.rs +++ b/src/sts/tests.rs @@ -12,112 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Unit/integration tests for RustfsAdminClient split operation modules. +//! Tests for the Tenant/kube-specific wrappers around `RustfsAdminClient`. +//! Wire-protocol tests (signing, hashing, response parsing) live in the +//! `rustfs-admin` crate. -use axum::{ - Router, - body::Body, - extract::State, - http::{Request, StatusCode}, - routing::{get, post, put}, -}; -use k8s_openapi::{ByteString, api::core::v1 as corev1}; -use serde_json::Value; -use std::{collections::BTreeMap, sync::Arc}; -use tokio::sync::Mutex; - -use super::{ - ADD_USER_PATH, ADMIN_SIGNING_SERVICE, CreateBucketResult, FORM_CONTENT_TYPE, JSON_CONTENT_TYPE, - LIST_CANNED_POLICIES_PATH, MAX_UPSTREAM_ERROR_BODY_BYTES, POOLS_DECOMMISSION_PATH, - POOLS_LIST_PATH, POOLS_STATUS_PATH, RustfsAdminClient, RustfsClientError, SERVER_INFO_PATH, - SET_POLICY_PATH, STS_SIGNING_SERVICE, USER_INFO_PATH, - helpers::{ - build_canonical_query, build_form_body, derive_signing_key, extract_canned_policy_document, - extract_credentials, hmac_sha256_hex, parse_assume_role_response, sha256_hex, - }, - tls_tenant_base_url, -}; - -const TEST_ACCESS_KEY: &str = "access"; -const TEST_SECRET_KEY: &str = "secret"; -const TEST_REGION: &str = "us-east-1"; - -#[test] -fn canonical_query_uses_sigv4_uri_encoding_and_encoded_sort_order() { - let query = build_canonical_query(&[ - ("z", "a b~c/雪"), - ("a~", "second"), - ("a ", "first"), - ("amp", "&="), - ("dup", "z"), - ("dup", "a"), - ("empty", ""), - ("雪", "key"), - ]); - - assert_eq!( - query, - "%E9%9B%AA=key&a%20=first&=%26%3D&a~=second&dup=a&dup=z&empty=&z=a%20b~c%2F%E9%9B%AA" - ); -} - -#[test] -fn form_body_keeps_html_form_encoding() { - assert_eq!( - build_form_body(&[("Policy", "a b~c/雪")]), - "Policy=a+b%7Ec%2F%E9%9B%AA" - ); -} - -#[test] -fn duplicate_query_values_match_independent_sigv4_verification() { - let query = - build_canonical_query(&[("dup", "z z"), ("dup", "a+a"), ("dup", "雪"), ("empty", "")]); - assert_eq!(query, "dup=%E9%9B%AA&dup=a%2Ba&dup=z%20z&empty="); - - let client = RustfsAdminClient::new_with_base_url( - "https://rustfs.example.test:9000", - TEST_ACCESS_KEY, - TEST_SECRET_KEY, - ); - let signed = client - .sign_request("GET", "/synthetic", &query, "", None, ADMIN_SIGNING_SERVICE) - .unwrap(); - let request = CapturedRequest { - method: "GET".to_string(), - path: "/synthetic".to_string(), - query, - body: String::new(), - host: "rustfs.example.test:9000".to_string(), - content_type: String::new(), - amz_date: signed.amz_date, - payload_hash: signed.payload_hash, - authorization: signed.authorization, - }; - - assert_sigv4_matches_wire(&request, ADMIN_SIGNING_SERVICE); -} - -fn secret_with_fields(fields: Vec<(&str, &[u8])>) -> corev1::Secret { - let mut data = BTreeMap::new(); - for (key, value) in fields { - data.insert(key.to_string(), ByteString(value.to_vec())); - } - - corev1::Secret { - data: Some(data), - ..Default::default() - } -} - -fn assert_oversized_upstream_body_hidden(err: RustfsClientError) { - assert_eq!( - err.to_string(), - format!( - "upstream returned 502 Bad Gateway: response body exceeded {MAX_UPSTREAM_ERROR_BODY_BYTES} bytes" - ) - ); -} +use super::tls_tenant_base_url; #[test] fn tls_tenant_base_url_uses_custom_cluster_domain() { @@ -130,1081 +29,3 @@ fn tls_tenant_base_url_uses_custom_cluster_domain() { "https://prod-rustfs-hl.mse.svc.k8s.mse.cloud:9000" ); } - -#[test] -fn extract_credentials_reports_missing_access_key() { - let secret = secret_with_fields(vec![("secretkey", b"sekret")]); - - let err = extract_credentials(secret.data.as_ref()).expect_err("expected missing access key"); - assert!(matches!( - err, - RustfsClientError::MissingCredentialKey { key: "accesskey" } - )); -} - -#[test] -fn extract_credentials_reports_non_utf8_access_key() { - let secret = secret_with_fields(vec![("accesskey", &[0xff, 0xfe]), ("secretkey", b"sekret")]); - - let err = extract_credentials(secret.data.as_ref()).expect_err("expected invalid utf8"); - assert!(matches!( - err, - RustfsClientError::InvalidCredentialValue { key: "accesskey" } - )); -} - -#[test] -fn extract_credentials_reports_missing_secret_key() { - let secret = secret_with_fields(vec![("accesskey", b"access")]); - - let err = extract_credentials(secret.data.as_ref()).expect_err("expected missing secret key"); - assert!(matches!( - err, - RustfsClientError::MissingCredentialKey { key: "secretkey" } - )); -} - -#[test] -fn extract_credentials_reports_non_utf8_secret_key() { - let secret = secret_with_fields(vec![("accesskey", b"access"), ("secretkey", &[0xff, 0xfe])]); - - let err = extract_credentials(secret.data.as_ref()).expect_err("expected invalid utf8"); - assert!(matches!( - err, - RustfsClientError::InvalidCredentialValue { key: "secretkey" } - )); -} - -#[test] -fn extract_credentials_reports_empty_secret_key() { - let secret = secret_with_fields(vec![("accesskey", b"abc"), ("secretkey", b"")]); - - let err = extract_credentials(secret.data.as_ref()).expect_err("expected empty secret key"); - assert!(matches!( - err, - RustfsClientError::EmptyCredentialValue { key: "secretkey" } - )); -} - -#[test] -fn parse_assume_role_xml_success_and_failure() { - let body_ok = "AKISECTOKEN2026-01-01T00:00:00Z"; - let parsed = - parse_assume_role_response(body_ok).expect("valid assume role response should parse"); - - assert_eq!(parsed.access_key_id, "AKI"); - assert_eq!(parsed.secret_access_key, "SEC"); - assert_eq!(parsed.session_token, "TOKEN"); - assert_eq!(parsed.expiration, "2026-01-01T00:00:00Z"); - - assert!(parse_assume_role_response("").is_none()); -} - -#[test] -fn unexpected_status_includes_upstream_xml_error_summary() { - let err = RustfsClientError::unexpected_status_with_body( - StatusCode::BAD_REQUEST, - r#"InvalidRequestinvalid resource: unknown "*"abc"#, - ); - - let message = err.to_string(); - assert_eq!( - message, - r#"upstream returned 400 Bad Request: InvalidRequest: invalid resource: unknown "*""# - ); - assert!(!message.contains("")); -} - -#[test] -fn unexpected_status_includes_upstream_json_error_summary() { - let err = RustfsClientError::unexpected_status_with_body( - StatusCode::BAD_REQUEST, - r#"{"code":"InvalidRequest","message":"policy Resource must use ARN form"}"#, - ); - - assert_eq!( - err.to_string(), - "upstream returned 400 Bad Request: InvalidRequest: policy Resource must use ARN form" - ); -} - -#[test] -fn unexpected_status_redacts_sensitive_upstream_error_summary() { - let err = RustfsClientError::unexpected_status_with_body( - StatusCode::BAD_REQUEST, - r#"{"code":"InvalidRequest","message":"secretkey: SK_TEST clientSecret: oidc-secret SecretAccessKey: SK_STS AccessKeyId: AKIA_STS SK_XML AKIA_XML"}"#, - ); - - let message = err.to_string(); - assert!(message.contains("secretkey: ")); - assert!(message.contains("clientSecret: ")); - assert!(message.contains("SecretAccessKey: ")); - assert!(message.contains("AccessKeyId: ")); - assert!(message.contains("")); - assert!(message.contains("")); - assert!(!message.contains("SK_TEST")); - assert!(!message.contains("oidc-secret")); - assert!(!message.contains("SK_STS")); - assert!(!message.contains("AKIA_STS")); - assert!(!message.contains("SK_XML")); - assert!(!message.contains("AKIA_XML")); -} - -#[test] -fn unexpected_status_hides_truncated_unstructured_response_body() { - let retained_body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES); - let err = RustfsClientError::unexpected_status_with_limited_body( - StatusCode::BAD_GATEWAY, - &retained_body, - true, - ); - - assert_eq!( - err.to_string(), - format!( - "upstream returned 502 Bad Gateway: response body exceeded {MAX_UPSTREAM_ERROR_BODY_BYTES} bytes" - ) - ); -} - -#[tokio::test] -async fn unexpected_response_preserves_exact_limit_unstructured_response_body() { - let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES); - let router = Router::new().route( - ADD_USER_PATH, - put(move || { - let body = body.clone(); - async move { (StatusCode::BAD_GATEWAY, body) } - }), - ); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let err = client - .add_user("app-user", "secret123") - .await - .expect_err("exact limit body should still report the retained body"); - - let message = err.to_string(); - assert!(message.contains("upstream returned 502 Bad Gateway")); - assert!(!message.contains("response body exceeded")); - - server.abort(); -} - -#[tokio::test] -async fn unexpected_response_hides_over_limit_unstructured_response_body() { - let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES + 1); - let router = Router::new().route( - ADD_USER_PATH, - put(move || { - let body = body.clone(); - async move { (StatusCode::BAD_GATEWAY, body) } - }), - ); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let err = client - .add_user("app-user", "secret123") - .await - .expect_err("oversized body should be hidden"); - - assert_oversized_upstream_body_hidden(err); - - server.abort(); -} - -#[derive(Clone, Default)] -struct Capture { - method: Arc>, - path: Arc>, - query: Arc>, - body: Arc>, - host: Arc>, - content_type: Arc>, - amz_date: Arc>, - payload_hash: Arc>, - authorization: Arc>, - object_lock_header: Arc>, -} - -#[derive(Debug)] -struct CapturedRequest { - method: String, - path: String, - query: String, - body: String, - host: String, - content_type: String, - amz_date: String, - payload_hash: String, - authorization: String, -} - -impl Capture { - async fn request(&self) -> CapturedRequest { - CapturedRequest { - method: self.method.lock().await.clone(), - path: self.path.lock().await.clone(), - query: self.query.lock().await.clone(), - body: self.body.lock().await.clone(), - host: self.host.lock().await.clone(), - content_type: self.content_type.lock().await.clone(), - amz_date: self.amz_date.lock().await.clone(), - payload_hash: self.payload_hash.lock().await.clone(), - authorization: self.authorization.lock().await.clone(), - } - } -} - -fn request_header(req: &Request, name: &str) -> String { - req.headers() - .get(name) - .and_then(|value| value.to_str().ok()) - .unwrap_or("") - .to_string() -} - -async fn capture_signed_request(capture: &Capture, req: Request) { - let method = req.method().as_str().to_string(); - let path = req.uri().path().to_string(); - let query = req.uri().query().unwrap_or("").to_string(); - let host = request_header(&req, "host"); - let content_type = request_header(&req, "content-type"); - let amz_date = request_header(&req, "x-amz-date"); - let payload_hash = request_header(&req, "x-amz-content-sha256"); - let authorization = request_header(&req, "authorization"); - let body = axum::body::to_bytes(req.into_body(), usize::MAX) - .await - .unwrap(); - let body = String::from_utf8(body.to_vec()).unwrap(); - - *capture.method.lock().await = method; - *capture.path.lock().await = path; - *capture.query.lock().await = query; - *capture.body.lock().await = body; - *capture.host.lock().await = host; - *capture.content_type.lock().await = content_type; - *capture.amz_date.lock().await = amz_date; - *capture.payload_hash.lock().await = payload_hash; - *capture.authorization.lock().await = authorization; -} - -fn assert_sigv4_matches_wire(request: &CapturedRequest, service: &str) { - let calculated_payload_hash = sha256_hex(request.body.as_bytes()); - assert_eq!(request.payload_hash, calculated_payload_hash); - - let signed_header_names = if request.content_type.is_empty() { - "host;x-amz-content-sha256;x-amz-date" - } else { - "content-type;host;x-amz-content-sha256;x-amz-date" - }; - let mut canonical_headers = String::new(); - if !request.content_type.is_empty() { - canonical_headers.push_str("content-type:"); - canonical_headers.push_str(request.content_type.trim()); - canonical_headers.push('\n'); - } - canonical_headers.push_str("host:"); - canonical_headers.push_str(request.host.trim()); - canonical_headers.push_str("\nx-amz-content-sha256:"); - canonical_headers.push_str(request.payload_hash.trim()); - canonical_headers.push_str("\nx-amz-date:"); - canonical_headers.push_str(request.amz_date.trim()); - canonical_headers.push('\n'); - - let canonical_request = format!( - "{}\n{}\n{}\n{}\n{}\n{}", - request.method, - request.path, - request.query, - canonical_headers, - signed_header_names, - request.payload_hash - ); - let date_stamp = request - .amz_date - .get(..8) - .expect("x-amz-date must start with YYYYMMDD"); - let credential_scope = format!("{date_stamp}/{TEST_REGION}/{service}/aws4_request"); - let string_to_sign = format!( - "AWS4-HMAC-SHA256\n{}\n{}\n{}", - request.amz_date, - credential_scope, - sha256_hex(canonical_request.as_bytes()) - ); - let signing_key = - derive_signing_key(TEST_SECRET_KEY, date_stamp, TEST_REGION, service).unwrap(); - let signature = hmac_sha256_hex(&signing_key, &string_to_sign).unwrap(); - let expected_authorization = format!( - "AWS4-HMAC-SHA256 Credential={TEST_ACCESS_KEY}/{credential_scope}, SignedHeaders={signed_header_names}, Signature={signature}" - ); - - assert_eq!(request.authorization, expected_authorization); -} - -#[tokio::test] -async fn assume_role_request_targets_root_path_and_action_is_assume_role() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new().route( - "/", - post( - move |State(c): State, req: Request| async move { - capture_signed_request(&c, req).await; - - let response = - "AKISECTOKEN2026-01-01T00:00:00Z"; - (StatusCode::OK, response) - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - - let creds = client - .assume_role(Some(r#"{"Statement":[{"Resource":"a b~+/雪"}]}"#), 3600) - .await - .unwrap(); - assert_eq!(creds.access_key_id, "AKI"); - - let request = capture.request().await; - assert_eq!(request.path, "/"); - assert_eq!( - request.body, - "Action=AssumeRole&DurationSeconds=3600&Policy=%7B%22Statement%22%3A%5B%7B%22Resource%22%3A%22a+b%7E%2B%2F%E9%9B%AA%22%7D%5D%7D&Version=2011-06-15" - ); - assert!(request.query.is_empty()); - assert_eq!(request.content_type, FORM_CONTENT_TYPE); - assert_sigv4_matches_wire(&request, STS_SIGNING_SERVICE); - - server.abort(); -} - -#[tokio::test] -async fn info_canned_policy_uses_expected_path_and_query() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new() - .route( - "/rustfs/admin/v3/info-canned-policy", - get( - move |State(c): State, req: Request| async move { - let path = req.uri().path().to_string(); - let query = req.uri().query().unwrap_or("").to_string(); - let authorization = req - .headers() - .get("authorization") - .and_then(|value| value.to_str().ok()) - .unwrap_or("") - .to_string(); - - *c.path.lock().await = path; - *c.query.lock().await = query; - *c.authorization.lock().await = authorization; - - ( - StatusCode::OK, - "{\"policy_name\":\"tenant-policy\",\"policy\":{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"allow\",\"Effect\":\"Allow\"}]}}", - ) - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - - let policy = client.get_canned_policy("tenant-policy").await.unwrap(); - let policy_value = serde_json::from_str::(&policy).unwrap(); - assert_eq!(policy_value["Version"], "2012-10-17"); - assert_eq!(policy_value["Statement"][0]["Sid"], "allow"); - - assert_eq!( - &*capture.path.lock().await, - "/rustfs/admin/v3/info-canned-policy" - ); - assert!(capture.query.lock().await.contains("name=tenant-policy")); - assert!( - capture - .authorization - .lock() - .await - .contains("/s3/aws4_request") - ); - - server.abort(); -} - -#[tokio::test] -async fn list_canned_policies_extracts_policy_document_and_canonicalizes_json() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new() - .route( - LIST_CANNED_POLICIES_PATH, - get( - move |State(c): State, req: Request| async move { - let path = req.uri().path().to_string(); - let query = req.uri().query().unwrap_or("").to_string(); - - *c.path.lock().await = path; - *c.query.lock().await = query; - - ( - StatusCode::OK, - serde_json::json!({ - "tenant-policy": { - "policy_name":"tenant-policy", - "policy":{ - "Statement": [{ - "Resource": "arn:aws:s3:::tenant", - "Effect": "Allow", - "Action": "s3:GetObject" - }], - "Version":"2012-10-17" - } - }, - "inline-policy": { - "Version": "2012-10-17", - "Statement": [{ - "Sid": "inline", - "Action": "s3:ListBucket", - "Effect": "Allow", - "Resource": ["arn:aws:s3:::tenant*"] - }] - } - }) - .to_string(), - ) - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let policies = client.list_canned_policies().await.unwrap(); - - let tenant_policy = serde_json::from_str::(&policies["tenant-policy"]).unwrap(); - assert_eq!(tenant_policy["Version"], "2012-10-17"); - assert_eq!(tenant_policy["Statement"][0]["Action"], "s3:GetObject"); - - let inline_policy = serde_json::from_str::(&policies["inline-policy"]).unwrap(); - assert_eq!(inline_policy["Version"], "2012-10-17"); - assert_eq!(inline_policy["Statement"][0]["Sid"], "inline"); - assert_eq!(&*capture.path.lock().await, LIST_CANNED_POLICIES_PATH); - assert!(capture.query.lock().await.is_empty()); - - server.abort(); -} - -#[tokio::test] -async fn add_canned_policy_uses_expected_path_query_body_and_admin_signing() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new() - .route( - "/rustfs/admin/v3/add-canned-policy", - put( - move |State(c): State, req: Request| async move { - let path = req.uri().path().to_string(); - let query = req.uri().query().unwrap_or("").to_string(); - let authorization = req - .headers() - .get("authorization") - .and_then(|value| value.to_str().ok()) - .unwrap_or("") - .to_string(); - let body_bytes = axum::body::to_bytes(req.into_body(), usize::MAX) - .await - .unwrap(); - let body = String::from_utf8(body_bytes.to_vec()).unwrap(); - - *c.path.lock().await = path; - *c.query.lock().await = query; - *c.authorization.lock().await = authorization; - *c.body.lock().await = body; - - StatusCode::OK - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let policy = r#"{"Version":"2012-10-17","Statement":[]}"#; - - client - .add_canned_policy("tenant-policy", policy) - .await - .unwrap(); - - assert_eq!( - &*capture.path.lock().await, - "/rustfs/admin/v3/add-canned-policy" - ); - assert!(capture.query.lock().await.contains("name=tenant-policy")); - assert_eq!(&*capture.body.lock().await, policy); - assert!( - capture - .authorization - .lock() - .await - .contains("/s3/aws4_request") - ); - - server.abort(); -} - -#[tokio::test] -async fn add_canned_policy_reports_upstream_policy_parse_error() { - let router = Router::new().route( - "/rustfs/admin/v3/add-canned-policy", - put(|| async { - ( - StatusCode::BAD_REQUEST, - r#"InvalidRequestinvalid resource: unknown "*""#, - ) - }), - ); - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let policy = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}"#; - let err = client - .add_canned_policy("tenant-policy", policy) - .await - .expect_err("invalid RustFS policy should include upstream parse details"); - - let message = err.to_string(); - assert!(message.contains("upstream returned 400 Bad Request")); - assert!(message.contains(r#"InvalidRequest: invalid resource: unknown "*""#)); - assert!(!message.contains("")); - - server.abort(); -} - -#[tokio::test] -async fn server_info_uses_expected_path_and_parses_wrapped_health_fields() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new() - .route( - SERVER_INFO_PATH, - get( - move |State(c): State, req: Request| async move { - let path = req.uri().path().to_string(); - let authorization = req - .headers() - .get("authorization") - .and_then(|value| value.to_str().ok()) - .unwrap_or("") - .to_string(); - - *c.path.lock().await = path; - *c.authorization.lock().await = authorization; - - ( - StatusCode::OK, - serde_json::json!({ - "info": { - "usage": {"size": 42}, - "backend": { - "onlineDisks": 3, - "offlineDisks": 1, - "standardSCParity": 2, - "totalSets": [1], - "totalDrivesPerSet": [4] - }, - "pools": { - "0": { - "0": { - "rawUsage": 100, - "rawCapacity": 400, - "usage": 50, - "objectsCount": 2, - "healDisks": 1 - } - } - } - }, - "admin_discovery": { - "runtimeCapabilities": "/rustfs/admin/v4/runtime/capabilities", - "clusterSnapshot": "/rustfs/admin/v4/cluster/snapshot", - "extensionsCatalog": "/rustfs/admin/v4/extensions/catalog" - }, - }) - .to_string(), - ) - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let info = client.server_info().await.unwrap(); - - let backend = info.backend.unwrap(); - assert_eq!(backend.online_disks, 3); - assert_eq!(backend.offline_disks, 1); - assert_eq!(backend.standard_sc_parity, Some(2)); - assert_eq!(info.usage.unwrap().size, 42); - assert_eq!(info.pools.unwrap()["0"]["0"].raw_capacity, 400); - assert_eq!(&*capture.path.lock().await, SERVER_INFO_PATH); - assert!( - capture - .authorization - .lock() - .await - .contains("/s3/aws4_request") - ); - - server.abort(); -} - -#[tokio::test] -async fn list_pools_parses_current_rustfs_pool_shape() { - let router = Router::new().route( - POOLS_LIST_PATH, - get(|| async { - ( - StatusCode::OK, - r#"[{"id":1,"cmdline":"http://tenant-pool-a-{0...3}.tenant-hl.ns.svc.cluster.local:9000/data/rustfs{0...3}","lastUpdate":"2026-05-20T00:00:00Z","totalSize":100,"currentSize":50,"usedSize":25,"used":25.0,"status":"running","decommissionInfo":{"startTime":"2026-05-20T00:00:00Z","complete":false,"failed":false,"canceled":false,"objectsDecommissioned":7,"objectsDecommissionedFailed":1,"bytesDecommissioned":9,"bytesDecommissionedFailed":2}}]"#, - ) - }), - ); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - - let pools = client.list_pools().await.unwrap(); - - assert_eq!(pools[0].id, 1); - assert_eq!(pools[0].status, "running"); - assert_eq!( - pools[0] - .decommission - .as_ref() - .and_then(|info| info.objects_decommissioned), - Some(7) - ); - - server.abort(); -} - -#[tokio::test] -async fn pool_decommission_start_uses_by_id_query_and_admin_signing() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new() - .route( - POOLS_DECOMMISSION_PATH, - post( - move |State(c): State, req: Request| async move { - *c.path.lock().await = req.uri().path().to_string(); - *c.query.lock().await = req.uri().query().unwrap_or("").to_string(); - *c.authorization.lock().await = req - .headers() - .get("authorization") - .and_then(|value| value.to_str().ok()) - .unwrap_or("") - .to_string(); - - StatusCode::OK - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - - client.start_pool_decommission_by_id("1").await.unwrap(); - - assert_eq!(&*capture.path.lock().await, POOLS_DECOMMISSION_PATH); - assert_eq!(&*capture.query.lock().await, "by-id=true&pool=1"); - assert!( - capture - .authorization - .lock() - .await - .contains("/s3/aws4_request") - ); - - server.abort(); -} - -#[tokio::test] -async fn pool_status_uses_by_id_query_and_parses_decommission_info() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new() - .route( - POOLS_STATUS_PATH, - get( - move |State(c): State, req: Request| async move { - *c.path.lock().await = req.uri().path().to_string(); - *c.query.lock().await = req.uri().query().unwrap_or("").to_string(); - - ( - StatusCode::OK, - r#"{"id":1,"cmdline":"http://tenant-pool-a-{0...3}.tenant-hl.ns.svc.cluster.local:9000/data/rustfs{0...3}","lastUpdate":"2026-05-20T00:00:00Z","decommissionInfo":{"startTime":"2026-05-20T00:00:00Z","complete":true,"failed":false,"canceled":false,"objectsDecommissioned":10,"objectsDecommissionedFailed":0,"bytesDecommissioned":20,"bytesDecommissionedFailed":0}}"#, - ) - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - - let status = client.pool_status_by_id("1").await.unwrap(); - - assert_eq!(status.id, 1); - assert_eq!(&*capture.path.lock().await, POOLS_STATUS_PATH); - assert_eq!(&*capture.query.lock().await, "by-id=true&pool=1"); - assert_eq!( - status.decommission.and_then(|info| info.complete), - Some(true) - ); - - server.abort(); -} - -#[tokio::test] -async fn add_user_uses_expected_path_query_and_body() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new() - .route( - ADD_USER_PATH, - put( - move |State(c): State, req: Request| async move { - capture_signed_request(&c, req).await; - StatusCode::OK - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - client.add_user("app user~+/雪", "secret123").await.unwrap(); - - let request = capture.request().await; - assert_eq!(request.path, ADD_USER_PATH); - assert_eq!(request.query, "accessKey=app%20user~%2B%2F%E9%9B%AA"); - assert_eq!( - request.body, - r#"{"secretKey":"secret123","status":"enabled"}"# - ); - assert_eq!(request.content_type, JSON_CONTENT_TYPE); - assert_sigv4_matches_wire(&request, ADMIN_SIGNING_SERVICE); - - server.abort(); -} - -#[tokio::test] -async fn user_exists_limits_unexpected_error_response_body() { - let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES + 1); - let router = Router::new().route( - USER_INFO_PATH, - get(move || { - let body = body.clone(); - async move { (StatusCode::BAD_GATEWAY, body) } - }), - ); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let err = client - .user_exists("app-user") - .await - .expect_err("unexpected user lookup error should hide oversized body"); - - assert_oversized_upstream_body_hidden(err); - - server.abort(); -} - -#[tokio::test] -async fn set_user_policy_uses_single_authoritative_mapping_call() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new() - .route( - SET_POLICY_PATH, - put( - move |State(c): State, req: Request| async move { - *c.path.lock().await = req.uri().path().to_string(); - *c.query.lock().await = req.uri().query().unwrap_or("").to_string(); - StatusCode::OK - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - client - .set_user_policy( - "app-user", - &["app-readwrite".to_string(), "diagnostics".to_string()], - ) - .await - .unwrap(); - - assert_eq!(&*capture.path.lock().await, SET_POLICY_PATH); - assert_eq!( - &*capture.query.lock().await, - "isGroup=false&policyName=app-readwrite%2Cdiagnostics&userOrGroup=app-user" - ); - - server.abort(); -} - -#[tokio::test] -async fn set_user_policy_rejects_empty_policy_list() { - let client = RustfsAdminClient::new_with_base_url("http://127.0.0.1:1", "access", "secret"); - - let err = client - .set_user_policy("app-user", &[]) - .await - .expect_err("empty policy list should be rejected before request"); - - assert!(matches!(err, RustfsClientError::InvalidPolicyName)); -} - -#[tokio::test] -async fn bucket_object_lock_enabled_parses_enabled_response() { - let router = Router::new().route( - "/app-data", - get(|req: Request| async move { - assert_eq!(req.uri().query().unwrap_or(""), "object-lock="); - ( - StatusCode::OK, - "Enabled", - ) - }), - ); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - - assert!(client.bucket_object_lock_enabled("app-data").await.unwrap()); - - server.abort(); -} - -#[tokio::test] -async fn bucket_object_lock_enabled_limits_unexpected_error_response_body() { - let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES + 1); - let router = Router::new().route( - "/app-data", - get(move |req: Request| { - let body = body.clone(); - async move { - assert_eq!(req.uri().query().unwrap_or(""), "object-lock="); - (StatusCode::BAD_GATEWAY, body) - } - }), - ); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let err = client - .bucket_object_lock_enabled("app-data") - .await - .expect_err("unexpected object-lock error should hide oversized body"); - - assert_oversized_upstream_body_hidden(err); - - server.abort(); -} - -#[tokio::test] -async fn create_bucket_sends_object_lock_header_and_region_body() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new() - .route( - "/app-data", - put( - move |State(c): State, req: Request| async move { - *c.path.lock().await = req.uri().path().to_string(); - *c.object_lock_header.lock().await = req - .headers() - .get("x-amz-bucket-object-lock-enabled") - .and_then(|value| value.to_str().ok()) - .unwrap_or("") - .to_string(); - let body_bytes = axum::body::to_bytes(req.into_body(), usize::MAX) - .await - .unwrap(); - *c.body.lock().await = String::from_utf8(body_bytes.to_vec()).unwrap(); - StatusCode::OK - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let result = client - .create_bucket("app-data", Some("us-west-2"), true) - .await - .unwrap(); - - assert_eq!(result, CreateBucketResult::Created); - assert_eq!(&*capture.path.lock().await, "/app-data"); - assert_eq!(&*capture.object_lock_header.lock().await, "true"); - assert!( - capture - .body - .lock() - .await - .contains("us-west-2") - ); - - server.abort(); -} - -#[tokio::test] -async fn create_bucket_limits_unexpected_error_response_body() { - let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES + 1); - let router = Router::new().route( - "/app-data", - put(move || { - let body = body.clone(); - async move { (StatusCode::BAD_GATEWAY, body) } - }), - ); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let err = client - .create_bucket("app-data", None, false) - .await - .expect_err("unexpected bucket create error should hide oversized body"); - - assert_oversized_upstream_body_hidden(err); - - server.abort(); -} - -#[test] -fn extract_canned_policy_document_accepts_raw_policy_document() { - let raw_policy = - "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"raw\",\"Effect\":\"Allow\"}]}"; - - let policy = extract_canned_policy_document(raw_policy).unwrap(); - - let policy_value = serde_json::from_str::(&policy).unwrap(); - assert_eq!(policy_value["Version"], "2012-10-17"); - assert_eq!(policy_value["Statement"][0]["Sid"], "raw"); -} diff --git a/src/sts/types.rs b/src/sts/types.rs index ecf8beb8..3769171c 100644 --- a/src/sts/types.rs +++ b/src/sts/types.rs @@ -113,13 +113,7 @@ pub fn parse_sts_form( }) } -#[derive(Debug, Clone)] -pub struct StsAssumeRoleCredentials { - pub access_key_id: String, - pub secret_access_key: String, - pub session_token: String, - pub expiration: String, -} +pub use rustfs_admin::StsAssumeRoleCredentials; #[derive(Debug, Clone)] pub struct StsWebIdentityResponseContext { diff --git a/src/tenant_monitor.rs b/src/tenant_monitor.rs index 6b428e8f..00279a33 100644 --- a/src/tenant_monitor.rs +++ b/src/tenant_monitor.rs @@ -14,7 +14,10 @@ use crate::{ metrics::{self, TenantStorageMetrics}, - sts::rustfs_client::{RustfsAdminClient, RustfsServerInfo}, + sts::rustfs_client::{ + RustfsServerInfo, client_from_tenant, client_from_tls_tenant_for_sts, + load_tenant_credentials, + }, types::v1alpha1::tenant::Tenant, }; use futures::{StreamExt, stream}; @@ -167,12 +170,11 @@ async fn poll_tenant_storage( tenant: &Tenant, cluster_domain: &str, ) -> Result> { - let credentials = RustfsAdminClient::load_tenant_credentials(client, tenant).await?; + let credentials = load_tenant_credentials(client, tenant).await?; let rustfs_client = if tenant.spec.tls.as_ref().is_some_and(|tls| tls.is_enabled()) { - RustfsAdminClient::from_tls_tenant_for_sts(client, tenant, credentials, cluster_domain) - .await? + client_from_tls_tenant_for_sts(client, tenant, credentials, cluster_domain).await? } else { - RustfsAdminClient::from_tenant(tenant, credentials)? + client_from_tenant(tenant, credentials)? }; let info = rustfs_client.server_info().await?; From ef72ff26e8e6a7c92a06aa4af245370cf52ea74f Mon Sep 17 00:00:00 2001 From: benjamin fuentes Date: Tue, 4 Aug 2026 09:35:53 +0200 Subject: [PATCH 2/4] fix(cosi): make grant idempotent, reject key reuse Align DriverGrantBucketAccess with Ceph-style isolation: deterministic secrets, never rotate existing users, and return AlreadyExists when preferredAccessKey is claimed by another BucketAccess. Co-authored-by: Cursor --- .github/workflows/ci.yml | 6 +- Cargo.lock | 43 +- Cargo.toml | 6 +- Dockerfile | 9 +- Makefile | 6 +- crates/cosi-driver/Cargo.toml | 36 +- crates/cosi-driver/build.rs | 27 +- crates/cosi-driver/proto/cosi.proto | 107 +- crates/cosi-driver/src/backend.rs | 291 ++-- crates/cosi-driver/src/driver.rs | 385 ++--- crates/cosi-driver/src/lib.rs | 30 - crates/cosi-driver/src/main.rs | 175 +-- crates/cosi-driver/src/parameters.rs | 255 ++-- crates/cosi-driver/src/policy.rs | 126 -- crates/rustfs-admin/Cargo.toml | 23 - crates/rustfs-admin/src/client.rs | 470 ------- crates/rustfs-admin/src/credentials.rs | 23 - crates/rustfs-admin/src/helpers.rs | 185 --- crates/rustfs-admin/src/lib.rs | 42 - crates/rustfs-admin/src/sanitize.rs | 353 ----- crates/rustfs-admin/src/tests.rs | 1192 ---------------- deploy/rustfs-operator/README.md | 21 - deploy/rustfs-operator/templates/NOTES.txt | 11 - deploy/rustfs-operator/templates/_helpers.tpl | 20 - .../templates/cosi-deployment.yaml | 88 -- .../rustfs-operator/templates/cosi-rbac.yaml | 59 - .../templates/cosi-serviceaccount.yaml | 21 - deploy/rustfs-operator/values.yaml | 64 - docs/operator-user-guide.md | 60 +- e2e/Cargo.lock | 18 +- e2e/tests/sts_functional.rs | 9 +- examples/README.md | 1 - examples/cosi/awscli-pod.yaml | 20 - examples/cosi/bucketaccess.yaml | 10 - examples/cosi/bucketaccessclass.yaml | 13 - examples/cosi/bucketclaim.yaml | 9 - examples/cosi/bucketclass.yaml | 21 - src/reconcile/pool_lifecycle.rs | 14 +- src/reconcile/provisioning.rs | 17 +- .../rustfs-admin/src => src/sts}/admin_ops.rs | 177 ++- .../rustfs-admin/src => src/sts}/core_ops.rs | 13 +- src/sts/helpers.rs | 211 ++- .../rustfs-admin/src => src/sts}/pool_ops.rs | 4 +- src/sts/rustfs_client.rs | 687 +++++++-- .../rustfs-admin/src => src/sts}/s3_ops.rs | 15 +- src/sts/server.rs | 8 +- .../rustfs-admin/src => src/sts}/sts_ops.rs | 7 +- src/sts/tests.rs | 1240 ++++++++++++++++- src/sts/types.rs | 8 +- src/tenant_monitor.rs | 12 +- 50 files changed, 2771 insertions(+), 3877 deletions(-) delete mode 100644 crates/cosi-driver/src/lib.rs delete mode 100644 crates/cosi-driver/src/policy.rs delete mode 100644 crates/rustfs-admin/Cargo.toml delete mode 100644 crates/rustfs-admin/src/client.rs delete mode 100644 crates/rustfs-admin/src/credentials.rs delete mode 100644 crates/rustfs-admin/src/helpers.rs delete mode 100644 crates/rustfs-admin/src/lib.rs delete mode 100644 crates/rustfs-admin/src/sanitize.rs delete mode 100644 crates/rustfs-admin/src/tests.rs delete mode 100644 deploy/rustfs-operator/templates/cosi-deployment.yaml delete mode 100644 deploy/rustfs-operator/templates/cosi-rbac.yaml delete mode 100644 deploy/rustfs-operator/templates/cosi-serviceaccount.yaml delete mode 100644 examples/cosi/awscli-pod.yaml delete mode 100644 examples/cosi/bucketaccess.yaml delete mode 100644 examples/cosi/bucketaccessclass.yaml delete mode 100644 examples/cosi/bucketclaim.yaml delete mode 100644 examples/cosi/bucketclass.yaml rename {crates/rustfs-admin/src => src/sts}/admin_ops.rs (79%) rename {crates/rustfs-admin/src => src/sts}/core_ops.rs (93%) rename {crates/rustfs-admin/src => src/sts}/pool_ops.rs (97%) rename {crates/rustfs-admin/src => src/sts}/s3_ops.rs (92%) rename {crates/rustfs-admin/src => src/sts}/sts_ops.rs (95%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 155da498..e1680f2c 100755 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,14 +101,14 @@ jobs: - name: Run tests run: | - cargo nextest run --workspace --no-tests pass - cargo test --workspace --doc + cargo nextest run --all --no-tests pass + cargo test --all --doc - name: Check code formatting run: cargo fmt --all --check - name: Run clippy lints - run: cargo clippy --workspace --all-features -- -D warnings + run: cargo clippy --all-features -- -D warnings - name: Check Rust-native e2e harness run: make e2e-check diff --git a/Cargo.lock b/Cargo.lock index 6115e525..0835f114 100755 --- a/Cargo.lock +++ b/Cargo.lock @@ -473,26 +473,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "cosi-driver" -version = "0.1.0" -dependencies = [ - "hex", - "k8s-openapi", - "kube", - "prost", - "protoc-bin-vendored", - "rustfs-admin", - "sha2", - "snafu", - "tokio", - "tokio-stream", - "tonic", - "tonic-build", - "tracing", - "tracing-subscriber", -] - [[package]] name = "cpufeatures" version = "0.2.17" @@ -1718,6 +1698,7 @@ dependencies = [ "const-str", "futures", "hex", + "hmac", "hostname", "http", "http-body-util", @@ -1727,8 +1708,8 @@ dependencies = [ "kube", "kube-leader-election", "rcgen", + "reqwest", "ring", - "rustfs-admin", "rustls", "rustls-pemfile", "rustls-webpki", @@ -1749,6 +1730,7 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", + "url", "utoipa", "utoipa-swagger-ui", ] @@ -2350,19 +2332,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] -name = "rustfs-admin" +name = "rustfs-cosi-driver" version = "0.1.0" dependencies = [ - "axum", - "chrono", "hex", - "hmac", - "reqwest", + "k8s-openapi", + "kube", + "operator", + "prost", + "protoc-bin-vendored", + "rustls", "serde", "serde_json", "sha2", + "thiserror 2.0.17", "tokio", - "url", + "tokio-stream", + "tonic", + "tonic-build", + "tracing", + "tracing-subscriber", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f269619e..db0d7e1e 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,13 +31,15 @@ rustls-pemfile = "2.2.0" webpki = { package = "rustls-webpki", version = "0.103" } rcgen = "0.13" sha2 = "0.10" +hmac = "0.12" hex = "0.4" base64 = "0.22" ring = "0.17" +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +url = "2.5" shadow-rs = "1.5.0" snafu = { version = "0.8.9", features = ["futures"] } kube-leader-election = { path = "crates/leader-election" } -rustfs-admin = { path = "crates/rustfs-admin" } hostname = "0.4" # Console dependencies @@ -61,7 +63,7 @@ shadow-rs = { version = "1.5.0", features = ["build"] } unused_variables = "allow" [workspace] -members = ["crates/leader-election", "crates/rustfs-admin", "crates/cosi-driver"] +members = ["crates/leader-election", "crates/cosi-driver"] [lints.clippy] unwrap_used = "deny" diff --git a/Dockerfile b/Dockerfile index 612cdd96..a598152e 100755 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,9 @@ ARG PNPM_VERSION=10.28.1 # Shared Cargo settings for slow / flaky networks (applies to all Rust stages) FROM ${RUST_BUILD_IMAGE} AS rust-base +RUN apt-get update \ + && apt-get install -y --no-install-recommends protobuf-compiler \ + && rm -rf /var/lib/apt/lists/* RUN mkdir -p /usr/local/cargo && \ printf '%s\n' \ '[http]' \ @@ -45,15 +48,15 @@ FROM rust-base AS cacher COPY --from=cargo-chef-installer /usr/local/cargo/bin/cargo-chef /usr/local/cargo/bin/cargo-chef WORKDIR /app COPY --from=planner /app/recipe.json recipe.json -RUN cargo chef cook --release --recipe-path recipe.json +RUN cargo chef cook --release --workspace --recipe-path recipe.json -# Stage 3: Build the binary +# Stage 3: Build the binaries (operator + COSI driver) FROM rust-base AS builder WORKDIR /app COPY . . COPY --from=cacher /app/target target COPY --from=cacher /usr/local/cargo /usr/local/cargo -RUN cargo build --release -p operator -p cosi-driver +RUN cargo build --release -p operator -p rustfs-cosi-driver # Stage 4: Build the static Console frontend FROM ${NODE_BUILD_IMAGE} AS console-web-builder diff --git a/Makefile b/Makefile index 8b8c4364..e0f4f5eb 100644 --- a/Makefile +++ b/Makefile @@ -60,11 +60,11 @@ fmt-check: # Run clippy checks. clippy: - cargo clippy --workspace --all-features -- -D warnings + cargo clippy --all-features -- -D warnings # Run Rust tests. test: - cargo test --workspace + cargo test --all # Run frontend ESLint checks. Run pnpm install in console-web first. console-lint: @@ -84,7 +84,7 @@ console-fmt-check: # Build the project. build: - cargo build --release --workspace + cargo build --release # Rust-native e2e harness (live-first, dedicated Kind) E2E_MANIFEST ?= e2e/Cargo.toml diff --git a/crates/cosi-driver/Cargo.toml b/crates/cosi-driver/Cargo.toml index 31afb2ca..5ef08481 100644 --- a/crates/cosi-driver/Cargo.toml +++ b/crates/cosi-driver/Cargo.toml @@ -1,9 +1,8 @@ [package] -name = "cosi-driver" +name = "rustfs-cosi-driver" version = "0.1.0" edition = "2024" license = "Apache-2.0" -description = "RustFS Container Object Storage Interface (COSI) v1alpha1 driver" publish = false [[bin]] @@ -11,29 +10,22 @@ name = "rustfs-cosi-driver" path = "src/main.rs" [dependencies] -hex = "0.4" -k8s-openapi = { version = "0.26.1", features = ["v1_30"] } -kube = { version = "2.0.1", features = ["client", "rustls-tls"] } -prost = "0.13" -rustfs-admin = { path = "../rustfs-admin" } -sha2 = "0.10" -snafu = { version = "0.8.9", features = ["futures"] } +operator = { path = "../.." } tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs", "signal", "net"] } tokio-stream = { version = "0.1", features = ["net"] } tonic = "0.12" -tracing = "0.1.44" -tracing-subscriber = { version = "0.3.22", features = ["env-filter"] } +prost = "0.13" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +kube = { version = "2.0.1", features = ["client", "rustls-tls"] } +k8s-openapi = { version = "0.26.1", features = ["v1_30"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +sha2 = "0.10" +hex = "0.4" +thiserror = "2" +rustls = { version = "0.23", default-features = false, features = ["ring"] } [build-dependencies] -protoc-bin-vendored = "3" tonic-build = "0.12" - -[dev-dependencies] -tokio = { version = "1.49.0", features = ["rt", "macros"] } - -[lints.rust] -unused_variables = "allow" - -[lints.clippy] -unwrap_used = "deny" -expect_used = "deny" +protoc-bin-vendored = "3" diff --git a/crates/cosi-driver/build.rs b/crates/cosi-driver/build.rs index 60df6919..cce675e1 100644 --- a/crates/cosi-driver/build.rs +++ b/crates/cosi-driver/build.rs @@ -1,24 +1,13 @@ -// Copyright 2025 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - fn main() -> Result<(), Box> { - let protoc = protoc_bin_vendored::protoc_bin_path()?; - // SAFETY: build scripts run single-threaded before compilation. - unsafe { - std::env::set_var("PROTOC", protoc); + // Prefer PATH `protoc` (Dockerfile installs protobuf-compiler); fall back to vendored. + if std::env::var_os("PROTOC").is_none() + && let Ok(protoc) = protoc_bin_vendored::protoc_bin_path() + { + // SAFETY: build script is single-threaded before codegen. + unsafe { + std::env::set_var("PROTOC", protoc); + } } - tonic_build::configure() .build_server(true) .build_client(false) diff --git a/crates/cosi-driver/proto/cosi.proto b/crates/cosi-driver/proto/cosi.proto index e14133e5..d26273d2 100644 --- a/crates/cosi-driver/proto/cosi.proto +++ b/crates/cosi-driver/proto/cosi.proto @@ -1,78 +1,43 @@ -// Code generated by make; DO NOT EDIT. syntax = "proto3"; package cosi.v1alpha1; +option go_package = "sigs.k8s.io/container-object-storage-interface/proto;cosi"; service Identity { - // This call is meant to retrieve the unique provisioner Identity. - // This identity will have to be set in BucketClaim.DriverName field in order to invoke this specific provisioner. rpc DriverGetInfo (DriverGetInfoRequest) returns (DriverGetInfoResponse) {} } service Provisioner { - // This call is made to create the bucket in the backend. - // This call is idempotent - // 1. If a bucket that matches both name and parameters already exists, then OK (success) must be returned. - // 2. If a bucket by same name, but different parameters is provided, then the appropriate error code ALREADY_EXISTS must be returned. rpc DriverCreateBucket (DriverCreateBucketRequest) returns (DriverCreateBucketResponse) {} - // This call is made to delete the bucket in the backend. - // If the bucket has already been deleted, then no error should be returned. rpc DriverDeleteBucket (DriverDeleteBucketRequest) returns (DriverDeleteBucketResponse) {} - - // This call grants access to an account. The account_name in the request shall be used as a unique identifier to create credentials. - // The account_id returned in the response will be used as the unique identifier for deleting this access when calling DriverRevokeBucketAccess. rpc DriverGrantBucketAccess (DriverGrantBucketAccessRequest) returns (DriverGrantBucketAccessResponse); - // This call revokes all access to a particular bucket from a principal. rpc DriverRevokeBucketAccess (DriverRevokeBucketAccessRequest) returns (DriverRevokeBucketAccessResponse); } -// S3SignatureVersion is the version of the signing algorithm for all s3 requests enum S3SignatureVersion { UnknownSignature = 0; - // S3V2, Signature version v2 S3V2 = 1; - // S3V4, Signature version v4 S3V4 = 2; } -enum AnonymousBucketAccessMode { - UnknownBucketAccessMode = 0; - // Default, disallow uncredentialed access to the backend storage. - Private = 1; - // Read only, uncredentialed users can call ListBucket and GetObject. - ReadOnly = 2; - // Write only, uncredentialed users can only call PutObject. - WriteOnly = 3; - // Read/Write, uncredentialed users can read objects as well as PutObject. - ReadWrite = 4; -} - enum AuthenticationType { UnknownAuthenticationType = 0; - // Default, KEY based authentication. Key = 1; - // Storageaccount based authentication. IAM = 2; } message S3 { - // region denotes the geographical region where the S3 server is running string region = 1; - // signature_version denotes the signature version for signing all s3 requests S3SignatureVersion signature_version = 2; } message AzureBlob { - // storage_account is the id of the azure storage account string storage_account = 1; } message GCS { - // private_key_name denotes the name of the private key in the storage backend string private_key_name = 1; - // project_id denotes the name of the project id in the storage backend string project_id = 2; - // service_account denotes the name of the service account in the storage backend string service_account = 3; } @@ -85,112 +50,48 @@ message Protocol { } message CredentialDetails { - // map of the details in the secrets for the protocol string map secrets = 1; } -message DriverGetInfoRequest { - // Intentionally left blank -} +message DriverGetInfoRequest {} message DriverGetInfoResponse { - // This field is REQUIRED - // The name MUST follow domain name notation format - // (https://tools.ietf.org/html/rfc1035#section-2.3.1). It SHOULD - // include the plugin's host company name and the plugin name, - // to minimize the possibility of collisions. It MUST be 63 - // characters or less, beginning and ending with an alphanumeric - // character () with dashes (-), dots (.), and - // alphanumerics between. string name = 1; } message DriverCreateBucketRequest { - // This field is REQUIRED - // name specifies the name of the bucket that should be created. string name = 1; - - // This field is OPTIONAL - // The caller should treat the values in parameters as opaque. - // The receiver is responsible for parsing and validating the values. map parameters = 2; } message DriverCreateBucketResponse { - // bucket_id returned here is expected to be the globally unique - // identifier for the bucket in the object storage provider. string bucket_id = 1; - - // bucket_info returned here stores the data specific to the - // bucket required by the object storage provider to connect to the bucket. Protocol bucket_info = 2; } message DriverDeleteBucketRequest { - // This field is REQUIRED - // bucket_id is a globally unique identifier for the bucket - // in the object storage provider string bucket_id = 1; - - // This field is OPTIONAL - // The caller should treat the values in delete_context as opaque. - // The receiver is responsible for parsing and validating the values. map delete_context = 2; } -message DriverDeleteBucketResponse { - // Intentionally left blank -} +message DriverDeleteBucketResponse {} message DriverGrantBucketAccessRequest { - // This field is REQUIRED - // bucket_id is a globally unique identifier for the bucket - // in the object storage provider string bucket_id = 1; - - // This field is REQUIRED - // name field is used to define the name of the bucket access object. string name = 2; - - // This field is REQUIRED - // Requested authentication type for the bucket access. - // Supported authentication types are KEY or IAM. AuthenticationType authentication_type = 3; - - // This field is OPTIONAL - // The caller should treat the values in parameters as opaque. - // The receiver is responsible for parsing and validating the values. map parameters = 4; } message DriverGrantBucketAccessResponse { - // This field is REQUIRED - // This is the account_id that is being provided access. This will - // be required later to revoke access. string account_id = 1; - - // This field is REQUIRED - // Credentials supplied for accessing the bucket ex: aws access key id and secret, etc. map credentials = 2; } message DriverRevokeBucketAccessRequest { - // This field is REQUIRED - // bucket_id is a globally unique identifier for the bucket - // in the object storage provider. string bucket_id = 1; - - // This field is REQUIRED - // This is the account_id that is having its access revoked. string account_id = 2; - - // This field is OPTIONAL - // The caller should treat the values in revoke_access_context as opaque. - // The receiver is responsible for parsing and validating the values. map revoke_access_context = 3; } -message DriverRevokeBucketAccessResponse { - // Intentionally left blank -} - +message DriverRevokeBucketAccessResponse {} diff --git a/crates/cosi-driver/src/backend.rs b/crates/cosi-driver/src/backend.rs index 93aaa6eb..b83e8da8 100644 --- a/crates/cosi-driver/src/backend.rs +++ b/crates/cosi-driver/src/backend.rs @@ -1,199 +1,146 @@ -// Copyright 2025 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//! Admin credential lookup + RustFS admin client construction. -//! Kubernetes Secret / ConfigMap loading and RustFS admin client construction. - -use k8s_openapi::ByteString; use k8s_openapi::api::core::v1::{ConfigMap, Secret}; use kube::{Api, Client}; -use rustfs_admin::{RustfsAdminClient, RustfsClientError, RustfsCredentials}; -use snafu::{ResultExt, Snafu}; -use std::collections::BTreeMap; +use operator::sts::rustfs_client::RustfsAdminClient; +use thiserror::Error; +use tracing::info; use crate::parameters::BackendParameters; -fn box_kube(err: kube::Error) -> Box { - Box::new(err) -} - -#[derive(Debug, Snafu)] +#[derive(Debug, Error)] pub enum BackendError { - #[snafu(display("failed to create kubernetes client: {source}"))] - KubeClient { source: Box }, - #[snafu(display("failed to read Secret {namespace}/{name}: {source}"))] + #[error("failed to read Secret {namespace}/{name}: {source}")] SecretLookup { namespace: String, name: String, - source: Box, + #[source] + source: kube::Error, }, - #[snafu(display("failed to read ConfigMap {namespace}/{name}: {source}"))] + #[error("secret key missing: {0}")] + MissingSecretKey(&'static str), + #[error("secret key empty: {0}")] + EmptySecretKey(&'static str), + #[error("secret key is not valid utf8: {0}")] + InvalidSecretKey(&'static str), + #[error("failed to read ConfigMap {namespace}/{name}: {source}")] ConfigMapLookup { namespace: String, name: String, - source: Box, - }, - #[snafu(display("Secret {namespace}/{name} missing key `{key}`"))] - MissingSecretKey { - namespace: String, - name: String, - key: &'static str, - }, - #[snafu(display("Secret {namespace}/{name} key `{key}` is not valid UTF-8"))] - InvalidSecretKey { - namespace: String, - name: String, - key: &'static str, + #[source] + source: kube::Error, }, - #[snafu(display("Secret {namespace}/{name} key `{key}` is empty"))] - EmptySecretKey { - namespace: String, - name: String, - key: &'static str, - }, - #[snafu(display("ConfigMap {namespace}/{name} missing CA data key"))] - MissingCaData { namespace: String, name: String }, - #[snafu(display("failed to build RustFS admin client: {source}"))] - ClientBuild { source: RustfsClientError }, -} - -#[derive(Clone)] -pub struct BackendFactory { - kube: Client, + #[error("configmap key missing: {0}")] + MissingCaData(&'static str), + #[error(transparent)] + ClientBuild(#[from] operator::sts::rustfs_client::RustfsClientError), } -impl BackendFactory { - pub async fn try_default() -> Result { - let kube = Client::try_default() - .await - .map_err(box_kube) - .context(KubeClientSnafu)?; - Ok(Self { kube }) - } - - #[cfg(test)] - pub fn from_client(kube: Client) -> Self { - Self { kube } - } - - pub async fn admin_client( - &self, - params: &BackendParameters, - ) -> Result { - let credentials = self - .load_credentials(¶ms.secret_namespace, ¶ms.secret_name) - .await?; - - match ( - params.tls_ca_configmap_name.as_deref(), - params.tls_ca_configmap_namespace.as_deref(), - ) { - (Some(name), Some(namespace)) => { - let ca_pem = self.load_ca_pem(namespace, name).await?; - RustfsAdminClient::new_with_base_url_and_ca_pem( - params.endpoint.clone(), - credentials.access_key, - credentials.secret_key, - &ca_pem, - ) - .context(ClientBuildSnafu) +#[allow(clippy::result_large_err)] +fn secret_value<'a>(secret: &'a Secret, keys: &[&'static str]) -> Result<&'a str, BackendError> { + let data = secret + .data + .as_ref() + .ok_or(BackendError::MissingSecretKey(keys[0]))?; + for key in keys { + if let Some(bytes) = data.get(*key) { + let value = + std::str::from_utf8(&bytes.0).map_err(|_| BackendError::InvalidSecretKey(key))?; + if value.is_empty() { + return Err(BackendError::EmptySecretKey(key)); } - _ => Ok(RustfsAdminClient::new_with_base_url( - params.endpoint.clone(), - credentials.access_key, - credentials.secret_key, - )), + return Ok(value); } } + Err(BackendError::MissingSecretKey(keys[0])) +} - async fn load_credentials( - &self, - namespace: &str, - name: &str, - ) -> Result { - let api: Api = Api::namespaced(self.kube.clone(), namespace); - let secret = api - .get(name) - .await - .map_err(box_kube) - .context(SecretLookupSnafu { - namespace: namespace.to_string(), - name: name.to_string(), - })?; - let data = secret.data.as_ref(); - Ok(RustfsCredentials { - access_key: secret_value(data, namespace, name, "accesskey")?, - secret_key: secret_value(data, namespace, name, "secretkey")?, - }) - } +pub async fn admin_client_from_params( + kube: &Client, + params: &BackendParameters, +) -> Result { + let secrets: Api = + Api::namespaced(kube.clone(), ¶ms.object_store_user_secret_namespace); + let secret = secrets + .get(¶ms.object_store_user_secret_name) + .await + .map_err(|source| BackendError::SecretLookup { + namespace: params.object_store_user_secret_namespace.clone(), + name: params.object_store_user_secret_name.clone(), + source, + })?; - async fn load_ca_pem(&self, namespace: &str, name: &str) -> Result, BackendError> { - let api: Api = Api::namespaced(self.kube.clone(), namespace); - let cm = api - .get(name) - .await - .map_err(box_kube) - .context(ConfigMapLookupSnafu { - namespace: namespace.to_string(), - name: name.to_string(), - })?; + let access_key = secret_value( + &secret, + &[ + "accesskey", + "accessKey", + "ACCESSKEY", + "AWS_ACCESS_KEY_ID", + "access_key", + "access-key", + "access_key_id", + "access-key-id", + "RUSTFS_ACCESS_KEY", + ], + )?; + let secret_key = secret_value( + &secret, + &[ + "secretkey", + "secretKey", + "SECRETKEY", + "AWS_SECRET_ACCESS_KEY", + "secret_key", + "secret-key", + "secret_access_key", + "secret-access-key", + "RUSTFS_SECRET_KEY", + ], + )?; - if let Some(data) = cm.data.as_ref() { - for key in ["ca.crt", "tls.crt", "ca-bundle.crt"] { - if let Some(value) = data.get(key).filter(|v| !v.trim().is_empty()) { - return Ok(value.as_bytes().to_vec()); - } - } - } - if let Some(bin) = cm.binary_data.as_ref() { - for key in ["ca.crt", "tls.crt", "ca-bundle.crt"] { - if let Some(value) = bin.get(key).filter(|v| !v.0.is_empty()) { - return Ok(value.0.clone()); - } - } - } - - Err(BackendError::MissingCaData { - namespace: namespace.to_string(), - name: name.to_string(), - }) - } -} + info!( + endpoint = %params.endpoint, + secret = %params.object_store_user_secret_name, + "building RustFS admin client" + ); -fn secret_value( - data: Option<&BTreeMap>, - namespace: &str, - name: &str, - key: &'static str, -) -> Result { - let raw = - data.and_then(|data| data.get(key)) - .ok_or_else(|| BackendError::MissingSecretKey { - namespace: namespace.to_string(), - name: name.to_string(), - key, + if let (Some(cm_name), Some(cm_ns)) = ( + params.tls_ca_configmap_name.as_ref(), + params + .tls_ca_configmap_namespace + .as_ref() + .or(Some(¶ms.object_store_user_secret_namespace)), + ) { + let cms: Api = Api::namespaced(kube.clone(), cm_ns); + let cm = cms + .get(cm_name) + .await + .map_err(|source| BackendError::ConfigMapLookup { + namespace: cm_ns.clone(), + name: cm_name.clone(), + source, })?; - let value = String::from_utf8(raw.0.clone()).map_err(|_| BackendError::InvalidSecretKey { - namespace: namespace.to_string(), - name: name.to_string(), - key, - })?; - if value.is_empty() { - return Err(BackendError::EmptySecretKey { - namespace: namespace.to_string(), - name: name.to_string(), - key, - }); + let ca = cm + .data + .as_ref() + .and_then(|d| { + d.get("ca.crt") + .or_else(|| d.get("tls.crt")) + .or_else(|| d.get("ca-bundle.crt")) + }) + .ok_or(BackendError::MissingCaData("ca.crt"))?; + return Ok(RustfsAdminClient::new_with_base_url_and_ca_pem( + params.endpoint.clone(), + access_key, + secret_key, + ca.as_bytes(), + )?); } - Ok(value) + + Ok(RustfsAdminClient::new_with_base_url( + params.endpoint.clone(), + access_key, + secret_key, + )) } diff --git a/crates/cosi-driver/src/driver.rs b/crates/cosi-driver/src/driver.rs index 34c4fafe..6a60632d 100644 --- a/crates/cosi-driver/src/driver.rs +++ b/crates/cosi-driver/src/driver.rs @@ -1,30 +1,17 @@ -// Copyright 2025 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! gRPC Identity and Provisioner servers for COSI v1alpha1. - -#![allow(clippy::result_large_err)] +//! COSI Identity + Provisioner gRPC services. use std::collections::HashMap; -use sha2::{Digest, Sha256}; +use kube::Client; +use operator::sts::rustfs_client::{CreateBucketResult, RustfsAdminClient}; use tonic::{Request, Response, Status}; -use tracing::info; +use tracing::{error, info}; -use crate::backend::{BackendError, BackendFactory}; -use crate::parameters::{BackendParameters, ParameterError}; -use crate::policy::{bucket_policy_document, policy_name_for}; +use crate::backend::{BackendError, admin_client_from_params}; +use crate::parameters::{ + BackendParameters, DRIVER_NAME, bucket_policy_document_for, credentials_for_account, + grant_owner_policy_document, grant_owner_policy_name, policy_name_for, +}; use crate::proto::cosi::v1alpha1::{ AuthenticationType, CredentialDetails, DriverCreateBucketRequest, DriverCreateBucketResponse, DriverDeleteBucketRequest, DriverDeleteBucketResponse, DriverGetInfoRequest, @@ -33,78 +20,148 @@ use crate::proto::cosi::v1alpha1::{ S3SignatureVersion, identity_server::Identity, provisioner_server::Provisioner, }; -pub const DRIVER_NAME: &str = "rustfs.objectstorage.k8s.io"; +#[derive(Clone)] +pub struct Driver { + kube: Client, +} -/// Deterministic secret so DriverGrantBucketAccess is idempotent across sidecar retries. -fn credentials_for_account(account_id: &str) -> String { - let digest = Sha256::digest(format!("rustfs-cosi-v1:{account_id}").as_bytes()); - hex::encode(digest) +impl Driver { + pub fn new(kube: Client) -> Self { + Self { kube } + } } -#[cfg(test)] -mod credential_tests { - use super::credentials_for_account; +fn map_backend(err: BackendError) -> Status { + error!(error = %err, "backend error"); + Status::internal(err.to_string()) +} - #[test] - fn credentials_are_deterministic_and_long_enough() { - let a = credentials_for_account("ba-test-uid"); - let b = credentials_for_account("ba-test-uid"); - assert_eq!(a, b); - assert!(a.len() >= 8); - assert_ne!(a, credentials_for_account("other-account")); +fn map_params(err: crate::parameters::ParameterError) -> Status { + Status::invalid_argument(err.to_string()) +} + +fn map_admin(err: operator::sts::rustfs_client::RustfsClientError) -> Status { + error!(error = %err, "rustfs admin error"); + Status::internal(err.to_string()) +} + +fn credential_map( + access_key: &str, + secret_key: &str, + params: &BackendParameters, + policy_buckets: &[String], +) -> HashMap { + let mut secrets = HashMap::new(); + secrets.insert("accessKeyID".to_string(), access_key.to_string()); + secrets.insert("accessSecretKey".to_string(), secret_key.to_string()); + secrets.insert("AWS_ACCESS_KEY_ID".to_string(), access_key.to_string()); + secrets.insert("AWS_SECRET_ACCESS_KEY".to_string(), secret_key.to_string()); + secrets.insert("ACCESSKEY".to_string(), access_key.to_string()); + secrets.insert("SECRETKEY".to_string(), secret_key.to_string()); + secrets.insert("endpoint".to_string(), params.endpoint.clone()); + secrets.insert("region".to_string(), params.region.clone()); + secrets.insert( + "BUCKETS".to_string(), + params + .buckets + .clone() + .unwrap_or_else(|| policy_buckets.join(",")), + ); + secrets +} + +fn user_owns_grant( + policy_names: &[String], + owner_policy: &str, + account_id: &str, + grant_name: &str, +) -> bool { + // Default Ceph-style path: account id is the COSI grant name itself. + if account_id == grant_name { + return true; } + policy_names.iter().any(|name| name == owner_policy) } -pub struct IdentityService { - pub name: String, +async fn ensure_grant_policies( + client: &RustfsAdminClient, + access_key: &str, + bucket_policy_name: &str, + bucket_policy_doc: &str, + owner_policy_name: &str, +) -> Result<(), Status> { + client + .add_canned_policy(bucket_policy_name, bucket_policy_doc) + .await + .map_err(map_admin)?; + client + .add_canned_policy(owner_policy_name, &grant_owner_policy_document()) + .await + .map_err(map_admin)?; + client + .set_user_policy( + access_key, + &[ + bucket_policy_name.to_string(), + owner_policy_name.to_string(), + ], + ) + .await + .map_err(map_admin)?; + Ok(()) } #[tonic::async_trait] -impl Identity for IdentityService { +impl Identity for Driver { async fn driver_get_info( &self, _request: Request, ) -> Result, Status> { Ok(Response::new(DriverGetInfoResponse { - name: self.name.clone(), + name: DRIVER_NAME.to_string(), })) } } -pub struct ProvisionerService { - pub backend: BackendFactory, -} - #[tonic::async_trait] -impl Provisioner for ProvisionerService { +impl Provisioner for Driver { async fn driver_create_bucket( &self, request: Request, ) -> Result, Status> { let req = request.into_inner(); - let bucket_name = req.name.trim(); - if bucket_name.is_empty() { + if req.name.trim().is_empty() { return Err(Status::invalid_argument("bucket name is required")); } - - let params = parse_params(&req.parameters)?; - let client = self - .backend - .admin_client(¶ms) + let params = BackendParameters::from_map(&req.parameters).map_err(map_params)?; + let client = admin_client_from_params(&self.kube, ¶ms) .await .map_err(map_backend)?; - info!(bucket = %bucket_name, endpoint = %params.endpoint, "creating bucket"); - client - .create_bucket(bucket_name, params.region.as_deref(), false) - .await - .map_err(map_admin)?; + let buckets = params.buckets_to_create(&req.name); + if buckets.is_empty() { + return Err(Status::invalid_argument( + "no buckets to create (buckets/bucketName empty or only *)", + )); + } + let bucket_id = params.primary_bucket_id(&req.name); + + for bucket in &buckets { + info!(bucket = %bucket, cosi_name = %req.name, "creating bucket"); + match client + .create_bucket(bucket, Some(params.region.as_str()), false) + .await + .map_err(map_admin)? + { + CreateBucketResult::Created | CreateBucketResult::AlreadyExists => {} + } + } Ok(Response::new(DriverCreateBucketResponse { - bucket_id: bucket_name.to_string(), + bucket_id, bucket_info: Some(Protocol { r#type: Some(crate::proto::cosi::v1alpha1::protocol::Type::S3(S3 { - region: params.region.unwrap_or_else(|| "us-east-1".to_string()), + region: params.region, signature_version: S3SignatureVersion::S3v4 as i32, })), }), @@ -116,21 +173,25 @@ impl Provisioner for ProvisionerService { request: Request, ) -> Result, Status> { let req = request.into_inner(); - let bucket_id = req.bucket_id.trim(); - if bucket_id.is_empty() { + if req.bucket_id.trim().is_empty() { return Err(Status::invalid_argument("bucket_id is required")); } - - let params = parse_params(&req.delete_context)?; - let client = self - .backend - .admin_client(¶ms) + let params = BackendParameters::from_map(&req.delete_context).map_err(map_params)?; + let client = admin_client_from_params(&self.kube, ¶ms) .await .map_err(map_backend)?; - info!(bucket = %bucket_id, "deleting bucket"); - client.delete_bucket(bucket_id).await.map_err(map_admin)?; + let buckets = params.buckets_to_create(&req.bucket_id); + let targets = if buckets.is_empty() { + vec![req.bucket_id.clone()] + } else { + buckets + }; + for bucket in &targets { + info!(bucket = %bucket, "deleting bucket"); + client.delete_bucket(bucket).await.map_err(map_admin)?; + } Ok(Response::new(DriverDeleteBucketResponse {})) } @@ -139,12 +200,10 @@ impl Provisioner for ProvisionerService { request: Request, ) -> Result, Status> { let req = request.into_inner(); - let bucket_id = req.bucket_id.trim(); - let account_name = req.name.trim(); - if bucket_id.is_empty() { + if req.bucket_id.trim().is_empty() { return Err(Status::invalid_argument("bucket_id is required")); } - if account_name.is_empty() { + if req.name.trim().is_empty() { return Err(Status::invalid_argument("account name is required")); } if req.authentication_type != AuthenticationType::Key as i32 @@ -155,56 +214,80 @@ impl Provisioner for ProvisionerService { )); } - let params = parse_params(&req.parameters)?; - let client = self - .backend - .admin_client(¶ms) + let params = BackendParameters::from_map(&req.parameters).map_err(map_params)?; + let client = admin_client_from_params(&self.kube, ¶ms) .await .map_err(map_backend)?; - let account_id = account_name.to_string(); - let secret_key = credentials_for_account(&account_id); - let policy_name = policy_name_for(&account_id, bucket_id); - let policy_doc = bucket_policy_document(bucket_id, params.access_policy); + let grant_name = req.name.clone(); + let access_key = params + .preferred_access_key + .clone() + .unwrap_or_else(|| grant_name.clone()); + let secret_key = credentials_for_account(&access_key); + let policy_buckets = params.buckets_for_policy(&req.bucket_id); + let bucket_policy_name = params + .policy + .clone() + .unwrap_or_else(|| policy_name_for(&access_key)); + let bucket_policy_doc = bucket_policy_document_for(&policy_buckets); + let owner_policy_name = grant_owner_policy_name(&grant_name); info!( - bucket = %bucket_id, - account = %account_id, - policy = %policy_name, + bucket = %req.bucket_id, + account = %access_key, + grant = %grant_name, + policy = %bucket_policy_name, + owner_policy = %owner_policy_name, + buckets = %policy_buckets.join(","), "granting bucket access" ); - client - .add_canned_policy(&policy_name, &policy_doc) - .await - .map_err(map_admin)?; - - if !client.user_exists(&account_id).await.map_err(map_admin)? { - client - .add_user(&account_id, &secret_key) - .await - .map_err(map_admin)?; + match client.get_user_info(&access_key).await.map_err(map_admin)? { + Some(info) => { + if !user_owns_grant( + &info.policy_names, + &owner_policy_name, + &access_key, + &grant_name, + ) { + return Err(Status::already_exists(format!( + "preferredAccessKey `{access_key}` is already bound to another BucketAccess; \ + omit preferredAccessKey or choose a unique value" + ))); + } + // Same grant retry (or Ceph-style account == grant name): never rotate secret. + ensure_grant_policies( + &client, + &access_key, + &bucket_policy_name, + &bucket_policy_doc, + &owner_policy_name, + ) + .await?; + } + None => { + client + .add_user(&access_key, &secret_key) + .await + .map_err(map_admin)?; + ensure_grant_policies( + &client, + &access_key, + &bucket_policy_name, + &bucket_policy_doc, + &owner_policy_name, + ) + .await?; + } } - client - .set_user_policy(&account_id, std::slice::from_ref(&policy_name)) - .await - .map_err(map_admin)?; - - let mut secrets = HashMap::new(); - secrets.insert("endpoint".to_string(), params.endpoint.clone()); - secrets.insert( - "region".to_string(), - params.region.unwrap_or_else(|| "us-east-1".to_string()), - ); - secrets.insert("accessKeyID".to_string(), account_id.clone()); - secrets.insert("accessSecretKey".to_string(), secret_key); - secrets.insert("bucketName".to_string(), bucket_id.to_string()); + let secrets = credential_map(&access_key, &secret_key, ¶ms, &policy_buckets); let mut credentials = HashMap::new(); credentials.insert("s3".to_string(), CredentialDetails { secrets }); Ok(Response::new(DriverGrantBucketAccessResponse { - account_id, + account_id: access_key, credentials, })) } @@ -214,65 +297,53 @@ impl Provisioner for ProvisionerService { request: Request, ) -> Result, Status> { let req = request.into_inner(); - let bucket_id = req.bucket_id.trim(); - let account_id = req.account_id.trim(); - if account_id.is_empty() { + if req.bucket_id.trim().is_empty() { + return Err(Status::invalid_argument("bucket_id is required")); + } + if req.account_id.trim().is_empty() { return Err(Status::invalid_argument("account_id is required")); } - let params = parse_params(&req.revoke_access_context)?; - let client = self - .backend - .admin_client(¶ms) + let params = BackendParameters::from_map(&req.revoke_access_context).map_err(map_params)?; + let client = admin_client_from_params(&self.kube, ¶ms) .await .map_err(map_backend)?; - let policy_name = if bucket_id.is_empty() { - None - } else { - Some(policy_name_for(account_id, bucket_id)) - }; - - info!(account = %account_id, bucket = %bucket_id, "revoking bucket access"); - client.remove_user(account_id).await.map_err(map_admin)?; - if let Some(policy_name) = policy_name { - client - .remove_canned_policy(&policy_name) - .await - .map_err(map_admin)?; - } - + info!( + bucket = %req.bucket_id, + account = %req.account_id, + "revoking bucket access" + ); + client + .remove_user(&req.account_id) + .await + .map_err(map_admin)?; Ok(Response::new(DriverRevokeBucketAccessResponse {})) } } -fn parse_params(params: &HashMap) -> Result { - BackendParameters::from_map(params).map_err(map_params) -} - -fn map_params(err: ParameterError) -> Status { - Status::invalid_argument(err.to_string()) -} +#[cfg(test)] +mod grant_tests { + use super::user_owns_grant; -fn map_backend(err: BackendError) -> Status { - Status::failed_precondition(err.to_string()) -} + #[test] + fn same_grant_name_as_account_is_idempotent() { + assert!(user_owns_grant(&[], "cosi-grant-ba-1", "ba-1", "ba-1")); + } -fn map_admin(err: rustfs_admin::RustfsClientError) -> Status { - match &err { - rustfs_admin::RustfsClientError::UnexpectedStatus { status, .. } - if status.as_u16() == 409 => - { - Status::already_exists(err.to_string()) - } - rustfs_admin::RustfsClientError::InvalidPolicyName - | rustfs_admin::RustfsClientError::InvalidPolicyDocument - | rustfs_admin::RustfsClientError::InvalidCredentialValue { .. } - | rustfs_admin::RustfsClientError::EmptyCredentialValue { .. } - | rustfs_admin::RustfsClientError::MissingCredentialKey { .. } - | rustfs_admin::RustfsClientError::RequestBuildFailed => { - Status::invalid_argument(err.to_string()) - } - _ => Status::internal(err.to_string()), + #[test] + fn preferred_key_requires_owner_marker() { + assert!(!user_owns_grant( + &["cosi-mlflow".to_string()], + "cosi-grant-ba-1", + "mlflow", + "ba-1" + )); + assert!(user_owns_grant( + &["cosi-mlflow".to_string(), "cosi-grant-ba-1".to_string()], + "cosi-grant-ba-1", + "mlflow", + "ba-1" + )); } } diff --git a/crates/cosi-driver/src/lib.rs b/crates/cosi-driver/src/lib.rs deleted file mode 100644 index 761d3e88..00000000 --- a/crates/cosi-driver/src/lib.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2025 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! RustFS COSI v1alpha1 driver library (Identity + Provisioner gRPC services). - -pub mod backend; -pub mod driver; -pub mod parameters; -pub mod policy; - -pub mod proto { - pub mod cosi { - pub mod v1alpha1 { - tonic::include_proto!("cosi.v1alpha1"); - } - } -} - -pub use driver::{DRIVER_NAME, IdentityService, ProvisionerService}; diff --git a/crates/cosi-driver/src/main.rs b/crates/cosi-driver/src/main.rs index ae7face5..f39059c8 100644 --- a/crates/cosi-driver/src/main.rs +++ b/crates/cosi-driver/src/main.rs @@ -1,56 +1,83 @@ -// Copyright 2025 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//! RustFS COSI driver — serves Identity + Provisioner on a Unix socket. + +mod backend; +mod driver; +mod parameters; + +pub mod proto { + pub mod cosi { + pub mod v1alpha1 { + tonic::include_proto!("cosi.v1alpha1"); + } + } +} use std::path::PathBuf; -use std::time::Duration; -use cosi_driver::backend::BackendFactory; -use cosi_driver::driver::{DRIVER_NAME, IdentityService, ProvisionerService}; -use cosi_driver::proto::cosi::v1alpha1::{ - identity_server::IdentityServer, provisioner_server::ProvisionerServer, -}; +use kube::Client; use tokio::net::UnixListener; use tokio_stream::wrappers::UnixListenerStream; -use tonic::transport::Server; -use tracing::{info, warn}; -use tracing_subscriber::EnvFilter; +use tracing::{error, info}; -const DEFAULT_ENDPOINT: &str = "unix:///var/lib/cosi/cosi.sock"; -const SHUTDOWN_GRACE: Duration = Duration::from_secs(5); +use crate::driver::Driver; +use crate::parameters::DRIVER_NAME; +use crate::proto::cosi::v1alpha1::{ + identity_server::IdentityServer, provisioner_server::ProvisionerServer, +}; -#[tokio::main] -async fn main() -> Result<(), Box> { - tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from_default_env().add_directive("info".parse()?)) - .init(); +fn parse_unix_endpoint(raw: &str) -> Result { + let trimmed = raw.trim(); + let path = trimmed + .strip_prefix("unix://") + .ok_or_else(|| format!("unsupported COSI_ENDPOINT `{trimmed}`"))?; + if path.is_empty() { + return Err("COSI_ENDPOINT unix path is empty".to_string()); + } + Ok(PathBuf::from(path)) +} - let endpoint = std::env::var("COSI_ENDPOINT").unwrap_or_else(|_| DEFAULT_ENDPOINT.to_string()); - let socket_path = parse_unix_endpoint(&endpoint)?; +#[tokio::main] +async fn main() { + // Required for rustls 0.23 when multiple crypto backends may be linked via deps. + let _ = rustls::crypto::ring::default_provider().install_default(); + + let _ = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_target(true) + .try_init(); + + let endpoint = std::env::var("COSI_ENDPOINT") + .unwrap_or_else(|_| "unix:///var/lib/cosi/cosi.sock".to_string()); + let sock_path = match parse_unix_endpoint(&endpoint) { + Ok(path) => path, + Err(err) => { + error!(error = %err, "invalid COSI_ENDPOINT"); + std::process::exit(2); + } + }; - if let Some(parent) = socket_path.parent() { - tokio::fs::create_dir_all(parent).await?; + if let Some(parent) = sock_path.parent() { + let _ = std::fs::create_dir_all(parent); } - let _ = tokio::fs::remove_file(&socket_path).await; + let _ = std::fs::remove_file(&sock_path); - let backend = BackendFactory::try_default().await?; - let identity = IdentityService { - name: DRIVER_NAME.to_string(), + let kube = match Client::try_default().await { + Ok(client) => client, + Err(err) => { + error!(error = %err, "failed to create Kubernetes client"); + std::process::exit(1); + } }; - let provisioner = ProvisionerService { backend }; - let uds = UnixListener::bind(&socket_path)?; - let uds_stream = UnixListenerStream::new(uds); + let listener = match UnixListener::bind(&sock_path) { + Ok(listener) => listener, + Err(err) => { + error!(error = %err, path = %sock_path.display(), "failed to bind COSI socket"); + std::process::exit(1); + } + }; + let incoming = UnixListenerStream::new(listener); + let driver = Driver::new(kube); info!( driver = DRIVER_NAME, @@ -58,66 +85,14 @@ async fn main() -> Result<(), Box> { "starting RustFS COSI driver" ); - let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>(); - tokio::spawn(async move { - wait_for_shutdown().await; - let _ = shutdown_tx.send(()); - }); - - Server::builder() - .add_service(IdentityServer::new(identity)) - .add_service(ProvisionerServer::new(provisioner)) - .serve_with_incoming_shutdown(uds_stream, async { - let _ = shutdown_rx.await; - info!("shutdown signal received"); - }) - .await?; - - let _ = tokio::fs::remove_file(&socket_path).await; - // Allow in-flight RPCs a brief window before process exit. - tokio::time::sleep(SHUTDOWN_GRACE).await; - info!("RustFS COSI driver stopped"); - Ok(()) -} - -fn parse_unix_endpoint(endpoint: &str) -> Result { - let endpoint = endpoint.trim(); - if let Some(path) = endpoint.strip_prefix("unix://") { - if path.is_empty() { - return Err("COSI_ENDPOINT unix path is empty".into()); - } - return Ok(PathBuf::from(path)); - } - if endpoint.starts_with('/') { - return Ok(PathBuf::from(endpoint)); - } - Err(format!( - "unsupported COSI_ENDPOINT `{endpoint}` (expected unix:///path/to.sock)" - )) -} - -async fn wait_for_shutdown() { - let ctrl_c = async { - if let Err(err) = tokio::signal::ctrl_c().await { - warn!(error = %err, "failed to install Ctrl+C handler"); - } - }; - - #[cfg(unix)] - let terminate = async { - match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) { - Ok(mut stream) => { - stream.recv().await; - } - Err(err) => warn!(error = %err, "failed to install SIGTERM handler"), - } - }; - - #[cfg(not(unix))] - let terminate = std::future::pending::<()>(); + let result = tonic::transport::Server::builder() + .add_service(IdentityServer::new(driver.clone())) + .add_service(ProvisionerServer::new(driver)) + .serve_with_incoming(incoming) + .await; - tokio::select! { - () = ctrl_c => {}, - () = terminate => {}, + if let Err(err) = result { + error!(error = %err, "RustFS COSI driver stopped"); + std::process::exit(1); } } diff --git a/crates/cosi-driver/src/parameters.rs b/crates/cosi-driver/src/parameters.rs index ed7f40dc..f8759213 100644 --- a/crates/cosi-driver/src/parameters.rs +++ b/crates/cosi-driver/src/parameters.rs @@ -1,119 +1,196 @@ -// Copyright 2025 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! BucketClass / BucketAccessClass parameter parsing (Rook-style). +//! RustFS COSI driver parameters (BucketClass / BucketAccessClass). use std::collections::HashMap; -use snafu::Snafu; +use thiserror::Error; -use crate::policy::AccessPolicy; +pub const DRIVER_NAME: &str = "rustfs.objectstorage.k8s.io"; -pub const PARAM_SECRET_NAME: &str = "objectStoreUserSecretName"; -pub const PARAM_SECRET_NAMESPACE: &str = "objectStoreUserSecretNamespace"; -pub const PARAM_ENDPOINT: &str = "endpoint"; -pub const PARAM_REGION: &str = "region"; -pub const PARAM_TLS_CA_CM_NAME: &str = "tlsCAConfigMapName"; -pub const PARAM_TLS_CA_CM_NAMESPACE: &str = "tlsCAConfigMapNamespace"; -pub const PARAM_POLICY: &str = "policy"; - -#[derive(Debug, Snafu)] -pub enum ParameterError { - #[snafu(display("missing required parameter `{name}`"))] - Missing { name: &'static str }, - #[snafu(display("parameter `{name}` must not be empty"))] - Empty { name: &'static str }, -} - -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] pub struct BackendParameters { - pub secret_name: String, - pub secret_namespace: String, pub endpoint: String, - pub region: Option, + pub object_store_user_secret_name: String, + pub object_store_user_secret_namespace: String, + pub region: String, + pub policy: Option, pub tls_ca_configmap_name: Option, pub tls_ca_configmap_namespace: Option, - pub access_policy: AccessPolicy, + /// Preferred S3 bucket name (overrides COSI-generated CreateBucket name). + pub bucket_name: Option, + /// Comma-separated bucket list to create / authorize (`*` = full access). + pub buckets: Option, + /// Preferred access-key / account name for GrantBucketAccess. + /// + /// Must be unique per BucketAccess. Reusing the same value across claims is + /// rejected by the driver (Ceph-style isolation). Prefer omitting this so the + /// COSI grant name (`ba-`) is used as the account id. + pub preferred_access_key: Option, +} + +#[derive(Debug, Error)] +pub enum ParameterError { + #[error("missing required parameter `{0}`")] + MissingRequired(&'static str), + #[error("parameter `{0}` is empty")] + Empty(&'static str), +} + +fn required(map: &HashMap, key: &'static str) -> Result { + let value = map + .get(key) + .cloned() + .ok_or(ParameterError::MissingRequired(key))?; + if value.trim().is_empty() { + return Err(ParameterError::Empty(key)); + } + Ok(value) +} + +fn optional(map: &HashMap, key: &str) -> Option { + map.get(key) + .cloned() + .filter(|value| !value.trim().is_empty()) } impl BackendParameters { - pub fn from_map(params: &HashMap) -> Result { + pub fn from_map(map: &HashMap) -> Result { Ok(Self { - secret_name: required(params, PARAM_SECRET_NAME)?, - secret_namespace: required(params, PARAM_SECRET_NAMESPACE)?, - endpoint: required(params, PARAM_ENDPOINT)?, - region: optional(params, PARAM_REGION), - tls_ca_configmap_name: optional(params, PARAM_TLS_CA_CM_NAME), - tls_ca_configmap_namespace: optional(params, PARAM_TLS_CA_CM_NAMESPACE), - access_policy: AccessPolicy::parse(params.get(PARAM_POLICY).map(String::as_str)), + endpoint: required(map, "endpoint")?, + object_store_user_secret_name: required(map, "objectStoreUserSecretName")?, + object_store_user_secret_namespace: required(map, "objectStoreUserSecretNamespace")?, + region: optional(map, "region").unwrap_or_else(|| "us-east-1".to_string()), + policy: optional(map, "policy"), + tls_ca_configmap_name: optional(map, "tlsCAConfigMapName"), + tls_ca_configmap_namespace: optional(map, "tlsCAConfigMapNamespace"), + bucket_name: optional(map, "bucketName"), + buckets: optional(map, "buckets"), + preferred_access_key: optional(map, "preferredAccessKey") + .or_else(|| optional(map, "accessKey")), }) } -} -fn required( - params: &HashMap, - name: &'static str, -) -> Result { - let value = params - .get(name) - .ok_or(ParameterError::Missing { name })? - .trim(); - if value.is_empty() { - return Err(ParameterError::Empty { name }); + /// Buckets to create (excludes `*`). Primary bucket_id is the first entry. + pub fn buckets_to_create(&self, fallback_name: &str) -> Vec { + let raw = self + .buckets + .as_deref() + .or(self.bucket_name.as_deref()) + .unwrap_or(fallback_name); + raw.split(',') + .map(str::trim) + .filter(|b| !b.is_empty() && *b != "*") + .map(ToOwned::to_owned) + .collect() + } + + /// Full bucket list for IAM policy (may include `*`). + pub fn buckets_for_policy(&self, fallback_name: &str) -> Vec { + let raw = self + .buckets + .as_deref() + .or(self.bucket_name.as_deref()) + .unwrap_or(fallback_name); + let list: Vec = raw + .split(',') + .map(str::trim) + .filter(|b| !b.is_empty()) + .map(ToOwned::to_owned) + .collect(); + if list.is_empty() { + vec![fallback_name.to_string()] + } else { + list + } } - Ok(value.to_string()) + + pub fn primary_bucket_id(&self, cosi_name: &str) -> String { + self.bucket_name + .clone() + .or_else(|| self.buckets_to_create(cosi_name).into_iter().next()) + .unwrap_or_else(|| cosi_name.to_string()) + } +} + +pub fn bucket_policy_document_for(buckets: &[String]) -> String { + let has_wildcard = buckets.iter().any(|b| b == "*"); + let resources: Vec = if has_wildcard { + vec!["arn:aws:s3:::*".to_string(), "arn:aws:s3:::*/*".to_string()] + } else { + buckets + .iter() + .flat_map(|b| [format!("arn:aws:s3:::{b}"), format!("arn:aws:s3:::{b}/*")]) + .collect() + }; + serde_json::json!({ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["s3:*"], + "Resource": resources + }] + }) + .to_string() +} + +pub fn sanitize_policy_fragment(value: &str) -> String { + value + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }) + .collect() +} + +pub fn policy_name_for(bucket: &str) -> String { + format!("cosi-{}", sanitize_policy_fragment(bucket)) } -fn optional(params: &HashMap, name: &str) -> Option { - params - .get(name) - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) +/// Owner marker policy bound to a specific COSI grant `name` (`ba-`). +pub fn grant_owner_policy_name(grant_name: &str) -> String { + format!("cosi-grant-{}", sanitize_policy_fragment(grant_name)) +} + +/// Minimal canned policy used only as an ownership marker for a BucketAccess grant. +pub fn grant_owner_policy_document() -> String { + serde_json::json!({ + "Version": "2012-10-17", + "Statement": [{ + "Sid": "CosiGrantOwner", + "Effect": "Allow", + "Action": ["s3:ListAllMyBuckets"], + "Resource": ["arn:aws:s3:::*"] + }] + }) + .to_string() +} + +/// Deterministic secret so DriverGrantBucketAccess is idempotent across sidecar retries. +pub fn credentials_for_account(account_id: &str) -> String { + use sha2::{Digest, Sha256}; + let digest = Sha256::digest(format!("rustfs-cosi-v1:{account_id}").as_bytes()); + hex::encode(digest) } #[cfg(test)] mod tests { - use super::*; + use super::{credentials_for_account, grant_owner_policy_name}; #[test] - fn parses_required_parameters() { - let mut map = HashMap::new(); - map.insert(PARAM_SECRET_NAME.to_string(), "creds".into()); - map.insert(PARAM_SECRET_NAMESPACE.to_string(), "ns".into()); - map.insert( - PARAM_ENDPOINT.to_string(), - "http://tenant-io.ns.svc:9000".into(), - ); - map.insert(PARAM_POLICY.to_string(), "readonly".into()); - - let parsed = BackendParameters::from_map(&map).expect("parse"); - assert_eq!(parsed.secret_name, "creds"); - assert_eq!(parsed.access_policy, AccessPolicy::Readonly); - assert_eq!(parsed.endpoint, "http://tenant-io.ns.svc:9000"); + fn credentials_are_deterministic_and_long_enough() { + let a = credentials_for_account("ba-test-uid"); + let b = credentials_for_account("ba-test-uid"); + assert_eq!(a, b); + assert!(a.len() >= 8); + assert_ne!(a, credentials_for_account("other-account")); } #[test] - fn rejects_missing_endpoint() { - let mut map = HashMap::new(); - map.insert(PARAM_SECRET_NAME.to_string(), "creds".into()); - map.insert(PARAM_SECRET_NAMESPACE.to_string(), "ns".into()); - let err = BackendParameters::from_map(&map).expect_err("missing endpoint"); - assert!(matches!( - err, - ParameterError::Missing { - name: PARAM_ENDPOINT - } - )); + fn grant_owner_policy_name_sanitizes() { + assert_eq!( + grant_owner_policy_name("ba-81733d1a-ac7a-4759-96f3-fbcc07c0cee9"), + "cosi-grant-ba-81733d1a-ac7a-4759-96f3-fbcc07c0cee9" + ); + assert_eq!( + grant_owner_policy_name("ba/weird.name"), + "cosi-grant-ba-weird-name" + ); } } diff --git a/crates/cosi-driver/src/policy.rs b/crates/cosi-driver/src/policy.rs deleted file mode 100644 index d062f999..00000000 --- a/crates/cosi-driver/src/policy.rs +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2025 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! IAM policy documents scoped to a single bucket for COSI BucketAccess grants. - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum AccessPolicy { - Readonly, - ReadWrite, -} - -impl AccessPolicy { - pub fn parse(value: Option<&str>) -> Self { - match value.map(str::trim).map(str::to_ascii_lowercase).as_deref() { - Some("readonly") | Some("read-only") | Some("read") => Self::Readonly, - _ => Self::ReadWrite, - } - } - - pub fn as_str(self) -> &'static str { - match self { - Self::Readonly => "readonly", - Self::ReadWrite => "readwrite", - } - } -} - -/// Build a canned IAM policy JSON document limited to `bucket`. -pub fn bucket_policy_document(bucket: &str, policy: AccessPolicy) -> String { - let actions = match policy { - AccessPolicy::Readonly => { - r#"[ - "s3:GetBucketLocation", - "s3:ListBucket", - "s3:GetObject" - ]"# - } - AccessPolicy::ReadWrite => { - r#"[ - "s3:GetBucketLocation", - "s3:ListBucket", - "s3:GetObject", - "s3:PutObject", - "s3:DeleteObject" - ]"# - } - }; - - format!( - r#"{{ - "Version": "2012-10-17", - "Statement": [ - {{ - "Effect": "Allow", - "Action": {actions}, - "Resource": [ - "arn:aws:s3:::{bucket}", - "arn:aws:s3:::{bucket}/*" - ] - }} - ] -}}"# - ) -} - -/// Deterministic canned policy name for a COSI account + bucket pair. -pub fn policy_name_for(account_id: &str, bucket_id: &str) -> String { - // RustFS policy names should stay reasonably short and DNS-safe. - let raw = format!("cosi-{account_id}-{bucket_id}"); - raw.chars() - .map(|ch| { - if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' { - ch - } else { - '-' - } - }) - .take(128) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_policy_defaults_to_readwrite() { - assert_eq!(AccessPolicy::parse(None), AccessPolicy::ReadWrite); - assert_eq!(AccessPolicy::parse(Some("")), AccessPolicy::ReadWrite); - assert_eq!( - AccessPolicy::parse(Some("readwrite")), - AccessPolicy::ReadWrite - ); - assert_eq!( - AccessPolicy::parse(Some("readonly")), - AccessPolicy::Readonly - ); - } - - #[test] - fn policy_document_includes_bucket_resources() { - let doc = bucket_policy_document("my-bucket", AccessPolicy::Readonly); - assert!(doc.contains("arn:aws:s3:::my-bucket")); - assert!(doc.contains("s3:GetObject")); - assert!(!doc.contains("s3:PutObject")); - } - - #[test] - fn policy_name_is_sanitized() { - let name = policy_name_for("ba.uid", "bucket/name"); - assert!(!name.contains('.')); - assert!(!name.contains('/')); - assert!(name.starts_with("cosi-")); - } -} diff --git a/crates/rustfs-admin/Cargo.toml b/crates/rustfs-admin/Cargo.toml deleted file mode 100644 index c91ecfe4..00000000 --- a/crates/rustfs-admin/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "rustfs-admin" -version = "0.1.0" -edition = "2024" -license = "Apache-2.0" -description = "Kube-agnostic RustFS admin/S3/STS client" - -[dependencies] -chrono = { version = "0.4", features = ["serde"] } -hex = "0.4" -hmac = "0.12" -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" -sha2 = "0.10" -url = "2.5" - -[dev-dependencies] -axum = { version = "0.7", features = ["macros", "json"] } -tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "sync", "time"] } - -[lints.rust] -unused_variables = "allow" diff --git a/crates/rustfs-admin/src/client.rs b/crates/rustfs-admin/src/client.rs deleted file mode 100644 index 11ae191b..00000000 --- a/crates/rustfs-admin/src/client.rs +++ /dev/null @@ -1,470 +0,0 @@ -// Copyright 2025 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Client types: credentials, response models, error type and the -//! `RustfsAdminClient` handle used by every ops module. -use std::{collections::BTreeMap, time::Duration}; - -use reqwest::{Certificate, Client as HttpClient, Response, StatusCode}; - -use crate::sanitize::redact_sensitive_pairs; - -pub(crate) const FORM_CONTENT_TYPE: &str = "application/x-www-form-urlencoded"; -pub(crate) const JSON_CONTENT_TYPE: &str = "application/json"; -pub(crate) const ASSUME_ROLE_PATH: &str = "/"; -pub(crate) const ADD_USER_PATH: &str = "/rustfs/admin/v3/add-user"; -pub(crate) const REMOVE_USER_PATH: &str = "/rustfs/admin/v3/remove-user"; -pub(crate) const USER_INFO_PATH: &str = "/rustfs/admin/v3/user-info"; -pub(crate) const SET_POLICY_PATH: &str = "/rustfs/admin/v3/set-policy"; -pub(crate) const LIST_CANNED_POLICIES_PATH: &str = "/rustfs/admin/v3/list-canned-policies"; -pub(crate) const ADD_CANNED_POLICY_PATH: &str = "/rustfs/admin/v3/add-canned-policy"; -pub(crate) const REMOVE_CANNED_POLICY_PATH: &str = "/rustfs/admin/v3/remove-canned-policy"; -pub(crate) const INFO_CANNED_POLICY_PATH: &str = "/rustfs/admin/v3/info-canned-policy"; -pub(crate) const SERVER_INFO_PATH: &str = "/rustfs/admin/v3/info"; -pub(crate) const POOLS_LIST_PATH: &str = "/rustfs/admin/v3/pools/list"; -pub(crate) const POOLS_STATUS_PATH: &str = "/rustfs/admin/v3/pools/status"; -pub(crate) const POOLS_DECOMMISSION_PATH: &str = "/rustfs/admin/v3/pools/decommission"; -pub(crate) const POOLS_CANCEL_PATH: &str = "/rustfs/admin/v3/pools/cancel"; -pub(crate) const ADMIN_SIGNING_SERVICE: &str = "s3"; -pub(crate) const STS_SIGNING_SERVICE: &str = "sts"; -pub(crate) const ADMIN_HTTP_CONNECT_TIMEOUT: Duration = Duration::from_secs(3); -pub(crate) const ADMIN_HTTP_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); -pub(crate) const MAX_UPSTREAM_ERROR_BODY_BYTES: usize = 8 * 1024; -pub(crate) const MAX_UPSTREAM_ERROR_DETAIL_CHARS: usize = 512; - -/// Credentials used to sign requests against a RustFS admin/S3/STS endpoint. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RustfsCredentials { - pub access_key: String, - pub secret_key: String, -} - -#[derive(Debug, Clone, serde::Deserialize, PartialEq)] -pub struct RustfsPoolListItem { - pub id: usize, - #[serde(rename = "cmdline")] - pub cmd_line: String, - #[serde(rename = "lastUpdate")] - pub last_update: String, - #[serde(rename = "totalSize")] - pub total_size: Option, - #[serde(rename = "currentSize")] - pub current_size: Option, - #[serde(rename = "usedSize")] - pub used_size: Option, - pub used: Option, - pub status: String, - #[serde(rename = "decommissionInfo")] - pub decommission: Option, -} - -#[derive(Debug, Clone, serde::Deserialize, PartialEq)] -pub struct RustfsPoolStatus { - pub id: usize, - #[serde(rename = "cmdline")] - pub cmd_line: String, - #[serde(rename = "lastUpdate")] - pub last_update: String, - #[serde(rename = "decommissionInfo")] - pub decommission: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum CreateBucketResult { - Created, - AlreadyExists, -} - -#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] -pub struct RustfsPoolDecommissionInfo { - #[serde(rename = "startTime")] - pub start_time: Option, - #[serde(rename = "startSize")] - pub start_size: Option, - #[serde(rename = "totalSize")] - pub total_size: Option, - #[serde(rename = "currentSize")] - pub current_size: Option, - pub complete: Option, - pub failed: Option, - pub canceled: Option, - #[serde(rename = "objectsDecommissioned")] - pub objects_decommissioned: Option, - #[serde(rename = "objectsDecommissionedFailed")] - pub objects_decommissioned_failed: Option, - #[serde(rename = "bytesDecommissioned")] - pub bytes_decommissioned: Option, - #[serde(rename = "bytesDecommissionedFailed")] - pub bytes_decommissioned_failed: Option, -} - -#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] -pub struct RustfsServerInfo { - #[serde(default)] - pub usage: Option, - #[serde(default)] - pub backend: Option, - #[serde(default)] - pub pools: Option>>, -} - -#[derive(Debug, Clone, serde::Deserialize, PartialEq)] -pub(crate) struct RustfsServerInfoResponse { - pub info: RustfsServerInfo, -} - -#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] -pub struct RustfsServerUsage { - #[serde(default)] - pub size: u64, -} - -#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] -pub struct RustfsErasureBackend { - #[serde(default, rename = "onlineDisks")] - pub online_disks: u64, - #[serde(default, rename = "offlineDisks")] - pub offline_disks: u64, - #[serde(default, rename = "standardSCParity", alias = "StandardSCParity")] - pub standard_sc_parity: Option, - #[serde(default, rename = "totalSets")] - pub total_sets: Vec, - #[serde(default, rename = "totalDrivesPerSet", alias = "drivesPerSet")] - pub drives_per_set: Vec, -} - -#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] -pub struct RustfsErasureSetInfo { - #[serde(default, rename = "rawUsage")] - pub raw_usage: u64, - #[serde(default, rename = "rawCapacity")] - pub raw_capacity: u64, - #[serde(default)] - pub usage: u64, - #[serde(default, rename = "objectsCount")] - pub objects_count: u64, - #[serde(default, rename = "healDisks")] - pub heal_disks: u64, -} - -/// Error type for RustFS admin/STS client operations. -/// -/// This also carries the Tenant/kube-related variants used by the operator's -/// kube-aware wrappers (see the `operator` crate's `sts::rustfs_client` -/// module), so both crates can share a single error type end-to-end. -#[derive(Debug)] -pub enum RustfsClientError { - MissingTenantNamespace, - MissingCredsSecret, - MissingCredentialKey { - key: &'static str, - }, - EmptyCredentialValue { - key: &'static str, - }, - InvalidCredentialValue { - key: &'static str, - }, - TenantSecretLookupFailed, - InvalidPolicyName, - InvalidPolicyDocument, - TenantTlsRequired, - TenantTlsNotReady, - TenantTlsClientCertificateRequired, - MissingTenantTlsCaKey { - secret: String, - key: String, - }, - TenantTlsCaSecretLookupFailed { - secret: String, - }, - InvalidTenantTlsCa, - TlsClientBuildFailed, - RequestBuildFailed, - RequestFailed, - UnexpectedStatus { - status: StatusCode, - detail: Option, - }, - ParseResponseFailed, - SigningFailed, -} - -impl std::fmt::Display for RustfsClientError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::MissingTenantNamespace => write!(f, "tenant namespace is missing"), - Self::MissingCredsSecret => write!(f, "tenant credsSecret is missing"), - Self::MissingCredentialKey { key } => write!(f, "secret key missing: {key}"), - Self::EmptyCredentialValue { key } => write!(f, "secret key empty: {key}"), - Self::InvalidCredentialValue { key } => { - write!(f, "secret key is not valid utf8: {key}") - } - Self::TenantSecretLookupFailed => { - write!(f, "failed to load tenant credential secret") - } - Self::InvalidPolicyName => write!(f, "invalid policy name"), - Self::InvalidPolicyDocument => write!(f, "failed to parse canned policy response"), - Self::TenantTlsRequired => write!(f, "STS requires a TLS-enabled tenant"), - Self::TenantTlsNotReady => write!(f, "tenant TLS status is not ready"), - Self::TenantTlsClientCertificateRequired => { - write!(f, "tenant TLS requires a client certificate") - } - Self::MissingTenantTlsCaKey { secret, key } => { - write!(f, "tenant TLS CA secret {secret} missing key {key}") - } - Self::TenantTlsCaSecretLookupFailed { secret } => { - write!(f, "failed to load tenant TLS CA secret {secret}") - } - Self::InvalidTenantTlsCa => write!(f, "tenant TLS CA is not a valid PEM bundle"), - Self::TlsClientBuildFailed => write!(f, "failed to build TLS HTTP client"), - Self::RequestBuildFailed => write!(f, "failed to construct request"), - Self::RequestFailed => write!(f, "request failed"), - Self::UnexpectedStatus { status, detail } => { - write!(f, "upstream returned {status}")?; - if let Some(detail) = detail { - write!(f, ": {detail}")?; - } - Ok(()) - } - Self::ParseResponseFailed => write!(f, "failed to parse AssumeRole response"), - Self::SigningFailed => write!(f, "failed to compute request signature"), - } - } -} - -impl std::error::Error for RustfsClientError {} - -impl RustfsClientError { - pub(crate) async fn unexpected_response(response: Response) -> Self { - let status = response.status(); - let (body, truncated) = read_limited_response_body(response).await; - Self::unexpected_status_with_limited_body(status, &body, truncated) - } - - pub(crate) async fn limited_response_body(response: Response) -> (String, bool) { - read_limited_response_body(response).await - } - - pub(crate) fn unexpected_status_with_limited_body( - status: StatusCode, - body: &str, - body_truncated: bool, - ) -> Self { - Self::UnexpectedStatus { - status, - detail: summarize_upstream_error_body(body, body_truncated), - } - } - - #[cfg(test)] - pub(crate) fn unexpected_status_with_body(status: StatusCode, body: &str) -> Self { - Self::unexpected_status_with_limited_body(status, body, false) - } -} - -async fn read_limited_response_body(mut response: Response) -> (String, bool) { - let mut body = Vec::new(); - let read_limit = MAX_UPSTREAM_ERROR_BODY_BYTES.saturating_add(1); - - loop { - let remaining = read_limit.saturating_sub(body.len()); - if remaining == 0 { - break; - } - - let chunk = match response.chunk().await { - Ok(Some(chunk)) => chunk, - Ok(None) => break, - Err(_) => break, - }; - if chunk.len() > remaining { - body.extend_from_slice(&chunk[..remaining]); - break; - } - body.extend_from_slice(&chunk); - } - - let truncated = body.len() > MAX_UPSTREAM_ERROR_BODY_BYTES; - if truncated { - body.truncate(MAX_UPSTREAM_ERROR_BODY_BYTES); - } - - (String::from_utf8_lossy(&body).into_owned(), truncated) -} - -fn summarize_upstream_error_body(body: &str, body_truncated: bool) -> Option { - let body = body.trim(); - if body.is_empty() { - return None; - } - - if let Some(message) = crate::helpers::extract_xml_tag(body, "Message") { - let message = decode_basic_xml_entities(&message); - let detail = match crate::helpers::extract_xml_tag(body, "Code") { - Some(code) if !code.trim().is_empty() => { - format!("{}: {message}", decode_basic_xml_entities(&code)) - } - _ => message, - }; - return Some(sanitize_error_detail(&detail)); - } - - if let Ok(value) = serde_json::from_str::(body) - && let Some(detail) = summarize_json_error(&value) - { - return Some(sanitize_error_detail(&detail)); - } - - if body_truncated { - return Some(format!( - "response body exceeded {MAX_UPSTREAM_ERROR_BODY_BYTES} bytes" - )); - } - - Some(sanitize_error_detail(body)) -} - -fn summarize_json_error(value: &serde_json::Value) -> Option { - if let Some(message) = value.as_str() { - return Some(message.to_string()); - } - - let object = value.as_object()?; - let message = ["message", "Message", "error", "Error"] - .iter() - .find_map(|key| object.get(*key).and_then(serde_json::Value::as_str))?; - let code = ["code", "Code"] - .iter() - .find_map(|key| object.get(*key).and_then(serde_json::Value::as_str)); - - Some(match code { - Some(code) if !code.trim().is_empty() => format!("{code}: {message}"), - _ => message.to_string(), - }) -} - -fn collapse_whitespace(value: &str) -> String { - value.split_whitespace().collect::>().join(" ") -} - -fn sanitize_error_detail(value: &str) -> String { - let detail = collapse_whitespace(value); - let detail = redact_sensitive_pairs(&detail); - truncate_error_detail(detail) -} - -fn truncate_error_detail(value: String) -> String { - let mut truncated = String::new(); - for (index, ch) in value.chars().enumerate() { - if index >= MAX_UPSTREAM_ERROR_DETAIL_CHARS { - truncated.push_str("..."); - return truncated; - } - truncated.push(ch); - } - truncated -} - -fn decode_basic_xml_entities(value: &str) -> String { - value - .replace("<", "<") - .replace(">", ">") - .replace(""", "\"") - .replace("'", "'") - .replace("&", "&") -} - -#[derive(Debug)] -pub(crate) struct SignedRequest { - pub(crate) amz_date: String, - pub(crate) payload_hash: String, - pub(crate) authorization: String, -} - -/// RustFS admin/S3/STS client. -pub struct RustfsAdminClient { - pub(crate) base_url: String, - pub(crate) access_key: String, - pub(crate) secret_key: String, - pub(crate) region: String, - pub(crate) http_client: HttpClient, -} - -pub(crate) fn default_http_client() -> HttpClient { - HttpClient::builder() - .connect_timeout(ADMIN_HTTP_CONNECT_TIMEOUT) - .timeout(ADMIN_HTTP_REQUEST_TIMEOUT) - .build() - .unwrap_or_else(|_| HttpClient::new()) -} - -impl RustfsAdminClient { - pub const STS_VERSION: &'static str = "2011-06-15"; - pub const STS_ACTION: &'static str = "AssumeRole"; - - pub fn new_with_base_url( - base_url: impl Into, - access_key: impl Into, - secret_key: impl Into, - ) -> Self { - Self::new_with_base_url_and_http_client( - base_url, - access_key, - secret_key, - default_http_client(), - ) - } - - pub fn new_with_base_url_and_ca_pem( - base_url: impl Into, - access_key: impl Into, - secret_key: impl Into, - ca_pem: &[u8], - ) -> Result { - let certs = Certificate::from_pem_bundle(ca_pem) - .map_err(|_| RustfsClientError::InvalidTenantTlsCa)?; - let mut builder = HttpClient::builder() - .connect_timeout(ADMIN_HTTP_CONNECT_TIMEOUT) - .timeout(ADMIN_HTTP_REQUEST_TIMEOUT); - for cert in certs { - builder = builder.add_root_certificate(cert); - } - let http_client = builder - .build() - .map_err(|_| RustfsClientError::TlsClientBuildFailed)?; - - Ok(Self::new_with_base_url_and_http_client( - base_url, - access_key, - secret_key, - http_client, - )) - } - - pub fn new_with_base_url_and_http_client( - base_url: impl Into, - access_key: impl Into, - secret_key: impl Into, - http_client: HttpClient, - ) -> Self { - Self { - base_url: base_url.into(), - access_key: access_key.into(), - secret_key: secret_key.into(), - region: "us-east-1".to_string(), - http_client, - } - } -} diff --git a/crates/rustfs-admin/src/credentials.rs b/crates/rustfs-admin/src/credentials.rs deleted file mode 100644 index 800938a8..00000000 --- a/crates/rustfs-admin/src/credentials.rs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2025 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Temporary credentials returned by the RustFS STS `AssumeRole` API. - -#[derive(Debug, Clone)] -pub struct StsAssumeRoleCredentials { - pub access_key_id: String, - pub secret_access_key: String, - pub session_token: String, - pub expiration: String, -} diff --git a/crates/rustfs-admin/src/helpers.rs b/crates/rustfs-admin/src/helpers.rs deleted file mode 100644 index 1b239c83..00000000 --- a/crates/rustfs-admin/src/helpers.rs +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright 2025 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Internal helper duties: signature/hash utilities and wire-format parsers. -use hmac::{Hmac, Mac}; -use reqwest::StatusCode; -use serde_json::Value; -use sha2::{Digest, Sha256}; -use url::form_urlencoded; - -use crate::client::RustfsClientError; -use crate::credentials::StsAssumeRoleCredentials; - -/// Encode an `application/x-www-form-urlencoded` request body. -pub(crate) fn build_form_body(params: &[(&str, &str)]) -> String { - let mut pairs: Vec<(String, String)> = params - .iter() - .map(|(k, v)| ((*k).to_string(), (*v).to_string())) - .collect(); - pairs.sort_by(|(k1, v1), (k2, v2)| k1.cmp(k2).then(v1.cmp(v2))); - - let mut serializer = form_urlencoded::Serializer::new(String::new()); - for (key, value) in pairs { - serializer.append_pair(&key, &value); - } - - serializer.finish() -} - -/// Encode and sort query parameters according to the AWS SigV4 rules. -pub(crate) fn build_canonical_query(params: &[(&str, &str)]) -> String { - let mut pairs: Vec<(String, String)> = params - .iter() - .map(|(key, value)| (uri_encode(key), uri_encode(value))) - .collect(); - pairs.sort_unstable(); - - pairs - .into_iter() - .map(|(key, value)| format!("{key}={value}")) - .collect::>() - .join("&") -} - -fn uri_encode(value: &str) -> String { - const HEX: &[u8; 16] = b"0123456789ABCDEF"; - - let mut encoded = String::with_capacity(value.len()); - for byte in value.bytes() { - if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { - encoded.push(char::from(byte)); - } else { - encoded.push('%'); - encoded.push(char::from(HEX[usize::from(byte >> 4)])); - encoded.push(char::from(HEX[usize::from(byte & 0x0f)])); - } - } - encoded -} - -pub(crate) fn create_bucket_body(region: Option<&str>) -> String { - let Some(region) = region.map(str::trim).filter(|region| !region.is_empty()) else { - return String::new(); - }; - - if region == "us-east-1" { - return String::new(); - } - - format!( - "{}", - escape_xml(region) - ) -} - -pub(crate) fn escape_xml(value: &str) -> String { - value - .replace('&', "&") - .replace('<', "<") - .replace('>', ">") - .replace('"', """) - .replace('\'', "'") -} - -pub(crate) fn body_mentions_not_found(body: &str) -> bool { - let body = body.to_ascii_lowercase(); - body.contains("nosuchuser") - || body.contains("no such user") - || body.contains("user not exist") - || body.contains("nosuchpolicy") - || body.contains("no such policy") - || body.contains("objectlockconfigurationnotfound") - || body.contains("not found") -} - -/// Whether the response body indicates the target bucket does not exist. -pub(crate) fn bucket_not_found(body: &str) -> bool { - let body = body.to_ascii_lowercase(); - body.contains("nosuchbucket") || body.contains("no such bucket") || body.contains("not found") -} - -pub(crate) fn bucket_already_exists(status: StatusCode, body: &str) -> bool { - if status == StatusCode::CONFLICT { - let body = body.to_ascii_lowercase(); - return body.contains("bucketalreadyexists") || body.contains("bucketalreadyownedbyyou"); - } - - false -} - -pub(crate) fn extract_canned_policy_document(body: &str) -> Result { - let value = serde_json::from_str::(body) - .map_err(|_| RustfsClientError::InvalidPolicyDocument)?; - let policy = value.get("policy").unwrap_or(&value); - - serde_json::to_string(policy).map_err(|_| RustfsClientError::InvalidPolicyDocument) -} - -pub(crate) fn sha256_hex(payload: &[u8]) -> String { - let mut hasher = Sha256::new(); - hasher.update(payload); - hex::encode(hasher.finalize()) -} - -pub(crate) fn hmac_sha256(key: &[u8], message: &str) -> Result, RustfsClientError> { - let mut mac = - Hmac::::new_from_slice(key).map_err(|_| RustfsClientError::SigningFailed)?; - mac.update(message.as_bytes()); - Ok(mac.finalize().into_bytes().to_vec()) -} - -pub(crate) fn hmac_sha256_hex(key: &[u8], message: &str) -> Result { - let bytes = hmac_sha256(key, message)?; - Ok(hex::encode(bytes)) -} - -pub(crate) fn derive_signing_key( - secret_key: &str, - date_stamp: &str, - region: &str, - service: &str, -) -> Result, RustfsClientError> { - let k_secret = format!("AWS4{secret_key}").into_bytes(); - let k_date = hmac_sha256(&k_secret, date_stamp)?; - let k_region = hmac_sha256(&k_date, region)?; - let k_service = hmac_sha256(&k_region, service)?; - hmac_sha256(&k_service, "aws4_request") -} - -pub(crate) fn parse_assume_role_response(body: &str) -> Option { - let access_key_id = extract_xml_tag(body, "AccessKeyId")?; - let secret_access_key = extract_xml_tag(body, "SecretAccessKey")?; - let session_token = extract_xml_tag(body, "SessionToken")?; - let expiration = extract_xml_tag(body, "Expiration")?; - - Some(StsAssumeRoleCredentials { - access_key_id, - secret_access_key, - session_token, - expiration, - }) -} - -pub(crate) fn extract_xml_tag(document: &str, tag: &str) -> Option { - let open = format!("<{tag}>"); - let close = format!(""); - - let open_idx = document.find(&open)?; - let start = open_idx + open.len(); - let rest = &document[start..]; - let end = rest.find(&close)?; - - Some(rest[..end].trim().to_string()) -} diff --git a/crates/rustfs-admin/src/lib.rs b/crates/rustfs-admin/src/lib.rs deleted file mode 100644 index 0759a5b3..00000000 --- a/crates/rustfs-admin/src/lib.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2025 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Kube-agnostic RustFS admin/S3/STS client. -//! -//! This crate contains the wire-protocol logic (request signing, HTTP -//! dispatch, response parsing) needed to talk to a RustFS server's admin, -//! S3 and STS APIs. It has no dependency on `kube` or `Tenant` types; -//! kube/Tenant-aware wrappers live in the `operator` crate's -//! `sts::rustfs_client` module. - -mod admin_ops; -mod client; -mod core_ops; -mod credentials; -mod helpers; -mod pool_ops; -mod s3_ops; -mod sanitize; -mod sts_ops; - -pub use client::{ - CreateBucketResult, RustfsAdminClient, RustfsClientError, RustfsCredentials, - RustfsErasureBackend, RustfsErasureSetInfo, RustfsPoolDecommissionInfo, RustfsPoolListItem, - RustfsPoolStatus, RustfsServerInfo, RustfsServerUsage, -}; -pub use credentials::StsAssumeRoleCredentials; - -#[cfg(test)] -#[path = "tests.rs"] -mod tests; diff --git a/crates/rustfs-admin/src/sanitize.rs b/crates/rustfs-admin/src/sanitize.rs deleted file mode 100644 index c18cd372..00000000 --- a/crates/rustfs-admin/src/sanitize.rs +++ /dev/null @@ -1,353 +0,0 @@ -// Copyright 2025 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Redact sensitive key/value pairs and XML tags from upstream error bodies -//! before they are surfaced in `RustfsClientError` messages. - -const SENSITIVE_KEYS: [&str; 22] = [ - "token", - "password", - "accesskey", - "access_key", - "access-key", - "accesskeyid", - "access_key_id", - "access-key-id", - "secretkey", - "secret_key", - "secret-key", - "secretaccesskey", - "secret_access_key", - "secret-access-key", - "clientsecret", - "client_secret", - "client-secret", - "sessiontoken", - "session_token", - "session-token", - "credential", - "credentials", -]; - -pub(crate) fn redact_sensitive_pairs(message: &str) -> String { - let message = redact_sensitive_xml_tags(message); - redact_sensitive_key_value_pairs(&message) -} - -fn is_sensitive_key(key: &str) -> bool { - matches!( - normalize_key(key).as_str(), - "token" - | "password" - | "accesskey" - | "accesskeyid" - | "secretkey" - | "secretaccesskey" - | "clientsecret" - | "sessiontoken" - | "credential" - | "credentials" - ) -} - -fn normalize_key(raw: &str) -> String { - raw.trim_matches(|ch: char| !ch.is_ascii_alphanumeric() && ch != '-' && ch != '_') - .chars() - .filter(|ch| ch.is_ascii_alphanumeric()) - .collect::() - .to_ascii_lowercase() -} - -fn redact_sensitive_xml_tags(message: &str) -> String { - let mut output = String::with_capacity(message.len()); - let mut cursor = 0usize; - - while cursor < message.len() { - let Some(ch) = message[cursor..].chars().next() else { - break; - }; - - if ch == '<' - && let Some(replacement) = redact_xml_tag_at(message, cursor) - { - output.push_str(&replacement.redacted); - cursor = replacement.end; - continue; - } - - output.push(ch); - cursor += ch.len_utf8(); - } - - output -} - -struct XmlRedaction { - redacted: String, - end: usize, -} - -fn redact_xml_tag_at(message: &str, cursor: usize) -> Option { - let tag_end = cursor + message[cursor..].find('>')?; - let tag_content = &message[cursor + 1..tag_end]; - if tag_content.starts_with('/') || tag_content.starts_with('?') || tag_content.starts_with('!') - { - return None; - } - let tag_name_end = tag_content - .find(|ch: char| ch.is_whitespace() || ch == '/') - .unwrap_or(tag_content.len()); - let tag_name = &tag_content[..tag_name_end]; - if tag_name.is_empty() || !is_sensitive_key(tag_name) { - return None; - } - - let open_end = tag_end + 1; - let close = format!(""); - let close_start = open_end + message[open_end..].find(&close)?; - let close_end = close_start + close.len(); - - Some(XmlRedaction { - redacted: format!( - "{}{}", - &message[cursor..open_end], - &message[close_start..close_end] - ), - end: close_end, - }) -} - -fn redact_sensitive_key_value_pairs(message: &str) -> String { - let bytes = message.as_bytes(); - let mut output = String::with_capacity(message.len()); - let mut cursor = 0usize; - - while cursor < bytes.len() { - let mut matched = false; - - for key in SENSITIVE_KEYS { - let key_len = key.len(); - - let unquoted_match = matches_key_at(message, cursor, key); - let quoted_match = cursor + key_len + 2 <= bytes.len() - && matches!(bytes[cursor] as char, '"' | '\'') - && bytes[cursor + key_len + 1] == bytes[cursor] - && matches_key_at(message, cursor + 1, key); - - let (key_start, key_end, cursor_after_key) = if unquoted_match { - if cursor > 0 { - let prev = bytes[cursor - 1] as char; - if prev.is_ascii_alphanumeric() || prev == '_' || prev == '-' { - continue; - } - } - (cursor, cursor + key_len, cursor + key_len) - } else if quoted_match { - let key_start = cursor + 1; - (key_start, key_start + key_len, key_start + key_len + 1) - } else { - continue; - }; - - let candidate = &message[key_start..key_end]; - - let sep_index = skip_whitespace(message, cursor_after_key); - if sep_index >= bytes.len() || !matches!(bytes[sep_index] as char, '=' | ':') { - continue; - } - - let value_start = skip_whitespace(message, sep_index + 1); - let value_end = parse_value_end(message, value_start); - if value_end <= value_start || !is_sensitive_key(candidate) { - continue; - } - - output.push_str(&message[cursor..value_start]); - output.push_str(&redacted_value(&message[value_start..value_end])); - cursor = value_end; - matched = true; - break; - } - - if !matched { - let Some(ch) = message[cursor..].chars().next() else { - break; - }; - output.push(ch); - cursor += ch.len_utf8(); - } - } - - output -} - -fn parse_value_end(input: &str, start: usize) -> usize { - if start >= input.len() { - return start; - } - - let mut chars = input[start..].char_indices(); - let Some((_, first)) = chars.next() else { - return start; - }; - if first == '"' || first == '\'' { - let mut previous = first; - for (offset, ch) in chars { - if ch == first && previous != '\\' { - return start + offset + ch.len_utf8(); - } - previous = ch; - } - return input.len(); - } - - for (offset, ch) in input[start..].char_indices() { - if ch.is_whitespace() || matches!(ch, ',' | ';' | '}' | ']' | ')') { - return start + offset; - } - } - input.len() -} - -fn skip_whitespace(input: &str, start: usize) -> usize { - for (offset, ch) in input[start..].char_indices() { - if !ch.is_whitespace() { - return start + offset; - } - } - input.len() -} - -fn matches_key_at(message: &str, start: usize, key: &str) -> bool { - let end = start + key.len(); - end <= message.len() - && message.is_char_boundary(start) - && message.is_char_boundary(end) - && message[start..end].eq_ignore_ascii_case(key) -} - -fn redacted_value(original: &str) -> String { - if original.len() >= 2 { - let bytes = original.as_bytes(); - let first = bytes[0]; - let last = bytes[bytes.len() - 1]; - if (first == b'"' && last == b'"') || (first == b'\'' && last == b'\'') { - let quote = first as char; - return format!("{quote}{quote}"); - } - } - "".to_string() -} - -#[cfg(test)] -mod tests { - use super::redact_sensitive_pairs; - - #[test] - fn preserves_required_key_names() { - let message = "Vault backend requires kmsSecret referencing a Secret with key vault-token"; - - assert_eq!(redact_sensitive_pairs(message), message); - } - - #[test] - fn redacts_colon_and_json_secret_values() { - let message = - "kms config token: tok_123 password: p@ss accesskey: AKIA_TEST secretkey: SK_TEST"; - - let sanitized = redact_sensitive_pairs(message); - - assert!(sanitized.contains("token")); - assert!(sanitized.contains("password")); - assert!(sanitized.contains("accesskey")); - assert!(sanitized.contains("secretkey")); - assert!(!sanitized.contains("tok_123")); - assert!(!sanitized.contains("p@ss")); - assert!(!sanitized.contains("AKIA_TEST")); - assert!(!sanitized.contains("SK_TEST")); - } - - #[test] - fn redacts_key_name_variants_and_xml_tags() { - let message = - r#"clientSecret: oidc-secret {"access_key":"AKIA_JSON"} SK_XML"#; - - let sanitized = redact_sensitive_pairs(message); - - assert!(sanitized.contains("clientSecret: ")); - assert!(sanitized.contains(r#""access_key":"""#)); - assert!(sanitized.contains("")); - assert!(!sanitized.contains("oidc-secret")); - assert!(!sanitized.contains("AKIA_JSON")); - assert!(!sanitized.contains("SK_XML")); - } - - #[test] - fn redacts_sts_credential_field_names() { - let message = r#"AccessKeyId: AKIA_TEXT SecretAccessKey: SK_TEXT {"access_key_id":"AKIA_JSON","secret-access-key":"SK_JSON"} AKIA_XML SK_XML"#; - - let sanitized = redact_sensitive_pairs(message); - - assert!(sanitized.contains("AccessKeyId: ")); - assert!(sanitized.contains("SecretAccessKey: ")); - assert!(sanitized.contains(r#""access_key_id":"""#)); - assert!(sanitized.contains(r#""secret-access-key":"""#)); - assert!(sanitized.contains("")); - assert!(sanitized.contains("")); - assert!(!sanitized.contains("AKIA_TEXT")); - assert!(!sanitized.contains("SK_TEXT")); - assert!(!sanitized.contains("AKIA_JSON")); - assert!(!sanitized.contains("SK_JSON")); - assert!(!sanitized.contains("AKIA_XML")); - assert!(!sanitized.contains("SK_XML")); - } - - #[test] - fn handles_unicode_without_panicking() { - let message = "错误🔐 token: tok_123 用户=测试 secretkey: SK_TEST 完成"; - - let sanitized = redact_sensitive_pairs(message); - - assert!(sanitized.contains("错误🔐")); - assert!(sanitized.contains("用户=测试")); - assert!(sanitized.contains("完成")); - assert!(sanitized.contains("token: ")); - assert!(sanitized.contains("secretkey: ")); - assert!(!sanitized.contains("tok_123")); - assert!(!sanitized.contains("SK_TEST")); - } - - #[test] - fn redacts_unicode_quoted_values() { - let message = "{\"说明\":\"🔐\",\"secretkey\":\"秘密值\"}"; - - let sanitized = redact_sensitive_pairs(message); - - assert!(sanitized.contains("\"说明\":\"🔐\"")); - assert!(sanitized.contains("\"secretkey\":\"\"")); - assert!(!sanitized.contains("秘密值")); - } - - #[test] - fn redacts_after_unicode_whitespace() { - let message = "token:\u{3000}tok_123 secretkey:\u{2003}SK_TEST"; - - let sanitized = redact_sensitive_pairs(message); - - assert!(sanitized.contains("token:\u{3000}")); - assert!(sanitized.contains("secretkey:\u{2003}")); - assert!(!sanitized.contains("tok_123")); - assert!(!sanitized.contains("SK_TEST")); - } -} diff --git a/crates/rustfs-admin/src/tests.rs b/crates/rustfs-admin/src/tests.rs deleted file mode 100644 index 520ea7aa..00000000 --- a/crates/rustfs-admin/src/tests.rs +++ /dev/null @@ -1,1192 +0,0 @@ -// Copyright 2025 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Unit/integration tests for RustfsAdminClient split operation modules. - -use axum::{ - Router, - body::Body, - extract::State, - http::{Request, StatusCode}, - routing::{get, post, put}, -}; -use serde_json::Value; -use std::sync::Arc; -use tokio::sync::Mutex; - -use crate::client::{ - ADD_USER_PATH, ADMIN_SIGNING_SERVICE, CreateBucketResult, FORM_CONTENT_TYPE, JSON_CONTENT_TYPE, - LIST_CANNED_POLICIES_PATH, MAX_UPSTREAM_ERROR_BODY_BYTES, POOLS_DECOMMISSION_PATH, - POOLS_LIST_PATH, POOLS_STATUS_PATH, RustfsAdminClient, RustfsClientError, SERVER_INFO_PATH, - SET_POLICY_PATH, STS_SIGNING_SERVICE, USER_INFO_PATH, -}; -use crate::helpers::{ - build_canonical_query, build_form_body, derive_signing_key, extract_canned_policy_document, - hmac_sha256_hex, parse_assume_role_response, sha256_hex, -}; - -const TEST_ACCESS_KEY: &str = "access"; -const TEST_SECRET_KEY: &str = "secret"; -const TEST_REGION: &str = "us-east-1"; - -#[test] -fn canonical_query_uses_sigv4_uri_encoding_and_encoded_sort_order() { - let query = build_canonical_query(&[ - ("z", "a b~c/雪"), - ("a~", "second"), - ("a ", "first"), - ("amp", "&="), - ("dup", "z"), - ("dup", "a"), - ("empty", ""), - ("雪", "key"), - ]); - - assert_eq!( - query, - "%E9%9B%AA=key&a%20=first&=%26%3D&a~=second&dup=a&dup=z&empty=&z=a%20b~c%2F%E9%9B%AA" - ); -} - -#[test] -fn form_body_keeps_html_form_encoding() { - assert_eq!( - build_form_body(&[("Policy", "a b~c/雪")]), - "Policy=a+b%7Ec%2F%E9%9B%AA" - ); -} - -#[test] -fn duplicate_query_values_match_independent_sigv4_verification() { - let query = - build_canonical_query(&[("dup", "z z"), ("dup", "a+a"), ("dup", "雪"), ("empty", "")]); - assert_eq!(query, "dup=%E9%9B%AA&dup=a%2Ba&dup=z%20z&empty="); - - let client = RustfsAdminClient::new_with_base_url( - "https://rustfs.example.test:9000", - TEST_ACCESS_KEY, - TEST_SECRET_KEY, - ); - let signed = client - .sign_request("GET", "/synthetic", &query, "", None, ADMIN_SIGNING_SERVICE) - .unwrap(); - let request = CapturedRequest { - method: "GET".to_string(), - path: "/synthetic".to_string(), - query, - body: String::new(), - host: "rustfs.example.test:9000".to_string(), - content_type: String::new(), - amz_date: signed.amz_date, - payload_hash: signed.payload_hash, - authorization: signed.authorization, - }; - - assert_sigv4_matches_wire(&request, ADMIN_SIGNING_SERVICE); -} - -fn assert_oversized_upstream_body_hidden(err: RustfsClientError) { - assert_eq!( - err.to_string(), - format!( - "upstream returned 502 Bad Gateway: response body exceeded {MAX_UPSTREAM_ERROR_BODY_BYTES} bytes" - ) - ); -} - -#[test] -fn parse_assume_role_xml_success_and_failure() { - let body_ok = "AKISECTOKEN2026-01-01T00:00:00Z"; - let parsed = - parse_assume_role_response(body_ok).expect("valid assume role response should parse"); - - assert_eq!(parsed.access_key_id, "AKI"); - assert_eq!(parsed.secret_access_key, "SEC"); - assert_eq!(parsed.session_token, "TOKEN"); - assert_eq!(parsed.expiration, "2026-01-01T00:00:00Z"); - - assert!(parse_assume_role_response("").is_none()); -} - -#[test] -fn unexpected_status_includes_upstream_xml_error_summary() { - let err = RustfsClientError::unexpected_status_with_body( - StatusCode::BAD_REQUEST, - r#"InvalidRequestinvalid resource: unknown "*"abc"#, - ); - - let message = err.to_string(); - assert_eq!( - message, - r#"upstream returned 400 Bad Request: InvalidRequest: invalid resource: unknown "*""# - ); - assert!(!message.contains("")); -} - -#[test] -fn unexpected_status_includes_upstream_json_error_summary() { - let err = RustfsClientError::unexpected_status_with_body( - StatusCode::BAD_REQUEST, - r#"{"code":"InvalidRequest","message":"policy Resource must use ARN form"}"#, - ); - - assert_eq!( - err.to_string(), - "upstream returned 400 Bad Request: InvalidRequest: policy Resource must use ARN form" - ); -} - -#[test] -fn unexpected_status_redacts_sensitive_upstream_error_summary() { - let err = RustfsClientError::unexpected_status_with_body( - StatusCode::BAD_REQUEST, - r#"{"code":"InvalidRequest","message":"secretkey: SK_TEST clientSecret: oidc-secret SecretAccessKey: SK_STS AccessKeyId: AKIA_STS SK_XML AKIA_XML"}"#, - ); - - let message = err.to_string(); - assert!(message.contains("secretkey: ")); - assert!(message.contains("clientSecret: ")); - assert!(message.contains("SecretAccessKey: ")); - assert!(message.contains("AccessKeyId: ")); - assert!(message.contains("")); - assert!(message.contains("")); - assert!(!message.contains("SK_TEST")); - assert!(!message.contains("oidc-secret")); - assert!(!message.contains("SK_STS")); - assert!(!message.contains("AKIA_STS")); - assert!(!message.contains("SK_XML")); - assert!(!message.contains("AKIA_XML")); -} - -#[test] -fn unexpected_status_hides_truncated_unstructured_response_body() { - let retained_body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES); - let err = RustfsClientError::unexpected_status_with_limited_body( - StatusCode::BAD_GATEWAY, - &retained_body, - true, - ); - - assert_eq!( - err.to_string(), - format!( - "upstream returned 502 Bad Gateway: response body exceeded {MAX_UPSTREAM_ERROR_BODY_BYTES} bytes" - ) - ); -} - -#[tokio::test] -async fn unexpected_response_preserves_exact_limit_unstructured_response_body() { - let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES); - let router = Router::new().route( - ADD_USER_PATH, - put(move || { - let body = body.clone(); - async move { (StatusCode::BAD_GATEWAY, body) } - }), - ); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let err = client - .add_user("app-user", "secret123") - .await - .expect_err("exact limit body should still report the retained body"); - - let message = err.to_string(); - assert!(message.contains("upstream returned 502 Bad Gateway")); - assert!(!message.contains("response body exceeded")); - - server.abort(); -} - -#[tokio::test] -async fn unexpected_response_hides_over_limit_unstructured_response_body() { - let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES + 1); - let router = Router::new().route( - ADD_USER_PATH, - put(move || { - let body = body.clone(); - async move { (StatusCode::BAD_GATEWAY, body) } - }), - ); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let err = client - .add_user("app-user", "secret123") - .await - .expect_err("oversized body should be hidden"); - - assert_oversized_upstream_body_hidden(err); - - server.abort(); -} - -#[derive(Clone, Default)] -struct Capture { - method: Arc>, - path: Arc>, - query: Arc>, - body: Arc>, - host: Arc>, - content_type: Arc>, - amz_date: Arc>, - payload_hash: Arc>, - authorization: Arc>, - object_lock_header: Arc>, -} - -#[derive(Debug)] -struct CapturedRequest { - method: String, - path: String, - query: String, - body: String, - host: String, - content_type: String, - amz_date: String, - payload_hash: String, - authorization: String, -} - -impl Capture { - async fn request(&self) -> CapturedRequest { - CapturedRequest { - method: self.method.lock().await.clone(), - path: self.path.lock().await.clone(), - query: self.query.lock().await.clone(), - body: self.body.lock().await.clone(), - host: self.host.lock().await.clone(), - content_type: self.content_type.lock().await.clone(), - amz_date: self.amz_date.lock().await.clone(), - payload_hash: self.payload_hash.lock().await.clone(), - authorization: self.authorization.lock().await.clone(), - } - } -} - -fn request_header(req: &Request, name: &str) -> String { - req.headers() - .get(name) - .and_then(|value| value.to_str().ok()) - .unwrap_or("") - .to_string() -} - -async fn capture_signed_request(capture: &Capture, req: Request) { - let method = req.method().as_str().to_string(); - let path = req.uri().path().to_string(); - let query = req.uri().query().unwrap_or("").to_string(); - let host = request_header(&req, "host"); - let content_type = request_header(&req, "content-type"); - let amz_date = request_header(&req, "x-amz-date"); - let payload_hash = request_header(&req, "x-amz-content-sha256"); - let authorization = request_header(&req, "authorization"); - let body = axum::body::to_bytes(req.into_body(), usize::MAX) - .await - .unwrap(); - let body = String::from_utf8(body.to_vec()).unwrap(); - - *capture.method.lock().await = method; - *capture.path.lock().await = path; - *capture.query.lock().await = query; - *capture.body.lock().await = body; - *capture.host.lock().await = host; - *capture.content_type.lock().await = content_type; - *capture.amz_date.lock().await = amz_date; - *capture.payload_hash.lock().await = payload_hash; - *capture.authorization.lock().await = authorization; -} - -fn assert_sigv4_matches_wire(request: &CapturedRequest, service: &str) { - let calculated_payload_hash = sha256_hex(request.body.as_bytes()); - assert_eq!(request.payload_hash, calculated_payload_hash); - - let signed_header_names = if request.content_type.is_empty() { - "host;x-amz-content-sha256;x-amz-date" - } else { - "content-type;host;x-amz-content-sha256;x-amz-date" - }; - let mut canonical_headers = String::new(); - if !request.content_type.is_empty() { - canonical_headers.push_str("content-type:"); - canonical_headers.push_str(request.content_type.trim()); - canonical_headers.push('\n'); - } - canonical_headers.push_str("host:"); - canonical_headers.push_str(request.host.trim()); - canonical_headers.push_str("\nx-amz-content-sha256:"); - canonical_headers.push_str(request.payload_hash.trim()); - canonical_headers.push_str("\nx-amz-date:"); - canonical_headers.push_str(request.amz_date.trim()); - canonical_headers.push('\n'); - - let canonical_request = format!( - "{}\n{}\n{}\n{}\n{}\n{}", - request.method, - request.path, - request.query, - canonical_headers, - signed_header_names, - request.payload_hash - ); - let date_stamp = request - .amz_date - .get(..8) - .expect("x-amz-date must start with YYYYMMDD"); - let credential_scope = format!("{date_stamp}/{TEST_REGION}/{service}/aws4_request"); - let string_to_sign = format!( - "AWS4-HMAC-SHA256\n{}\n{}\n{}", - request.amz_date, - credential_scope, - sha256_hex(canonical_request.as_bytes()) - ); - let signing_key = - derive_signing_key(TEST_SECRET_KEY, date_stamp, TEST_REGION, service).unwrap(); - let signature = hmac_sha256_hex(&signing_key, &string_to_sign).unwrap(); - let expected_authorization = format!( - "AWS4-HMAC-SHA256 Credential={TEST_ACCESS_KEY}/{credential_scope}, SignedHeaders={signed_header_names}, Signature={signature}" - ); - - assert_eq!(request.authorization, expected_authorization); -} - -#[tokio::test] -async fn assume_role_request_targets_root_path_and_action_is_assume_role() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new().route( - "/", - post( - move |State(c): State, req: Request| async move { - capture_signed_request(&c, req).await; - - let response = - "AKISECTOKEN2026-01-01T00:00:00Z"; - (StatusCode::OK, response) - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - - let creds = client - .assume_role(Some(r#"{"Statement":[{"Resource":"a b~+/雪"}]}"#), 3600) - .await - .unwrap(); - assert_eq!(creds.access_key_id, "AKI"); - - let request = capture.request().await; - assert_eq!(request.path, "/"); - assert_eq!( - request.body, - "Action=AssumeRole&DurationSeconds=3600&Policy=%7B%22Statement%22%3A%5B%7B%22Resource%22%3A%22a+b%7E%2B%2F%E9%9B%AA%22%7D%5D%7D&Version=2011-06-15" - ); - assert!(request.query.is_empty()); - assert_eq!(request.content_type, FORM_CONTENT_TYPE); - assert_sigv4_matches_wire(&request, STS_SIGNING_SERVICE); - - server.abort(); -} - -#[tokio::test] -async fn info_canned_policy_uses_expected_path_and_query() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new() - .route( - "/rustfs/admin/v3/info-canned-policy", - get( - move |State(c): State, req: Request| async move { - let path = req.uri().path().to_string(); - let query = req.uri().query().unwrap_or("").to_string(); - let authorization = req - .headers() - .get("authorization") - .and_then(|value| value.to_str().ok()) - .unwrap_or("") - .to_string(); - - *c.path.lock().await = path; - *c.query.lock().await = query; - *c.authorization.lock().await = authorization; - - ( - StatusCode::OK, - "{\"policy_name\":\"tenant-policy\",\"policy\":{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"allow\",\"Effect\":\"Allow\"}]}}", - ) - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - - let policy = client.get_canned_policy("tenant-policy").await.unwrap(); - let policy_value = serde_json::from_str::(&policy).unwrap(); - assert_eq!(policy_value["Version"], "2012-10-17"); - assert_eq!(policy_value["Statement"][0]["Sid"], "allow"); - - assert_eq!( - &*capture.path.lock().await, - "/rustfs/admin/v3/info-canned-policy" - ); - assert!(capture.query.lock().await.contains("name=tenant-policy")); - assert!( - capture - .authorization - .lock() - .await - .contains("/s3/aws4_request") - ); - - server.abort(); -} - -#[tokio::test] -async fn list_canned_policies_extracts_policy_document_and_canonicalizes_json() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new() - .route( - LIST_CANNED_POLICIES_PATH, - get( - move |State(c): State, req: Request| async move { - let path = req.uri().path().to_string(); - let query = req.uri().query().unwrap_or("").to_string(); - - *c.path.lock().await = path; - *c.query.lock().await = query; - - ( - StatusCode::OK, - serde_json::json!({ - "tenant-policy": { - "policy_name":"tenant-policy", - "policy":{ - "Statement": [{ - "Resource": "arn:aws:s3:::tenant", - "Effect": "Allow", - "Action": "s3:GetObject" - }], - "Version":"2012-10-17" - } - }, - "inline-policy": { - "Version": "2012-10-17", - "Statement": [{ - "Sid": "inline", - "Action": "s3:ListBucket", - "Effect": "Allow", - "Resource": ["arn:aws:s3:::tenant*"] - }] - } - }) - .to_string(), - ) - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let policies = client.list_canned_policies().await.unwrap(); - - let tenant_policy = serde_json::from_str::(&policies["tenant-policy"]).unwrap(); - assert_eq!(tenant_policy["Version"], "2012-10-17"); - assert_eq!(tenant_policy["Statement"][0]["Action"], "s3:GetObject"); - - let inline_policy = serde_json::from_str::(&policies["inline-policy"]).unwrap(); - assert_eq!(inline_policy["Version"], "2012-10-17"); - assert_eq!(inline_policy["Statement"][0]["Sid"], "inline"); - assert_eq!(&*capture.path.lock().await, LIST_CANNED_POLICIES_PATH); - assert!(capture.query.lock().await.is_empty()); - - server.abort(); -} - -#[tokio::test] -async fn add_canned_policy_uses_expected_path_query_body_and_admin_signing() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new() - .route( - "/rustfs/admin/v3/add-canned-policy", - put( - move |State(c): State, req: Request| async move { - let path = req.uri().path().to_string(); - let query = req.uri().query().unwrap_or("").to_string(); - let authorization = req - .headers() - .get("authorization") - .and_then(|value| value.to_str().ok()) - .unwrap_or("") - .to_string(); - let body_bytes = axum::body::to_bytes(req.into_body(), usize::MAX) - .await - .unwrap(); - let body = String::from_utf8(body_bytes.to_vec()).unwrap(); - - *c.path.lock().await = path; - *c.query.lock().await = query; - *c.authorization.lock().await = authorization; - *c.body.lock().await = body; - - StatusCode::OK - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let policy = r#"{"Version":"2012-10-17","Statement":[]}"#; - - client - .add_canned_policy("tenant-policy", policy) - .await - .unwrap(); - - assert_eq!( - &*capture.path.lock().await, - "/rustfs/admin/v3/add-canned-policy" - ); - assert!(capture.query.lock().await.contains("name=tenant-policy")); - assert_eq!(&*capture.body.lock().await, policy); - assert!( - capture - .authorization - .lock() - .await - .contains("/s3/aws4_request") - ); - - server.abort(); -} - -#[tokio::test] -async fn add_canned_policy_reports_upstream_policy_parse_error() { - let router = Router::new().route( - "/rustfs/admin/v3/add-canned-policy", - put(|| async { - ( - StatusCode::BAD_REQUEST, - r#"InvalidRequestinvalid resource: unknown "*""#, - ) - }), - ); - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let policy = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}"#; - let err = client - .add_canned_policy("tenant-policy", policy) - .await - .expect_err("invalid RustFS policy should include upstream parse details"); - - let message = err.to_string(); - assert!(message.contains("upstream returned 400 Bad Request")); - assert!(message.contains(r#"InvalidRequest: invalid resource: unknown "*""#)); - assert!(!message.contains("")); - - server.abort(); -} - -#[tokio::test] -async fn remove_canned_policy_is_idempotent_when_already_gone() { - let router = Router::new().route( - "/rustfs/admin/v3/remove-canned-policy", - axum::routing::delete(|| async { (StatusCode::NOT_FOUND, "NoSuchPolicy") }), - ); - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - client - .remove_canned_policy("tenant-policy") - .await - .expect("removing an already-gone policy should be treated as success"); - - server.abort(); -} - -#[tokio::test] -async fn server_info_uses_expected_path_and_parses_wrapped_health_fields() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new() - .route( - SERVER_INFO_PATH, - get( - move |State(c): State, req: Request| async move { - let path = req.uri().path().to_string(); - let authorization = req - .headers() - .get("authorization") - .and_then(|value| value.to_str().ok()) - .unwrap_or("") - .to_string(); - - *c.path.lock().await = path; - *c.authorization.lock().await = authorization; - - ( - StatusCode::OK, - serde_json::json!({ - "info": { - "usage": {"size": 42}, - "backend": { - "onlineDisks": 3, - "offlineDisks": 1, - "standardSCParity": 2, - "totalSets": [1], - "totalDrivesPerSet": [4] - }, - "pools": { - "0": { - "0": { - "rawUsage": 100, - "rawCapacity": 400, - "usage": 50, - "objectsCount": 2, - "healDisks": 1 - } - } - } - }, - "admin_discovery": { - "runtimeCapabilities": "/rustfs/admin/v4/runtime/capabilities", - "clusterSnapshot": "/rustfs/admin/v4/cluster/snapshot", - "extensionsCatalog": "/rustfs/admin/v4/extensions/catalog" - }, - }) - .to_string(), - ) - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let info = client.server_info().await.unwrap(); - - let backend = info.backend.unwrap(); - assert_eq!(backend.online_disks, 3); - assert_eq!(backend.offline_disks, 1); - assert_eq!(backend.standard_sc_parity, Some(2)); - assert_eq!(info.usage.unwrap().size, 42); - assert_eq!(info.pools.unwrap()["0"]["0"].raw_capacity, 400); - assert_eq!(&*capture.path.lock().await, SERVER_INFO_PATH); - assert!( - capture - .authorization - .lock() - .await - .contains("/s3/aws4_request") - ); - - server.abort(); -} - -#[tokio::test] -async fn list_pools_parses_current_rustfs_pool_shape() { - let router = Router::new().route( - POOLS_LIST_PATH, - get(|| async { - ( - StatusCode::OK, - r#"[{"id":1,"cmdline":"http://tenant-pool-a-{0...3}.tenant-hl.ns.svc.cluster.local:9000/data/rustfs{0...3}","lastUpdate":"2026-05-20T00:00:00Z","totalSize":100,"currentSize":50,"usedSize":25,"used":25.0,"status":"running","decommissionInfo":{"startTime":"2026-05-20T00:00:00Z","complete":false,"failed":false,"canceled":false,"objectsDecommissioned":7,"objectsDecommissionedFailed":1,"bytesDecommissioned":9,"bytesDecommissionedFailed":2}}]"#, - ) - }), - ); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - - let pools = client.list_pools().await.unwrap(); - - assert_eq!(pools[0].id, 1); - assert_eq!(pools[0].status, "running"); - assert_eq!( - pools[0] - .decommission - .as_ref() - .and_then(|info| info.objects_decommissioned), - Some(7) - ); - - server.abort(); -} - -#[tokio::test] -async fn pool_decommission_start_uses_by_id_query_and_admin_signing() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new() - .route( - POOLS_DECOMMISSION_PATH, - post( - move |State(c): State, req: Request| async move { - *c.path.lock().await = req.uri().path().to_string(); - *c.query.lock().await = req.uri().query().unwrap_or("").to_string(); - *c.authorization.lock().await = req - .headers() - .get("authorization") - .and_then(|value| value.to_str().ok()) - .unwrap_or("") - .to_string(); - - StatusCode::OK - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - - client.start_pool_decommission_by_id("1").await.unwrap(); - - assert_eq!(&*capture.path.lock().await, POOLS_DECOMMISSION_PATH); - assert_eq!(&*capture.query.lock().await, "by-id=true&pool=1"); - assert!( - capture - .authorization - .lock() - .await - .contains("/s3/aws4_request") - ); - - server.abort(); -} - -#[tokio::test] -async fn pool_status_uses_by_id_query_and_parses_decommission_info() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new() - .route( - POOLS_STATUS_PATH, - get( - move |State(c): State, req: Request| async move { - *c.path.lock().await = req.uri().path().to_string(); - *c.query.lock().await = req.uri().query().unwrap_or("").to_string(); - - ( - StatusCode::OK, - r#"{"id":1,"cmdline":"http://tenant-pool-a-{0...3}.tenant-hl.ns.svc.cluster.local:9000/data/rustfs{0...3}","lastUpdate":"2026-05-20T00:00:00Z","decommissionInfo":{"startTime":"2026-05-20T00:00:00Z","complete":true,"failed":false,"canceled":false,"objectsDecommissioned":10,"objectsDecommissionedFailed":0,"bytesDecommissioned":20,"bytesDecommissionedFailed":0}}"#, - ) - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - - let status = client.pool_status_by_id("1").await.unwrap(); - - assert_eq!(status.id, 1); - assert_eq!(&*capture.path.lock().await, POOLS_STATUS_PATH); - assert_eq!(&*capture.query.lock().await, "by-id=true&pool=1"); - assert_eq!( - status.decommission.and_then(|info| info.complete), - Some(true) - ); - - server.abort(); -} - -#[tokio::test] -async fn add_user_uses_expected_path_query_and_body() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new() - .route( - ADD_USER_PATH, - put( - move |State(c): State, req: Request| async move { - capture_signed_request(&c, req).await; - StatusCode::OK - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - client.add_user("app user~+/雪", "secret123").await.unwrap(); - - let request = capture.request().await; - assert_eq!(request.path, ADD_USER_PATH); - assert_eq!(request.query, "accessKey=app%20user~%2B%2F%E9%9B%AA"); - assert_eq!( - request.body, - r#"{"secretKey":"secret123","status":"enabled"}"# - ); - assert_eq!(request.content_type, JSON_CONTENT_TYPE); - assert_sigv4_matches_wire(&request, ADMIN_SIGNING_SERVICE); - - server.abort(); -} - -#[tokio::test] -async fn remove_user_is_idempotent_when_already_gone() { - let router = Router::new().route( - "/rustfs/admin/v3/remove-user", - axum::routing::delete(|| async { (StatusCode::NOT_FOUND, "NoSuchUser") }), - ); - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - client - .remove_user("app-user") - .await - .expect("removing an already-gone user should be treated as success"); - - server.abort(); -} - -#[tokio::test] -async fn user_exists_limits_unexpected_error_response_body() { - let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES + 1); - let router = Router::new().route( - USER_INFO_PATH, - get(move || { - let body = body.clone(); - async move { (StatusCode::BAD_GATEWAY, body) } - }), - ); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let err = client - .user_exists("app-user") - .await - .expect_err("unexpected user lookup error should hide oversized body"); - - assert_oversized_upstream_body_hidden(err); - - server.abort(); -} - -#[tokio::test] -async fn set_user_policy_uses_single_authoritative_mapping_call() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new() - .route( - SET_POLICY_PATH, - put( - move |State(c): State, req: Request| async move { - *c.path.lock().await = req.uri().path().to_string(); - *c.query.lock().await = req.uri().query().unwrap_or("").to_string(); - StatusCode::OK - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - client - .set_user_policy( - "app-user", - &["app-readwrite".to_string(), "diagnostics".to_string()], - ) - .await - .unwrap(); - - assert_eq!(&*capture.path.lock().await, SET_POLICY_PATH); - assert_eq!( - &*capture.query.lock().await, - "isGroup=false&policyName=app-readwrite%2Cdiagnostics&userOrGroup=app-user" - ); - - server.abort(); -} - -#[tokio::test] -async fn set_user_policy_rejects_empty_policy_list() { - let client = RustfsAdminClient::new_with_base_url("http://127.0.0.1:1", "access", "secret"); - - let err = client - .set_user_policy("app-user", &[]) - .await - .expect_err("empty policy list should be rejected before request"); - - assert!(matches!(err, RustfsClientError::InvalidPolicyName)); -} - -#[tokio::test] -async fn bucket_object_lock_enabled_parses_enabled_response() { - let router = Router::new().route( - "/app-data", - get(|req: Request| async move { - assert_eq!(req.uri().query().unwrap_or(""), "object-lock="); - ( - StatusCode::OK, - "Enabled", - ) - }), - ); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - - assert!(client.bucket_object_lock_enabled("app-data").await.unwrap()); - - server.abort(); -} - -#[tokio::test] -async fn bucket_object_lock_enabled_limits_unexpected_error_response_body() { - let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES + 1); - let router = Router::new().route( - "/app-data", - get(move |req: Request| { - let body = body.clone(); - async move { - assert_eq!(req.uri().query().unwrap_or(""), "object-lock="); - (StatusCode::BAD_GATEWAY, body) - } - }), - ); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let err = client - .bucket_object_lock_enabled("app-data") - .await - .expect_err("unexpected object-lock error should hide oversized body"); - - assert_oversized_upstream_body_hidden(err); - - server.abort(); -} - -#[tokio::test] -async fn create_bucket_sends_object_lock_header_and_region_body() { - let capture = Capture::default(); - let route_capture = capture.clone(); - - let router = Router::new() - .route( - "/app-data", - put( - move |State(c): State, req: Request| async move { - *c.path.lock().await = req.uri().path().to_string(); - *c.object_lock_header.lock().await = req - .headers() - .get("x-amz-bucket-object-lock-enabled") - .and_then(|value| value.to_str().ok()) - .unwrap_or("") - .to_string(); - let body_bytes = axum::body::to_bytes(req.into_body(), usize::MAX) - .await - .unwrap(); - *c.body.lock().await = String::from_utf8(body_bytes.to_vec()).unwrap(); - StatusCode::OK - }, - ), - ) - .with_state(route_capture.clone()); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let result = client - .create_bucket("app-data", Some("us-west-2"), true) - .await - .unwrap(); - - assert_eq!(result, CreateBucketResult::Created); - assert_eq!(&*capture.path.lock().await, "/app-data"); - assert_eq!(&*capture.object_lock_header.lock().await, "true"); - assert!( - capture - .body - .lock() - .await - .contains("us-west-2") - ); - - server.abort(); -} - -#[tokio::test] -async fn create_bucket_limits_unexpected_error_response_body() { - let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES + 1); - let router = Router::new().route( - "/app-data", - put(move || { - let body = body.clone(); - async move { (StatusCode::BAD_GATEWAY, body) } - }), - ); - - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let err = client - .create_bucket("app-data", None, false) - .await - .expect_err("unexpected bucket create error should hide oversized body"); - - assert_oversized_upstream_body_hidden(err); - - server.abort(); -} - -#[tokio::test] -async fn delete_bucket_is_idempotent_when_already_gone() { - let router = Router::new().route( - "/app-data", - axum::routing::delete(|| async { (StatusCode::NOT_FOUND, "NoSuchBucket") }), - ); - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - - let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - client - .delete_bucket("app-data") - .await - .expect("removing an already-gone bucket should be treated as success"); - - server.abort(); -} - -#[test] -fn extract_canned_policy_document_accepts_raw_policy_document() { - let raw_policy = - "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"raw\",\"Effect\":\"Allow\"}]}"; - - let policy = extract_canned_policy_document(raw_policy).unwrap(); - - let policy_value = serde_json::from_str::(&policy).unwrap(); - assert_eq!(policy_value["Version"], "2012-10-17"); - assert_eq!(policy_value["Statement"][0]["Sid"], "raw"); -} diff --git a/deploy/rustfs-operator/README.md b/deploy/rustfs-operator/README.md index 7c6dfb39..b8eb3860 100755 --- a/deploy/rustfs-operator/README.md +++ b/deploy/rustfs-operator/README.md @@ -76,27 +76,6 @@ manifests remain consistent. | `sts.service.type` | Kubernetes Service type for STS | `ClusterIP` | | `sts.service.port` | Kubernetes Service port for STS | `4223` | -### COSI Driver Configuration (experimental) - -Install the COSI controller first (`release-0.2`), then enable the chart-managed driver: - -```bash -kubectl apply -k 'github.com/kubernetes-sigs/container-object-storage-interface//?ref=release-0.2' -helm upgrade --install rustfs-operator deploy/rustfs-operator/ --set cosi.enabled=true -``` - -| Parameter | Description | Default | -|-----------|-------------|---------| -| `cosi.enabled` | Deploy the RustFS COSI driver + sidecar | `false` | -| `cosi.driverName` | Driver name used in BucketClass / BucketAccessClass | `rustfs.objectstorage.k8s.io` | -| `cosi.replicas` | Driver Deployment replicas | `1` | -| `cosi.image.repository` | Driver image (empty = operator image) | `""` | -| `cosi.image.tag` | Driver image tag (empty = operator tag) | `""` | -| `cosi.sidecar.image.repository` | Official COSI provisioner sidecar image | `gcr.io/k8s-staging-sig-storage/objectstorage-sidecar` | -| `cosi.sidecar.image.tag` | Sidecar image tag | `v20230130-v0.1.0-24-gc0cf995` | - -Example manifests: `examples/cosi/`. See the operator user guide section on COSI. - The RustFS operator STS endpoint intentionally uses an explicit Tenant route: ```text diff --git a/deploy/rustfs-operator/templates/NOTES.txt b/deploy/rustfs-operator/templates/NOTES.txt index d810a244..91c3351f 100755 --- a/deploy/rustfs-operator/templates/NOTES.txt +++ b/deploy/rustfs-operator/templates/NOTES.txt @@ -48,15 +48,4 @@ To open the Operator Console locally: {{ end }} {{ end }} - -{{- if .Values.cosi.enabled }} -COSI driver is enabled (experimental, v1alpha1). Driver name: {{ .Values.cosi.driverName }} - -Ensure the COSI controller is installed: - - kubectl apply -k 'github.com/kubernetes-sigs/container-object-storage-interface//?ref=release-0.2' - -Example manifests: examples/cosi/ - -{{- end }} For more information, visit: {{ .Chart.Home }} diff --git a/deploy/rustfs-operator/templates/_helpers.tpl b/deploy/rustfs-operator/templates/_helpers.tpl index 4d0e475c..6903f303 100755 --- a/deploy/rustfs-operator/templates/_helpers.tpl +++ b/deploy/rustfs-operator/templates/_helpers.tpl @@ -86,23 +86,3 @@ Create the name of the console service account to use {{- default "default" .Values.console.serviceAccount.name }} {{- end }} {{- end }} - -{{/* -COSI driver service account name -*/}} -{{- define "rustfs-operator.cosiServiceAccountName" -}} -{{- if .Values.cosi.serviceAccount.create }} -{{- default (printf "%s-cosi" (include "rustfs-operator.fullname" .)) .Values.cosi.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.cosi.serviceAccount.name }} -{{- end }} -{{- end }} - -{{/* -COSI driver image (falls back to operator image) -*/}} -{{- define "rustfs-operator.cosiImage" -}} -{{- $repo := default .Values.operator.image.repository .Values.cosi.image.repository -}} -{{- $tag := default .Values.operator.image.tag .Values.cosi.image.tag -}} -{{- printf "%s:%s" $repo $tag -}} -{{- end }} diff --git a/deploy/rustfs-operator/templates/cosi-deployment.yaml b/deploy/rustfs-operator/templates/cosi-deployment.yaml deleted file mode 100644 index ea4c16f7..00000000 --- a/deploy/rustfs-operator/templates/cosi-deployment.yaml +++ /dev/null @@ -1,88 +0,0 @@ -{{- if .Values.cosi.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "rustfs-operator.fullname" . }}-cosi - namespace: {{ include "rustfs-operator.namespace" . }} - labels: - {{- include "rustfs-operator.labels" . | nindent 4 }} - app.kubernetes.io/component: cosi - {{- with .Values.commonAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - replicas: {{ .Values.cosi.replicas }} - selector: - matchLabels: - {{- include "rustfs-operator.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: cosi - template: - metadata: - labels: - {{- include "rustfs-operator.selectorLabels" . | nindent 8 }} - app.kubernetes.io/component: cosi - spec: - serviceAccountName: {{ include "rustfs-operator.cosiServiceAccountName" . }} - {{- with (.Values.cosi.imagePullSecrets | default .Values.operator.imagePullSecrets) }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.cosi.podSecurityContext }} - securityContext: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - - name: socket - emptyDir: {} - containers: - - name: rustfs-cosi-driver - image: {{ include "rustfs-operator.cosiImage" . }} - imagePullPolicy: {{ .Values.cosi.image.pullPolicy }} - command: ["/app/rustfs-cosi-driver"] - env: - - name: COSI_ENDPOINT - value: unix:///var/lib/cosi/cosi.sock - - name: RUST_LOG - value: info - volumeMounts: - - name: socket - mountPath: /var/lib/cosi - {{- with .Values.cosi.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.cosi.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - - name: objectstorage-provisioner - image: "{{ .Values.cosi.sidecar.image.repository }}:{{ .Values.cosi.sidecar.image.tag }}" - imagePullPolicy: {{ .Values.cosi.sidecar.image.pullPolicy }} - args: - - "--v=4" - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: socket - mountPath: /var/lib/cosi - {{- with .Values.cosi.sidecar.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.cosi.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.cosi.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.cosi.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} -{{- end }} diff --git a/deploy/rustfs-operator/templates/cosi-rbac.yaml b/deploy/rustfs-operator/templates/cosi-rbac.yaml deleted file mode 100644 index e7abda07..00000000 --- a/deploy/rustfs-operator/templates/cosi-rbac.yaml +++ /dev/null @@ -1,59 +0,0 @@ -{{- if and .Values.cosi.enabled .Values.cosi.rbac.create }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "rustfs-operator.fullname" . }}-cosi - labels: - {{- include "rustfs-operator.labels" . | nindent 4 }} - app.kubernetes.io/component: cosi - {{- with .Values.commonAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -rules: - # Driver reads Tenant admin Secrets and optional TLS CA ConfigMaps referenced by BucketClass parameters. - - apiGroups: [""] - resources: ["secrets", "configmaps"] - verbs: ["get", "list", "watch"] - # Sidecar reconciles COSI API objects (v1alpha1). - - apiGroups: ["objectstorage.k8s.io"] - resources: - - buckets - - bucketaccesses - - bucketclaims - - bucketaccessclasses - - bucketclasses - verbs: ["get", "list", "watch", "update", "patch", "create", "delete"] - - apiGroups: ["objectstorage.k8s.io"] - resources: - - buckets/status - - bucketaccesses/status - - bucketclaims/status - verbs: ["get", "update", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["create", "patch", "update"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "rustfs-operator.fullname" . }}-cosi - labels: - {{- include "rustfs-operator.labels" . | nindent 4 }} - app.kubernetes.io/component: cosi - {{- with .Values.commonAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "rustfs-operator.fullname" . }}-cosi -subjects: - - kind: ServiceAccount - name: {{ include "rustfs-operator.cosiServiceAccountName" . }} - namespace: {{ include "rustfs-operator.namespace" . }} -{{- end }} diff --git a/deploy/rustfs-operator/templates/cosi-serviceaccount.yaml b/deploy/rustfs-operator/templates/cosi-serviceaccount.yaml deleted file mode 100644 index ab743972..00000000 --- a/deploy/rustfs-operator/templates/cosi-serviceaccount.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- if .Values.cosi.enabled }} -{{- if .Values.cosi.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "rustfs-operator.cosiServiceAccountName" . }} - namespace: {{ include "rustfs-operator.namespace" . }} - labels: - {{- include "rustfs-operator.labels" . | nindent 4 }} - app.kubernetes.io/component: cosi - {{- if or .Values.cosi.serviceAccount.annotations .Values.commonAnnotations }} - annotations: - {{- with .Values.commonAnnotations }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .Values.cosi.serviceAccount.annotations }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- end }} -{{- end }} -{{- end }} diff --git a/deploy/rustfs-operator/values.yaml b/deploy/rustfs-operator/values.yaml index 0edda480..6dfc20e8 100755 --- a/deploy/rustfs-operator/values.yaml +++ b/deploy/rustfs-operator/values.yaml @@ -307,67 +307,3 @@ console: # - secretName: console-tls # hosts: # - console.example.com - -# COSI (Container Object Storage Interface) driver — experimental, v1alpha1. -# Requires the COSI controller/CRDs installed separately: -# kubectl apply -k 'github.com/kubernetes-sigs/container-object-storage-interface//?ref=release-0.2' -cosi: - enabled: false - - driverName: rustfs.objectstorage.k8s.io - - replicas: 1 - - # Driver binary image (defaults to the operator image which embeds rustfs-cosi-driver). - image: - repository: "" # defaults to operator.image.repository - tag: "" # defaults to operator.image.tag - pullPolicy: IfNotPresent - - imagePullSecrets: [] - - # Official COSI provisioner sidecar (UNIX socket to the driver). - sidecar: - image: - repository: gcr.io/k8s-staging-sig-storage/objectstorage-sidecar - tag: v20230130-v0.1.0-24-gc0cf995 - pullPolicy: IfNotPresent - resources: - requests: - cpu: 10m - memory: 32Mi - limits: - memory: 256Mi - - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 500m - memory: 256Mi - - podSecurityContext: - fsGroup: 65534 - - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 65534 - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - - nodeSelector: {} - tolerations: [] - affinity: {} - - serviceAccount: - create: true - annotations: {} - name: "" - - rbac: - create: true diff --git a/docs/operator-user-guide.md b/docs/operator-user-guide.md index d31b89b4..c7e01461 100644 --- a/docs/operator-user-guide.md +++ b/docs/operator-user-guide.md @@ -930,49 +930,7 @@ Current STS constraints: - Caller-supplied `Policy` request parameters are rejected; issued credentials use the matched `PolicyBinding` policies. - Tenants requiring client certificates for upstream Tenant calls are rejected by Operator STS. -## 10. COSI (Experimental) - -The operator ships an optional [Container Object Storage Interface](https://github.com/kubernetes-sigs/container-object-storage-interface) (COSI) **v1alpha1** driver (`rustfs.objectstorage.k8s.io`). Applications request buckets with `BucketClaim` / `BucketAccess` instead of embedding bucket lists on the Tenant CR. Tenant bootstrap provisioning (`spec.buckets`) and COSI can coexist. - -### 10.1 Prerequisites - -1. Install the COSI controller and CRDs (`release-0.2`): - -```bash -kubectl apply -k 'github.com/kubernetes-sigs/container-object-storage-interface//?ref=release-0.2' -``` - -2. Enable the driver in the Helm chart: - -```bash -helm upgrade --install rustfs-operator ./deploy/rustfs-operator \ - --set cosi.enabled=true -``` - -The chart deploys a pod with two containers: `rustfs-cosi-driver` and the official `objectstorage-sidecar`, sharing `unix:///var/lib/cosi/cosi.sock`. - -### 10.2 Point a BucketClass at a Tenant - -`BucketClass` / `BucketAccessClass` parameters (Rook-style) identify the Tenant admin credentials and S3 endpoint: - -| Parameter | Required | Description | -|-----------|----------|-------------| -| `objectStoreUserSecretName` | yes | Secret with `accesskey` / `secretkey` (Tenant `spec.credsSecret`) | -| `objectStoreUserSecretNamespace` | yes | Namespace of that Secret | -| `endpoint` | yes | S3 URL, e.g. `http://{tenant}-io.{ns}.svc:9000` | -| `region` | no | Defaults to `us-east-1` | -| `policy` | no | On `BucketAccessClass`: `readonly` or `readwrite` (default) | -| `tlsCAConfigMapName` / `tlsCAConfigMapNamespace` | no | PEM CA for HTTPS Tenants | - -See `examples/cosi/` for full manifests. - -### 10.3 Limitations - -- Experimental; protocol **S3** and authentication **KEY** only. -- Does not replace Operator STS / `PolicyBinding` for temporary credentials. -- Driver name is fixed: `rustfs.objectstorage.k8s.io`. - -## 11. Monitoring and Status +## 10. Monitoring and Status Check Tenant status: @@ -1030,7 +988,7 @@ operator: enabled: true ``` -## 12. Operations +## 11. Operations ### Change RustFS Image @@ -1068,7 +1026,7 @@ kubectl create secret generic rustfs-admin-creds \ kubectl rollout restart statefulset -n -l rustfs.tenant= ``` -## 13. Troubleshooting +## 12. Troubleshooting ### Tenant is Blocked @@ -1126,7 +1084,7 @@ kubectl logs -n rustfs-system \ For the RustFS Tenant Console, use the Tenant admin credentials from `spec.credsSecret` or configured RustFS environment variables. -## 14. Best Practices +## 13. Best Practices - Use `spec.credsSecret` or an external secret manager for production credentials. - Enable Kubernetes Secret encryption at rest. @@ -1139,7 +1097,15 @@ For the RustFS Tenant Console, use the Tenant admin credentials from `spec.creds - Keep Tenant examples under version control, but never commit raw Secret values. - Check `status.conditions` before debugging lower-level StatefulSets. -## 15. Related Documentation +## 13.1 COSI `preferredAccessKey` + +When using the RustFS COSI driver (`rustfs.objectstorage.k8s.io`): + +- Prefer omitting `preferredAccessKey` so each `BucketAccess` gets a unique account id derived from the COSI grant name (`ba-`), matching Ceph COSI isolation. +- If you set `preferredAccessKey` (or `accessKey`), the value must be unique per `BucketAccess`. Reusing the same key across claims is rejected with `AlreadyExists` so credentials are never rotated out from under another workload. +- Grant retries for the same `BucketAccess` are idempotent and return the same secret; the driver does not overwrite an existing user's secret key. + +## 14. Related Documentation - [Project README](../README.md) - [Deployment guide](../deploy/README.md) diff --git a/e2e/Cargo.lock b/e2e/Cargo.lock index eedfc07c..b338af3d 100644 --- a/e2e/Cargo.lock +++ b/e2e/Cargo.lock @@ -2526,6 +2526,7 @@ dependencies = [ "const-str", "futures", "hex", + "hmac 0.12.1", "hostname", "http 1.4.0", "http-body-util", @@ -2535,8 +2536,8 @@ dependencies = [ "kube", "kube-leader-election", "rcgen", + "reqwest", "ring", - "rustfs-admin", "rustls 0.23.40", "rustls-pemfile", "rustls-webpki 0.103.13", @@ -2557,6 +2558,7 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", + "url", "utoipa", "utoipa-swagger-ui", ] @@ -3108,20 +3110,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rustfs-admin" -version = "0.1.0" -dependencies = [ - "chrono", - "hex", - "hmac 0.12.1", - "reqwest", - "serde", - "serde_json", - "sha2 0.10.9", - "url", -] - [[package]] name = "rustfs-operator-e2e" version = "0.1.0" diff --git a/e2e/tests/sts_functional.rs b/e2e/tests/sts_functional.rs index 16d1af75..3855e967 100644 --- a/e2e/tests/sts_functional.rs +++ b/e2e/tests/sts_functional.rs @@ -21,10 +21,7 @@ use k8s_openapi::api::core::v1 as corev1; use kube::Api; use operator::{ console::state::AppState, - sts::{ - rustfs_client::{RustfsAdminClient, load_tenant_credentials, load_tenant_tls_ca}, - server::routes, - }, + sts::{rustfs_client::RustfsAdminClient, server::routes}, types::v1alpha1::tenant::Tenant, }; use rustfs_operator_e2e::framework::{ @@ -276,7 +273,7 @@ async fn ensure_rustfs_canned_policy( let rustfs_url = local_https_base_url(&rustfs_host, &rustfs_port_forward_spec); let mut rustfs_port_forward = PortForwardSpec::start_tenant_io(config).context("start RustFS tenant IO port-forward")?; - let tenant_ca = load_tenant_tls_ca(kube_client, tenant) + let tenant_ca = RustfsAdminClient::load_tenant_tls_ca(kube_client, tenant) .await .context("load TLS Tenant CA")? .context("TLS Tenant should publish a CA Secret reference")?; @@ -287,7 +284,7 @@ async fn ensure_rustfs_canned_policy( )?; wait_for_port_forward(&mut rustfs_port_forward, &rustfs_url, &rustfs_probe_client).await?; - let credentials = load_tenant_credentials(kube_client, tenant) + let credentials = RustfsAdminClient::load_tenant_credentials(kube_client, tenant) .await .context("load RustFS tenant credentials")?; let rustfs_admin = RustfsAdminClient::new_with_base_url_and_http_client( diff --git a/examples/README.md b/examples/README.md index 4d1f1ec5..156b096e 100755 --- a/examples/README.md +++ b/examples/README.md @@ -12,7 +12,6 @@ This directory contains example Tenant configurations for the RustFS Kubernetes | [simple-tenant.yaml](./simple-tenant.yaml) | Documentation Reference | ⭐⭐ Moderate | Configurable | Learning all options | | [secret-credentials-tenant.yaml](./secret-credentials-tenant.yaml) | Secret-based Credentials | ⭐ Simple | Configurable | **Production credential security** | | [provisioning-tenant.yaml](./provisioning-tenant.yaml) | Policy/User/Bucket Provisioning | ⭐⭐ Moderate | 40Gi | Tenant bootstrap automation | -| [cosi/](./cosi/) | COSI BucketClaim / BucketAccess | ⭐⭐ Moderate | — | Dynamic buckets via COSI v1alpha1 | | [multi-cert-tls-tenant.yaml](./multi-cert-tls-tenant.yaml) | Public/Internal TLS | ⭐⭐ Moderate | 40Gi | Separate SNI certificates | | [multi-pool-tenant.yaml](./multi-pool-tenant.yaml) | Multiple Pools | ⭐⭐ Moderate | ~160Gi | Multi-pool setups | | [production-ha-tenant.yaml](./production-ha-tenant.yaml) | Production HA | ⭐⭐⭐ Advanced | 6.4TB | HA with zone distribution | diff --git a/examples/cosi/awscli-pod.yaml b/examples/cosi/awscli-pod.yaml deleted file mode 100644 index 828ca39a..00000000 --- a/examples/cosi/awscli-pod.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Sample app that mounts the COSI BucketAccess Secret and prints BucketInfo. -apiVersion: v1 -kind: Pod -metadata: - name: cosi-awscli - namespace: default -spec: - containers: - - name: awscli - image: amazon/aws-cli:2.15.0 - command: ["sleep", "3600"] - volumeMounts: - - name: cosi-secrets - mountPath: /data/cosi - readOnly: true - volumes: - - name: cosi-secrets - secret: - secretName: sample-bucket-access-secret - restartPolicy: Never diff --git a/examples/cosi/bucketaccess.yaml b/examples/cosi/bucketaccess.yaml deleted file mode 100644 index 941f99cb..00000000 --- a/examples/cosi/bucketaccess.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: objectstorage.k8s.io/v1alpha1 -kind: BucketAccess -metadata: - name: sample-bucket-access - namespace: default -spec: - bucketClaimName: sample-bucket-claim - bucketAccessClassName: rustfs-bucket-access-class - credentialsSecretName: sample-bucket-access-secret - protocol: s3 diff --git a/examples/cosi/bucketaccessclass.yaml b/examples/cosi/bucketaccessclass.yaml deleted file mode 100644 index 28af551b..00000000 --- a/examples/cosi/bucketaccessclass.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: objectstorage.k8s.io/v1alpha1 -kind: BucketAccessClass -metadata: - name: rustfs-bucket-access-class -driverName: rustfs.objectstorage.k8s.io -authenticationType: KEY -parameters: - objectStoreUserSecretName: provisioning-admin-creds - objectStoreUserSecretNamespace: default - endpoint: http://provisioning-demo-io.default.svc:9000 - region: us-east-1 - # readonly | readwrite (default) - policy: readwrite diff --git a/examples/cosi/bucketclaim.yaml b/examples/cosi/bucketclaim.yaml deleted file mode 100644 index bc7b7014..00000000 --- a/examples/cosi/bucketclaim.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: objectstorage.k8s.io/v1alpha1 -kind: BucketClaim -metadata: - name: sample-bucket-claim - namespace: default -spec: - bucketClassName: rustfs-bucketclass - protocols: - - s3 diff --git a/examples/cosi/bucketclass.yaml b/examples/cosi/bucketclass.yaml deleted file mode 100644 index d15f3329..00000000 --- a/examples/cosi/bucketclass.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Example BucketClass for a RustFS Tenant. -# -# Prerequisites: -# 1. COSI controller: kubectl apply -k 'github.com/kubernetes-sigs/container-object-storage-interface//?ref=release-0.2' -# 2. Operator chart with cosi.enabled=true -# 3. A ready Tenant whose admin Secret matches objectStoreUserSecretName/Namespace -# 4. endpoint points at the Tenant S3 Service (port 9000) -# -# Replace secret name/namespace and endpoint for your Tenant. - -apiVersion: objectstorage.k8s.io/v1alpha1 -kind: BucketClass -metadata: - name: rustfs-bucketclass -driverName: rustfs.objectstorage.k8s.io -deletionPolicy: Delete -parameters: - objectStoreUserSecretName: provisioning-admin-creds - objectStoreUserSecretNamespace: default - endpoint: http://provisioning-demo-io.default.svc:9000 - region: us-east-1 diff --git a/src/reconcile/pool_lifecycle.rs b/src/reconcile/pool_lifecycle.rs index f0edb516..c723dafd 100644 --- a/src/reconcile/pool_lifecycle.rs +++ b/src/reconcile/pool_lifecycle.rs @@ -25,7 +25,7 @@ use super::{Error, context}; use crate::context::Context; use crate::sts::rustfs_client::{ RustfsAdminClient, RustfsClientError, RustfsPoolDecommissionInfo, RustfsPoolListItem, - RustfsPoolStatus, client_from_tenant, client_from_tls_tenant_for_sts, load_tenant_credentials, + RustfsPoolStatus, }; use crate::types::v1alpha1::pool::Pool; use crate::types::v1alpha1::pool_lifecycle::{DecommissionAction, DecommissionRequest}; @@ -453,11 +453,17 @@ async fn rustfs_admin_client( ctx: &Context, tenant: &Tenant, ) -> Result { - let credentials = load_tenant_credentials(&ctx.client, tenant).await?; + let credentials = RustfsAdminClient::load_tenant_credentials(&ctx.client, tenant).await?; if tenant.spec.tls.as_ref().is_some_and(|tls| tls.is_enabled()) { - client_from_tls_tenant_for_sts(&ctx.client, tenant, credentials, ctx.cluster_domain()).await + RustfsAdminClient::from_tls_tenant_for_sts( + &ctx.client, + tenant, + credentials, + ctx.cluster_domain(), + ) + .await } else { - client_from_tenant(tenant, credentials) + RustfsAdminClient::from_tenant(tenant, credentials) } } diff --git a/src/reconcile/provisioning.rs b/src/reconcile/provisioning.rs index f761061e..6fdf633f 100644 --- a/src/reconcile/provisioning.rs +++ b/src/reconcile/provisioning.rs @@ -13,10 +13,7 @@ // limitations under the License. use crate::context::{self, Context}; -use crate::sts::rustfs_client::{ - CreateBucketResult, RustfsAdminClient, RustfsClientError, client_from_tenant, - client_from_tls_tenant_for_sts, load_tenant_credentials, -}; +use crate::sts::rustfs_client::{CreateBucketResult, RustfsAdminClient, RustfsClientError}; use crate::types::v1alpha1::provisioning::{ ProvisioningBucket, ProvisioningPolicy, ProvisioningUser, duplicate_user_credentials_secret_names, @@ -499,11 +496,17 @@ async fn rustfs_admin_client( ctx: &Context, tenant: &Tenant, ) -> Result { - let credentials = load_tenant_credentials(&ctx.client, tenant).await?; + let credentials = RustfsAdminClient::load_tenant_credentials(&ctx.client, tenant).await?; if tenant.spec.tls.as_ref().is_some_and(|tls| tls.is_enabled()) { - client_from_tls_tenant_for_sts(&ctx.client, tenant, credentials, ctx.cluster_domain()).await + RustfsAdminClient::from_tls_tenant_for_sts( + &ctx.client, + tenant, + credentials, + ctx.cluster_domain(), + ) + .await } else { - client_from_tenant(tenant, credentials) + RustfsAdminClient::from_tenant(tenant, credentials) } } diff --git a/crates/rustfs-admin/src/admin_ops.rs b/src/sts/admin_ops.rs similarity index 79% rename from crates/rustfs-admin/src/admin_ops.rs rename to src/sts/admin_ops.rs index 837c6191..e80cd340 100644 --- a/crates/rustfs-admin/src/admin_ops.rs +++ b/src/sts/admin_ops.rs @@ -18,18 +18,44 @@ use std::collections::BTreeMap; -use reqwest::StatusCode; -use serde_json::Value; - -use crate::client::{ +use super::helpers::{ + body_mentions_not_found, build_canonical_query, extract_canned_policy_document, +}; +use super::{ ADD_CANNED_POLICY_PATH, ADD_USER_PATH, ADMIN_SIGNING_SERVICE, INFO_CANNED_POLICY_PATH, - JSON_CONTENT_TYPE, LIST_CANNED_POLICIES_PATH, REMOVE_CANNED_POLICY_PATH, REMOVE_USER_PATH, - RustfsAdminClient, RustfsClientError, RustfsServerInfo, RustfsServerInfoResponse, + JSON_CONTENT_TYPE, LIST_CANNED_POLICIES_PATH, REMOVE_USER_PATH, RustfsAdminClient, + RustfsClientError, RustfsServerInfo, RustfsServerInfoResponse, RustfsUserInfo, SERVER_INFO_PATH, SET_POLICY_PATH, USER_INFO_PATH, }; -use crate::helpers::{ - body_mentions_not_found, build_canonical_query, extract_canned_policy_document, -}; +use reqwest::StatusCode; +use serde_json::Value; + +fn parse_user_info_policy_names(body: &Value) -> Vec { + let Some(field) = body + .get("policyName") + .or_else(|| body.get("policy_name")) + .or_else(|| body.get("PolicyName")) + else { + return Vec::new(); + }; + + match field { + Value::String(raw) => raw + .split(',') + .map(str::trim) + .filter(|name| !name.is_empty()) + .map(ToOwned::to_owned) + .collect(), + Value::Array(items) => items + .iter() + .filter_map(|item| item.as_str()) + .map(str::trim) + .filter(|name| !name.is_empty()) + .map(ToOwned::to_owned) + .collect(), + _ => Vec::new(), + } +} impl RustfsAdminClient { // Admin duties: user and policy management APIs. @@ -122,45 +148,6 @@ impl RustfsAdminClient { Ok(()) } - /// Remove a RustFS canned policy. Succeeds if the policy is already gone. - pub async fn remove_canned_policy(&self, policy_name: &str) -> Result<(), RustfsClientError> { - if policy_name.trim().is_empty() { - return Err(RustfsClientError::InvalidPolicyName); - } - - let query = build_canonical_query(&[("name", policy_name)]); - let path = REMOVE_CANNED_POLICY_PATH; - let url = format!("{}{}?{query}", self.base_url.trim_end_matches('/'), path); - - let signed = self.sign_request("DELETE", path, &query, "", None, ADMIN_SIGNING_SERVICE)?; - let host = self.host()?; - - let response = self - .http_client - .delete(url) - .header("x-amz-date", &signed.amz_date) - .header("x-amz-content-sha256", &signed.payload_hash) - .header("authorization", &signed.authorization) - .header("host", host) - .send() - .await - .map_err(|_| RustfsClientError::RequestFailed)?; - - if response.status().is_success() { - return Ok(()); - } - - let status = response.status(); - let (body, truncated) = RustfsClientError::limited_response_body(response).await; - if status == StatusCode::NOT_FOUND || body_mentions_not_found(&body) { - return Ok(()); - } - - Err(RustfsClientError::unexpected_status_with_limited_body( - status, &body, truncated, - )) - } - pub async fn list_canned_policies( &self, ) -> Result, RustfsClientError> { @@ -196,7 +183,11 @@ impl RustfsAdminClient { .map_err(|_| RustfsClientError::ParseResponseFailed) } - pub async fn user_exists(&self, access_key: &str) -> Result { + /// Fetch IAM user info. Returns `Ok(None)` when the user does not exist. + pub async fn get_user_info( + &self, + access_key: &str, + ) -> Result, RustfsClientError> { if access_key.trim().is_empty() { return Err(RustfsClientError::InvalidCredentialValue { key: "accesskey" }); } @@ -219,13 +210,21 @@ impl RustfsAdminClient { .map_err(|_| RustfsClientError::RequestFailed)?; if response.status().is_success() { - return Ok(true); + let body = response + .text() + .await + .map_err(|_| RustfsClientError::RequestFailed)?; + let parsed: Value = + serde_json::from_str(&body).map_err(|_| RustfsClientError::ParseResponseFailed)?; + return Ok(Some(RustfsUserInfo { + policy_names: parse_user_info_policy_names(&parsed), + })); } let status = response.status(); let (body, truncated) = RustfsClientError::limited_response_body(response).await; if status == StatusCode::NOT_FOUND || body_mentions_not_found(&body) { - return Ok(false); + return Ok(None); } Err(RustfsClientError::unexpected_status_with_limited_body( @@ -233,6 +232,10 @@ impl RustfsAdminClient { )) } + pub async fn user_exists(&self, access_key: &str) -> Result { + Ok(self.get_user_info(access_key).await?.is_some()) + } + pub async fn add_user( &self, access_key: &str, @@ -257,7 +260,31 @@ impl RustfsAdminClient { .map(|_| ()) } - /// Remove a RustFS user. Succeeds if the user is already gone. + pub async fn set_user_policy( + &self, + access_key: &str, + policies: &[String], + ) -> Result<(), RustfsClientError> { + if access_key.trim().is_empty() { + return Err(RustfsClientError::InvalidCredentialValue { key: "accesskey" }); + } + if policies.is_empty() || policies.iter().any(|policy| policy.trim().is_empty()) { + return Err(RustfsClientError::InvalidPolicyName); + } + + let policy_names = policies.join(","); + let query = build_canonical_query(&[ + ("isGroup", "false"), + ("policyName", policy_names.as_str()), + ("userOrGroup", access_key), + ]); + + self.send_admin_request("PUT", SET_POLICY_PATH, &query, "", None) + .await + .map(|_| ()) + } + + /// Remove a RustFS user. Missing users are treated as success (idempotent). pub async fn remove_user(&self, access_key: &str) -> Result<(), RustfsClientError> { if access_key.trim().is_empty() { return Err(RustfsClientError::InvalidCredentialValue { key: "accesskey" }); @@ -266,7 +293,6 @@ impl RustfsAdminClient { let query = build_canonical_query(&[("accessKey", access_key)]); let path = REMOVE_USER_PATH; let url = format!("{}{}?{query}", self.base_url.trim_end_matches('/'), path); - let signed = self.sign_request("DELETE", path, &query, "", None, ADMIN_SIGNING_SERVICE)?; let host = self.host()?; @@ -295,28 +321,33 @@ impl RustfsAdminClient { status, &body, truncated, )) } +} - pub async fn set_user_policy( - &self, - access_key: &str, - policies: &[String], - ) -> Result<(), RustfsClientError> { - if access_key.trim().is_empty() { - return Err(RustfsClientError::InvalidCredentialValue { key: "accesskey" }); - } - if policies.is_empty() || policies.iter().any(|policy| policy.trim().is_empty()) { - return Err(RustfsClientError::InvalidPolicyName); - } +#[cfg(test)] +mod parse_tests { + use super::parse_user_info_policy_names; + use serde_json::json; + + #[test] + fn parses_comma_separated_policy_name() { + let body = json!({"policyName":"cosi-mlflow,cosi-grant-ba-1"}); + assert_eq!( + parse_user_info_policy_names(&body), + vec!["cosi-mlflow".to_string(), "cosi-grant-ba-1".to_string()] + ); + } - let policy_names = policies.join(","); - let query = build_canonical_query(&[ - ("isGroup", "false"), - ("policyName", policy_names.as_str()), - ("userOrGroup", access_key), - ]); + #[test] + fn parses_policy_name_array_and_snake_case() { + let body = json!({"policy_name":["a","b"]}); + assert_eq!( + parse_user_info_policy_names(&body), + vec!["a".to_string(), "b".to_string()] + ); + } - self.send_admin_request("PUT", SET_POLICY_PATH, &query, "", None) - .await - .map(|_| ()) + #[test] + fn missing_policy_field_yields_empty() { + assert!(parse_user_info_policy_names(&json!({"status":"enabled"})).is_empty()); } } diff --git a/crates/rustfs-admin/src/core_ops.rs b/src/sts/core_ops.rs similarity index 93% rename from crates/rustfs-admin/src/core_ops.rs rename to src/sts/core_ops.rs index d89f4052..ffdb9ae8 100644 --- a/crates/rustfs-admin/src/core_ops.rs +++ b/src/sts/core_ops.rs @@ -16,11 +16,11 @@ use chrono::Utc; use url::Url; -use crate::client::{ADMIN_SIGNING_SERVICE, RustfsAdminClient, RustfsClientError, SignedRequest}; -use crate::helpers::{derive_signing_key, hmac_sha256_hex, sha256_hex}; +use super::helpers::{derive_signing_key, hmac_sha256_hex, sha256_hex}; +use super::{ADMIN_SIGNING_SERVICE, RustfsAdminClient, RustfsClientError, SignedRequest}; impl RustfsAdminClient { - pub(crate) async fn send_admin_request( + pub(super) async fn send_admin_request( &self, method: &str, path: &str, @@ -49,7 +49,6 @@ impl RustfsAdminClient { "GET" => self.http_client.get(url), "POST" => self.http_client.post(url), "PUT" => self.http_client.put(url), - "DELETE" => self.http_client.delete(url), _ => return Err(RustfsClientError::RequestBuildFailed), } .header("x-amz-date", &signed.amz_date) @@ -83,7 +82,7 @@ impl RustfsAdminClient { .map_err(|_| RustfsClientError::RequestFailed) } - pub(crate) fn sign_request( + pub(super) fn sign_request( &self, method: &str, path: &str, @@ -105,7 +104,7 @@ impl RustfsAdminClient { ) } - pub(crate) fn sign_request_with_extra_headers( + pub(super) fn sign_request_with_extra_headers( &self, method: &str, path: &str, @@ -165,7 +164,7 @@ impl RustfsAdminClient { }) } - pub(crate) fn host(&self) -> Result { + pub(super) fn host(&self) -> Result { let parsed = Url::parse(&self.base_url).map_err(|_| RustfsClientError::RequestBuildFailed)?; let mut host = parsed diff --git a/src/sts/helpers.rs b/src/sts/helpers.rs index 8e772baa..b0975bde 100644 --- a/src/sts/helpers.rs +++ b/src/sts/helpers.rs @@ -12,15 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Internal helper duties: Tenant/kube credential and TLS status parsing. -//! Wire-protocol helpers (signing, hashing, response parsing) live in the -//! kube-agnostic `rustfs-admin` crate. +//! Internal helper duties: shared credential parsing, signature/hash utilities, and parsers. use std::collections::BTreeMap; +use hmac::{Hmac, Mac}; use k8s_openapi::ByteString; +use reqwest::StatusCode; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use url::form_urlencoded; use crate::Tenant; -use crate::sts::rustfs_client::{RustfsClientError, RustfsCredentials}; +use crate::sts::types::StsAssumeRoleCredentials; + +use super::{RustfsClientError, RustfsCredentials}; pub(super) fn extract_credentials( data: Option<&BTreeMap>, @@ -64,82 +69,158 @@ pub(super) fn get_secret_value( Ok(value) } -#[cfg(test)] -mod tests { - use k8s_openapi::{ByteString, api::core::v1 as corev1}; - use std::collections::BTreeMap; +/// Encode an `application/x-www-form-urlencoded` request body. +pub(super) fn build_form_body(params: &[(&str, &str)]) -> String { + let mut pairs: Vec<(String, String)> = params + .iter() + .map(|(k, v)| ((*k).to_string(), (*v).to_string())) + .collect(); + pairs.sort_by(|(k1, v1), (k2, v2)| k1.cmp(k2).then(v1.cmp(v2))); + + let mut serializer = form_urlencoded::Serializer::new(String::new()); + for (key, value) in pairs { + serializer.append_pair(&key, &value); + } - use super::extract_credentials; - use crate::sts::rustfs_client::RustfsClientError; + serializer.finish() +} - fn secret_with_fields(fields: Vec<(&str, &[u8])>) -> corev1::Secret { - let mut data = BTreeMap::new(); - for (key, value) in fields { - data.insert(key.to_string(), ByteString(value.to_vec())); - } +/// Encode and sort query parameters according to the AWS SigV4 rules. +pub(super) fn build_canonical_query(params: &[(&str, &str)]) -> String { + let mut pairs: Vec<(String, String)> = params + .iter() + .map(|(key, value)| (uri_encode(key), uri_encode(value))) + .collect(); + pairs.sort_unstable(); + + pairs + .into_iter() + .map(|(key, value)| format!("{key}={value}")) + .collect::>() + .join("&") +} - corev1::Secret { - data: Some(data), - ..Default::default() +fn uri_encode(value: &str) -> String { + const HEX: &[u8; 16] = b"0123456789ABCDEF"; + + let mut encoded = String::with_capacity(value.len()); + for byte in value.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + encoded.push(char::from(byte)); + } else { + encoded.push('%'); + encoded.push(char::from(HEX[usize::from(byte >> 4)])); + encoded.push(char::from(HEX[usize::from(byte & 0x0f)])); } } + encoded +} - #[test] - fn extract_credentials_reports_missing_access_key() { - let secret = secret_with_fields(vec![("secretkey", b"sekret")]); +pub(super) fn create_bucket_body(region: Option<&str>) -> String { + let Some(region) = region.map(str::trim).filter(|region| !region.is_empty()) else { + return String::new(); + }; - let err = - extract_credentials(secret.data.as_ref()).expect_err("expected missing access key"); - assert!(matches!( - err, - RustfsClientError::MissingCredentialKey { key: "accesskey" } - )); + if region == "us-east-1" { + return String::new(); } - #[test] - fn extract_credentials_reports_non_utf8_access_key() { - let secret = - secret_with_fields(vec![("accesskey", &[0xff, 0xfe]), ("secretkey", b"sekret")]); + format!( + "{}", + escape_xml(region) + ) +} - let err = extract_credentials(secret.data.as_ref()).expect_err("expected invalid utf8"); - assert!(matches!( - err, - RustfsClientError::InvalidCredentialValue { key: "accesskey" } - )); - } +pub(super) fn escape_xml(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} - #[test] - fn extract_credentials_reports_missing_secret_key() { - let secret = secret_with_fields(vec![("accesskey", b"access")]); +pub(super) fn body_mentions_not_found(body: &str) -> bool { + let body = body.to_ascii_lowercase(); + body.contains("nosuchuser") + || body.contains("no such user") + || body.contains("user not exist") + || body.contains("nosuchpolicy") + || body.contains("no such policy") + || body.contains("objectlockconfigurationnotfound") + || body.contains("not found") +} - let err = - extract_credentials(secret.data.as_ref()).expect_err("expected missing secret key"); - assert!(matches!( - err, - RustfsClientError::MissingCredentialKey { key: "secretkey" } - )); +pub(super) fn bucket_already_exists(status: StatusCode, body: &str) -> bool { + if status == StatusCode::CONFLICT { + let body = body.to_ascii_lowercase(); + return body.contains("bucketalreadyexists") || body.contains("bucketalreadyownedbyyou"); } - #[test] - fn extract_credentials_reports_non_utf8_secret_key() { - let secret = - secret_with_fields(vec![("accesskey", b"access"), ("secretkey", &[0xff, 0xfe])]); + false +} - let err = extract_credentials(secret.data.as_ref()).expect_err("expected invalid utf8"); - assert!(matches!( - err, - RustfsClientError::InvalidCredentialValue { key: "secretkey" } - )); - } +pub(super) fn extract_canned_policy_document(body: &str) -> Result { + let value = serde_json::from_str::(body) + .map_err(|_| RustfsClientError::InvalidPolicyDocument)?; + let policy = value.get("policy").unwrap_or(&value); - #[test] - fn extract_credentials_reports_empty_secret_key() { - let secret = secret_with_fields(vec![("accesskey", b"abc"), ("secretkey", b"")]); + serde_json::to_string(policy).map_err(|_| RustfsClientError::InvalidPolicyDocument) +} - let err = extract_credentials(secret.data.as_ref()).expect_err("expected empty secret key"); - assert!(matches!( - err, - RustfsClientError::EmptyCredentialValue { key: "secretkey" } - )); - } +pub(super) fn sha256_hex(payload: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(payload); + hex::encode(hasher.finalize()) +} + +pub(super) fn hmac_sha256(key: &[u8], message: &str) -> Result, RustfsClientError> { + let mut mac = + Hmac::::new_from_slice(key).map_err(|_| RustfsClientError::SigningFailed)?; + mac.update(message.as_bytes()); + Ok(mac.finalize().into_bytes().to_vec()) +} + +pub(super) fn hmac_sha256_hex(key: &[u8], message: &str) -> Result { + let bytes = hmac_sha256(key, message)?; + Ok(hex::encode(bytes)) +} + +pub(super) fn derive_signing_key( + secret_key: &str, + date_stamp: &str, + region: &str, + service: &str, +) -> Result, RustfsClientError> { + let k_secret = format!("AWS4{secret_key}").into_bytes(); + let k_date = hmac_sha256(&k_secret, date_stamp)?; + let k_region = hmac_sha256(&k_date, region)?; + let k_service = hmac_sha256(&k_region, service)?; + hmac_sha256(&k_service, "aws4_request") +} + +pub(super) fn parse_assume_role_response(body: &str) -> Option { + let access_key_id = extract_xml_tag(body, "AccessKeyId")?; + let secret_access_key = extract_xml_tag(body, "SecretAccessKey")?; + let session_token = extract_xml_tag(body, "SessionToken")?; + let expiration = extract_xml_tag(body, "Expiration")?; + + Some(StsAssumeRoleCredentials { + access_key_id, + secret_access_key, + session_token, + expiration, + }) +} + +pub(super) fn extract_xml_tag(document: &str, tag: &str) -> Option { + let open = format!("<{tag}>"); + let close = format!(""); + + let open_idx = document.find(&open)?; + let start = open_idx + open.len(); + let rest = &document[start..]; + let end = rest.find(&close)?; + + Some(rest[..end].trim().to_string()) } diff --git a/crates/rustfs-admin/src/pool_ops.rs b/src/sts/pool_ops.rs similarity index 97% rename from crates/rustfs-admin/src/pool_ops.rs rename to src/sts/pool_ops.rs index c5e84452..a5322ee9 100644 --- a/crates/rustfs-admin/src/pool_ops.rs +++ b/src/sts/pool_ops.rs @@ -15,11 +15,11 @@ //! Pool boundary: //! - list/status and decommission lifecycle operations for tenant pools. -use crate::client::{ +use super::helpers::build_canonical_query; +use super::{ POOLS_CANCEL_PATH, POOLS_DECOMMISSION_PATH, POOLS_LIST_PATH, POOLS_STATUS_PATH, RustfsAdminClient, RustfsClientError, RustfsPoolListItem, RustfsPoolStatus, }; -use crate::helpers::build_canonical_query; impl RustfsAdminClient { // Pool duties: list/status and decommission lifecycle operations. diff --git a/src/sts/rustfs_client.rs b/src/sts/rustfs_client.rs index c1507c82..9b513789 100644 --- a/src/sts/rustfs_client.rs +++ b/src/sts/rustfs_client.rs @@ -12,33 +12,427 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Kube/Tenant wrappers around the kube-agnostic RustFS admin/S3/STS client. -//! -//! The wire-protocol client implementation (request signing, HTTP dispatch, -//! response parsing) lives in the `rustfs-admin` crate and is re-exported -//! here. This module only adds the Tenant/kube-specific constructors that -//! need access to `kube::Client` and the `Tenant` CRD type. +use std::{collections::BTreeMap, time::Duration}; use k8s_openapi::api::core::v1 as corev1; use kube::{Api, Client}; +use reqwest::{Certificate, Client as HttpClient, Response, StatusCode}; use crate::Tenant; use crate::cluster_dns; +use crate::utils::sanitize::redact_sensitive_pairs; -/// helpers: Tenant/kube credential and TLS status parsing. +/// admin_ops: tenant admin operations (user/policy APIs). +#[path = "admin_ops.rs"] +mod admin_ops; +/// core_ops: shared request signing/dispatch internals. +#[path = "core_ops.rs"] +mod core_ops; +/// helpers: credential parsing, signing/hash utilities and parsers. #[path = "helpers.rs"] mod helpers; +/// pool_ops: pool lifecycle and status operations. +#[path = "pool_ops.rs"] +mod pool_ops; +/// s3_ops: bucket/object-lock operations for S3-compatible endpoints. +#[path = "s3_ops.rs"] +mod s3_ops; +/// sts_ops: temporary credential flows, AssumeRole request/response. +#[path = "sts_ops.rs"] +mod sts_ops; -pub use rustfs_admin::{ - CreateBucketResult, RustfsAdminClient, RustfsClientError, RustfsCredentials, - RustfsErasureBackend, RustfsErasureSetInfo, RustfsPoolDecommissionInfo, RustfsPoolListItem, - RustfsPoolStatus, RustfsServerInfo, RustfsServerUsage, StsAssumeRoleCredentials, -}; +const FORM_CONTENT_TYPE: &str = "application/x-www-form-urlencoded"; +const JSON_CONTENT_TYPE: &str = "application/json"; +const ASSUME_ROLE_PATH: &str = "/"; +const ADD_USER_PATH: &str = "/rustfs/admin/v3/add-user"; +const REMOVE_USER_PATH: &str = "/rustfs/admin/v3/remove-user"; +const USER_INFO_PATH: &str = "/rustfs/admin/v3/user-info"; +const SET_POLICY_PATH: &str = "/rustfs/admin/v3/set-policy"; +const LIST_CANNED_POLICIES_PATH: &str = "/rustfs/admin/v3/list-canned-policies"; +const ADD_CANNED_POLICY_PATH: &str = "/rustfs/admin/v3/add-canned-policy"; +const INFO_CANNED_POLICY_PATH: &str = "/rustfs/admin/v3/info-canned-policy"; +const SERVER_INFO_PATH: &str = "/rustfs/admin/v3/info"; +const POOLS_LIST_PATH: &str = "/rustfs/admin/v3/pools/list"; +const POOLS_STATUS_PATH: &str = "/rustfs/admin/v3/pools/status"; +const POOLS_DECOMMISSION_PATH: &str = "/rustfs/admin/v3/pools/decommission"; +const POOLS_CANCEL_PATH: &str = "/rustfs/admin/v3/pools/cancel"; +const ADMIN_SIGNING_SERVICE: &str = "s3"; +const STS_SIGNING_SERVICE: &str = "sts"; +const ADMIN_HTTP_CONNECT_TIMEOUT: Duration = Duration::from_secs(3); +const ADMIN_HTTP_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); +const MAX_UPSTREAM_ERROR_BODY_BYTES: usize = 8 * 1024; +const MAX_UPSTREAM_ERROR_DETAIL_CHARS: usize = 512; -pub(super) fn tls_tenant_base_url( - tenant: &Tenant, - cluster_domain: &str, -) -> Result { +/// Credentials read from Tenant `.spec.credsSecret`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RustfsCredentials { + pub access_key: String, + pub secret_key: String, +} + +#[derive(Debug, Clone, serde::Deserialize, PartialEq)] +pub struct RustfsPoolListItem { + pub id: usize, + #[serde(rename = "cmdline")] + pub cmd_line: String, + #[serde(rename = "lastUpdate")] + pub last_update: String, + #[serde(rename = "totalSize")] + pub total_size: Option, + #[serde(rename = "currentSize")] + pub current_size: Option, + #[serde(rename = "usedSize")] + pub used_size: Option, + pub used: Option, + pub status: String, + #[serde(rename = "decommissionInfo")] + pub decommission: Option, +} + +#[derive(Debug, Clone, serde::Deserialize, PartialEq)] +pub struct RustfsPoolStatus { + pub id: usize, + #[serde(rename = "cmdline")] + pub cmd_line: String, + #[serde(rename = "lastUpdate")] + pub last_update: String, + #[serde(rename = "decommissionInfo")] + pub decommission: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CreateBucketResult { + Created, + AlreadyExists, +} + +/// Subset of `/rustfs/admin/v3/user-info` used by COSI grant ownership checks. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct RustfsUserInfo { + pub policy_names: Vec, +} + +#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] +pub struct RustfsPoolDecommissionInfo { + #[serde(rename = "startTime")] + pub start_time: Option, + #[serde(rename = "startSize")] + pub start_size: Option, + #[serde(rename = "totalSize")] + pub total_size: Option, + #[serde(rename = "currentSize")] + pub current_size: Option, + pub complete: Option, + pub failed: Option, + pub canceled: Option, + #[serde(rename = "objectsDecommissioned")] + pub objects_decommissioned: Option, + #[serde(rename = "objectsDecommissionedFailed")] + pub objects_decommissioned_failed: Option, + #[serde(rename = "bytesDecommissioned")] + pub bytes_decommissioned: Option, + #[serde(rename = "bytesDecommissionedFailed")] + pub bytes_decommissioned_failed: Option, +} + +#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] +pub struct RustfsServerInfo { + #[serde(default)] + pub usage: Option, + #[serde(default)] + pub backend: Option, + #[serde(default)] + pub pools: Option>>, +} + +#[derive(Debug, Clone, serde::Deserialize, PartialEq)] +pub(super) struct RustfsServerInfoResponse { + pub info: RustfsServerInfo, +} + +#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] +pub struct RustfsServerUsage { + #[serde(default)] + pub size: u64, +} + +#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] +pub struct RustfsErasureBackend { + #[serde(default, rename = "onlineDisks")] + pub online_disks: u64, + #[serde(default, rename = "offlineDisks")] + pub offline_disks: u64, + #[serde(default, rename = "standardSCParity", alias = "StandardSCParity")] + pub standard_sc_parity: Option, + #[serde(default, rename = "totalSets")] + pub total_sets: Vec, + #[serde(default, rename = "totalDrivesPerSet", alias = "drivesPerSet")] + pub drives_per_set: Vec, +} + +#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] +pub struct RustfsErasureSetInfo { + #[serde(default, rename = "rawUsage")] + pub raw_usage: u64, + #[serde(default, rename = "rawCapacity")] + pub raw_capacity: u64, + #[serde(default)] + pub usage: u64, + #[serde(default, rename = "objectsCount")] + pub objects_count: u64, + #[serde(default, rename = "healDisks")] + pub heal_disks: u64, +} + +/// Error type for RustFS admin/STS client operations. +#[derive(Debug)] +pub enum RustfsClientError { + MissingTenantNamespace, + MissingCredsSecret, + MissingCredentialKey { + key: &'static str, + }, + EmptyCredentialValue { + key: &'static str, + }, + InvalidCredentialValue { + key: &'static str, + }, + TenantSecretLookupFailed, + InvalidPolicyName, + InvalidPolicyDocument, + TenantTlsRequired, + TenantTlsNotReady, + TenantTlsClientCertificateRequired, + MissingTenantTlsCaKey { + secret: String, + key: String, + }, + TenantTlsCaSecretLookupFailed { + secret: String, + }, + InvalidTenantTlsCa, + TlsClientBuildFailed, + RequestBuildFailed, + RequestFailed, + UnexpectedStatus { + status: StatusCode, + detail: Option, + }, + ParseResponseFailed, + SigningFailed, +} + +impl std::fmt::Display for RustfsClientError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::MissingTenantNamespace => write!(f, "tenant namespace is missing"), + Self::MissingCredsSecret => write!(f, "tenant credsSecret is missing"), + Self::MissingCredentialKey { key } => write!(f, "secret key missing: {key}"), + Self::EmptyCredentialValue { key } => write!(f, "secret key empty: {key}"), + Self::InvalidCredentialValue { key } => { + write!(f, "secret key is not valid utf8: {key}") + } + Self::TenantSecretLookupFailed => { + write!(f, "failed to load tenant credential secret") + } + Self::InvalidPolicyName => write!(f, "invalid policy name"), + Self::InvalidPolicyDocument => write!(f, "failed to parse canned policy response"), + Self::TenantTlsRequired => write!(f, "STS requires a TLS-enabled tenant"), + Self::TenantTlsNotReady => write!(f, "tenant TLS status is not ready"), + Self::TenantTlsClientCertificateRequired => { + write!(f, "tenant TLS requires a client certificate") + } + Self::MissingTenantTlsCaKey { secret, key } => { + write!(f, "tenant TLS CA secret {secret} missing key {key}") + } + Self::TenantTlsCaSecretLookupFailed { secret } => { + write!(f, "failed to load tenant TLS CA secret {secret}") + } + Self::InvalidTenantTlsCa => write!(f, "tenant TLS CA is not a valid PEM bundle"), + Self::TlsClientBuildFailed => write!(f, "failed to build TLS HTTP client"), + Self::RequestBuildFailed => write!(f, "failed to construct request"), + Self::RequestFailed => write!(f, "request failed"), + Self::UnexpectedStatus { status, detail } => { + write!(f, "upstream returned {status}")?; + if let Some(detail) = detail { + write!(f, ": {detail}")?; + } + Ok(()) + } + Self::ParseResponseFailed => write!(f, "failed to parse AssumeRole response"), + Self::SigningFailed => write!(f, "failed to compute request signature"), + } + } +} + +impl std::error::Error for RustfsClientError {} + +impl RustfsClientError { + pub(super) async fn unexpected_response(response: Response) -> Self { + let status = response.status(); + let (body, truncated) = read_limited_response_body(response).await; + Self::unexpected_status_with_limited_body(status, &body, truncated) + } + + pub(super) async fn limited_response_body(response: Response) -> (String, bool) { + read_limited_response_body(response).await + } + + fn unexpected_status_with_limited_body( + status: StatusCode, + body: &str, + body_truncated: bool, + ) -> Self { + Self::UnexpectedStatus { + status, + detail: summarize_upstream_error_body(body, body_truncated), + } + } + + #[cfg(test)] + pub(super) fn unexpected_status_with_body(status: StatusCode, body: &str) -> Self { + Self::unexpected_status_with_limited_body(status, body, false) + } +} + +async fn read_limited_response_body(mut response: Response) -> (String, bool) { + let mut body = Vec::new(); + let read_limit = MAX_UPSTREAM_ERROR_BODY_BYTES.saturating_add(1); + + loop { + let remaining = read_limit.saturating_sub(body.len()); + if remaining == 0 { + break; + } + + let chunk = match response.chunk().await { + Ok(Some(chunk)) => chunk, + Ok(None) => break, + Err(_) => break, + }; + if chunk.len() > remaining { + body.extend_from_slice(&chunk[..remaining]); + break; + } + body.extend_from_slice(&chunk); + } + + let truncated = body.len() > MAX_UPSTREAM_ERROR_BODY_BYTES; + if truncated { + body.truncate(MAX_UPSTREAM_ERROR_BODY_BYTES); + } + + (String::from_utf8_lossy(&body).into_owned(), truncated) +} + +fn summarize_upstream_error_body(body: &str, body_truncated: bool) -> Option { + let body = body.trim(); + if body.is_empty() { + return None; + } + + if let Some(message) = helpers::extract_xml_tag(body, "Message") { + let message = decode_basic_xml_entities(&message); + let detail = match helpers::extract_xml_tag(body, "Code") { + Some(code) if !code.trim().is_empty() => { + format!("{}: {message}", decode_basic_xml_entities(&code)) + } + _ => message, + }; + return Some(sanitize_error_detail(&detail)); + } + + if let Ok(value) = serde_json::from_str::(body) + && let Some(detail) = summarize_json_error(&value) + { + return Some(sanitize_error_detail(&detail)); + } + + if body_truncated { + return Some(format!( + "response body exceeded {MAX_UPSTREAM_ERROR_BODY_BYTES} bytes" + )); + } + + Some(sanitize_error_detail(body)) +} + +fn summarize_json_error(value: &serde_json::Value) -> Option { + if let Some(message) = value.as_str() { + return Some(message.to_string()); + } + + let object = value.as_object()?; + let message = ["message", "Message", "error", "Error"] + .iter() + .find_map(|key| object.get(*key).and_then(serde_json::Value::as_str))?; + let code = ["code", "Code"] + .iter() + .find_map(|key| object.get(*key).and_then(serde_json::Value::as_str)); + + Some(match code { + Some(code) if !code.trim().is_empty() => format!("{code}: {message}"), + _ => message.to_string(), + }) +} + +fn collapse_whitespace(value: &str) -> String { + value.split_whitespace().collect::>().join(" ") +} + +fn sanitize_error_detail(value: &str) -> String { + let detail = collapse_whitespace(value); + let detail = redact_sensitive_pairs(&detail); + truncate_error_detail(detail) +} + +fn truncate_error_detail(value: String) -> String { + let mut truncated = String::new(); + for (index, ch) in value.chars().enumerate() { + if index >= MAX_UPSTREAM_ERROR_DETAIL_CHARS { + truncated.push_str("..."); + return truncated; + } + truncated.push(ch); + } + truncated +} + +fn decode_basic_xml_entities(value: &str) -> String { + value + .replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'") + .replace("&", "&") +} + +#[derive(Debug)] +struct SignedRequest { + amz_date: String, + payload_hash: String, + authorization: String, +} + +/// RustFS admin/STS client. +pub struct RustfsAdminClient { + base_url: String, + access_key: String, + secret_key: String, + region: String, + http_client: HttpClient, +} + +fn default_http_client() -> HttpClient { + HttpClient::builder() + .connect_timeout(ADMIN_HTTP_CONNECT_TIMEOUT) + .timeout(ADMIN_HTTP_REQUEST_TIMEOUT) + .build() + .unwrap_or_else(|_| HttpClient::new()) +} + +fn tls_tenant_base_url(tenant: &Tenant, cluster_domain: &str) -> Result { let namespace = tenant .namespace() .map_err(|_| RustfsClientError::MissingTenantNamespace)?; @@ -47,124 +441,179 @@ pub(super) fn tls_tenant_base_url( Ok(format!("https://{service_fqdn}:9000")) } -/// Build a RustFS admin client using the tenant's in-cluster (plain HTTP) service address. -pub fn client_from_tenant( - tenant: &Tenant, - credentials: RustfsCredentials, -) -> Result { - let namespace = tenant - .namespace() - .map_err(|_| RustfsClientError::MissingTenantNamespace)?; - let service_name = tenant - .new_io_service() - .metadata - .name - .unwrap_or_else(|| format!("{}-io", tenant.name())); - - Ok(RustfsAdminClient::new_with_base_url( - format!("http://{service_name}.{namespace}.svc:9000"), - credentials.access_key, - credentials.secret_key, - )) -} - -/// Build a RustFS admin client against the tenant's TLS-enabled headless service, -/// trusting the tenant's CA if one is published. Requires TLS to be enabled. -pub async fn client_from_tls_tenant_for_sts( - kube_client: &Client, - tenant: &Tenant, - credentials: RustfsCredentials, - cluster_domain: &str, -) -> Result { - if !helpers::tenant_tls_enabled(tenant) { - return Err(RustfsClientError::TenantTlsRequired); - } - if helpers::tenant_tls_client_certificate_required(tenant) { - return Err(RustfsClientError::TenantTlsClientCertificateRequired); - } - - let base_url = tls_tenant_base_url(tenant, cluster_domain)?; - - match load_tenant_tls_ca(kube_client, tenant).await? { - Some(ca_pem) => RustfsAdminClient::new_with_base_url_and_ca_pem( +impl RustfsAdminClient { + pub const STS_VERSION: &'static str = "2011-06-15"; + pub const STS_ACTION: &'static str = "AssumeRole"; + + pub fn new_with_base_url( + base_url: impl Into, + access_key: impl Into, + secret_key: impl Into, + ) -> Self { + Self::new_with_base_url_and_http_client( base_url, - credentials.access_key, - credentials.secret_key, - &ca_pem, - ), - None => Ok(RustfsAdminClient::new_with_base_url( + access_key, + secret_key, + default_http_client(), + ) + } + + pub fn new_with_base_url_and_ca_pem( + base_url: impl Into, + access_key: impl Into, + secret_key: impl Into, + ca_pem: &[u8], + ) -> Result { + let certs = Certificate::from_pem_bundle(ca_pem) + .map_err(|_| RustfsClientError::InvalidTenantTlsCa)?; + let mut builder = HttpClient::builder() + .connect_timeout(ADMIN_HTTP_CONNECT_TIMEOUT) + .timeout(ADMIN_HTTP_REQUEST_TIMEOUT); + for cert in certs { + builder = builder.add_root_certificate(cert); + } + let http_client = builder + .build() + .map_err(|_| RustfsClientError::TlsClientBuildFailed)?; + + Ok(Self::new_with_base_url_and_http_client( base_url, + access_key, + secret_key, + http_client, + )) + } + + pub fn new_with_base_url_and_http_client( + base_url: impl Into, + access_key: impl Into, + secret_key: impl Into, + http_client: HttpClient, + ) -> Self { + Self { + base_url: base_url.into(), + access_key: access_key.into(), + secret_key: secret_key.into(), + region: "us-east-1".to_string(), + http_client, + } + } + + pub fn from_tenant( + tenant: &Tenant, + credentials: RustfsCredentials, + ) -> Result { + let namespace = tenant + .namespace() + .map_err(|_| RustfsClientError::MissingTenantNamespace)?; + let service_name = tenant + .new_io_service() + .metadata + .name + .unwrap_or_else(|| format!("{}-io", tenant.name())); + + Ok(Self::new_with_base_url( + format!("http://{service_name}.{namespace}.svc:9000"), credentials.access_key, credentials.secret_key, - )), + )) } -} -/// Load the tenant's TLS CA bundle, if the tenant publishes one. -pub async fn load_tenant_tls_ca( - kube_client: &Client, - tenant: &Tenant, -) -> Result>, RustfsClientError> { - if !helpers::tenant_tls_enabled(tenant) { - return Ok(None); - } + pub async fn from_tls_tenant_for_sts( + kube_client: &Client, + tenant: &Tenant, + credentials: RustfsCredentials, + cluster_domain: &str, + ) -> Result { + if !helpers::tenant_tls_enabled(tenant) { + return Err(RustfsClientError::TenantTlsRequired); + } + if helpers::tenant_tls_client_certificate_required(tenant) { + return Err(RustfsClientError::TenantTlsClientCertificateRequired); + } - let tls_status = tenant - .status - .as_ref() - .and_then(|status| status.certificates.tls.as_ref()) - .filter(|tls| tls.ready) - .ok_or(RustfsClientError::TenantTlsNotReady)?; + let base_url = tls_tenant_base_url(tenant, cluster_domain)?; - let Some(ca_ref) = tls_status.ca_secret_ref.as_ref() else { - return Ok(None); - }; + match Self::load_tenant_tls_ca(kube_client, tenant).await? { + Some(ca_pem) => Self::new_with_base_url_and_ca_pem( + base_url, + credentials.access_key, + credentials.secret_key, + &ca_pem, + ), + None => Ok(Self::new_with_base_url( + base_url, + credentials.access_key, + credentials.secret_key, + )), + } + } - let namespace = tenant - .namespace() - .map_err(|_| RustfsClientError::MissingTenantNamespace)?; - let api: Api = Api::namespaced(kube_client.clone(), &namespace); - let secret = api.get(&ca_ref.name).await.map_err(|_| { - RustfsClientError::TenantTlsCaSecretLookupFailed { - secret: ca_ref.name.clone(), + pub async fn load_tenant_tls_ca( + kube_client: &Client, + tenant: &Tenant, + ) -> Result>, RustfsClientError> { + if !helpers::tenant_tls_enabled(tenant) { + return Ok(None); } - })?; - let key = ca_ref.key.as_deref().unwrap_or("ca.crt"); - let ca_pem = secret - .data - .as_ref() - .and_then(|data| data.get(key)) - .map(|bytes| bytes.0.clone()) - .filter(|bytes| !bytes.is_empty()) - .ok_or_else(|| RustfsClientError::MissingTenantTlsCaKey { - secret: ca_ref.name.clone(), - key: key.to_string(), + + let tls_status = tenant + .status + .as_ref() + .and_then(|status| status.certificates.tls.as_ref()) + .filter(|tls| tls.ready) + .ok_or(RustfsClientError::TenantTlsNotReady)?; + + let Some(ca_ref) = tls_status.ca_secret_ref.as_ref() else { + return Ok(None); + }; + + let namespace = tenant + .namespace() + .map_err(|_| RustfsClientError::MissingTenantNamespace)?; + let api: Api = Api::namespaced(kube_client.clone(), &namespace); + let secret = api.get(&ca_ref.name).await.map_err(|_| { + RustfsClientError::TenantTlsCaSecretLookupFailed { + secret: ca_ref.name.clone(), + } })?; + let key = ca_ref.key.as_deref().unwrap_or("ca.crt"); + let ca_pem = secret + .data + .as_ref() + .and_then(|data| data.get(key)) + .map(|bytes| bytes.0.clone()) + .filter(|bytes| !bytes.is_empty()) + .ok_or_else(|| RustfsClientError::MissingTenantTlsCaKey { + secret: ca_ref.name.clone(), + key: key.to_string(), + })?; - Ok(Some(ca_pem)) -} + Ok(Some(ca_pem)) + } -/// Read the Tenant credential Secret and return an access/secret key pair. -pub async fn load_tenant_credentials( - kube_client: &Client, - tenant: &Tenant, -) -> Result { - let reference = tenant - .spec - .creds_secret - .as_ref() - .ok_or(RustfsClientError::MissingCredsSecret)?; + /// Read Tenant credential Secret and return access/secret key pair. + pub async fn load_tenant_credentials( + kube_client: &Client, + tenant: &Tenant, + ) -> Result { + let reference = tenant + .spec + .creds_secret + .as_ref() + .ok_or(RustfsClientError::MissingCredsSecret)?; - let namespace = tenant - .namespace() - .map_err(|_| RustfsClientError::MissingTenantNamespace)?; - let api: Api = Api::namespaced(kube_client.clone(), &namespace); - let secret = api - .get(&reference.name) - .await - .map_err(|_| RustfsClientError::TenantSecretLookupFailed)?; + let namespace = tenant + .namespace() + .map_err(|_| RustfsClientError::MissingTenantNamespace)?; + let api: Api = Api::namespaced(kube_client.clone(), &namespace); + let secret = api + .get(&reference.name) + .await + .map_err(|_| RustfsClientError::TenantSecretLookupFailed)?; - helpers::extract_credentials(secret.data.as_ref()) + helpers::extract_credentials(secret.data.as_ref()) + } } #[cfg(test)] diff --git a/crates/rustfs-admin/src/s3_ops.rs b/src/sts/s3_ops.rs similarity index 92% rename from crates/rustfs-admin/src/s3_ops.rs rename to src/sts/s3_ops.rs index 6fca0a02..023dc1a0 100644 --- a/crates/rustfs-admin/src/s3_ops.rs +++ b/src/sts/s3_ops.rs @@ -13,18 +13,15 @@ // limitations under the License. //! S3 boundary: -//! - bucket lifecycle methods (create/lookup/delete) +//! - bucket lifecycle methods (create/lookup features) //! - request semantics for S3-style object storage operations. use reqwest::StatusCode; -use crate::client::{ - ADMIN_SIGNING_SERVICE, CreateBucketResult, RustfsAdminClient, RustfsClientError, -}; -use crate::helpers::{ - body_mentions_not_found, bucket_already_exists, bucket_not_found, build_canonical_query, - create_bucket_body, +use super::helpers::{ + body_mentions_not_found, bucket_already_exists, build_canonical_query, create_bucket_body, }; +use super::{ADMIN_SIGNING_SERVICE, CreateBucketResult, RustfsAdminClient, RustfsClientError}; impl RustfsAdminClient { // S3 duties: bucket operations exposed by the RustFS/S3-compatible endpoint. @@ -94,7 +91,7 @@ impl RustfsAdminClient { )) } - /// Delete a bucket. Succeeds if the bucket is already gone. + /// Delete a bucket. Missing buckets are treated as success (idempotent). pub async fn delete_bucket(&self, bucket: &str) -> Result<(), RustfsClientError> { if bucket.trim().is_empty() { return Err(RustfsClientError::RequestBuildFailed); @@ -121,7 +118,7 @@ impl RustfsAdminClient { let status = response.status(); let (body, truncated) = RustfsClientError::limited_response_body(response).await; - if status == StatusCode::NOT_FOUND || bucket_not_found(&body) { + if status == StatusCode::NOT_FOUND || body_mentions_not_found(&body) { return Ok(()); } diff --git a/src/sts/server.rs b/src/sts/server.rs index fd04ec63..7914ca71 100644 --- a/src/sts/server.rs +++ b/src/sts/server.rs @@ -32,9 +32,7 @@ use crate::http_admission::{ use crate::metrics::UnauthenticatedRequestOutcome; use crate::sts::binding; use crate::sts::error::{StsError, StsErrorType, render_sts_error_xml_with_type}; -use crate::sts::rustfs_client::{ - RustfsAdminClient, RustfsClientError, client_from_tls_tenant_for_sts, load_tenant_credentials, -}; +use crate::sts::rustfs_client::{RustfsAdminClient, RustfsClientError}; use crate::sts::session_policy; use crate::sts::token_review::{self, TokenReviewError}; use crate::sts::types::{ @@ -644,11 +642,11 @@ async fn create_rustfs_admin_client( }); } - let credentials = load_tenant_credentials(client, tenant) + let credentials = RustfsAdminClient::load_tenant_credentials(client, tenant) .await .map_err(|_| StsError::InternalError)?; - client_from_tls_tenant_for_sts(client, tenant, credentials, cluster_domain) + RustfsAdminClient::from_tls_tenant_for_sts(client, tenant, credentials, cluster_domain) .await .map_err(map_rustfs_client_creation_error) } diff --git a/crates/rustfs-admin/src/sts_ops.rs b/src/sts/sts_ops.rs similarity index 95% rename from crates/rustfs-admin/src/sts_ops.rs rename to src/sts/sts_ops.rs index 2eff08c4..ba45bccb 100644 --- a/crates/rustfs-admin/src/sts_ops.rs +++ b/src/sts/sts_ops.rs @@ -14,12 +14,11 @@ //! STS boundary: //! - temporary credentials and AssumeRole request composition/response parsing. - -use crate::client::{ +use super::helpers::{build_form_body, parse_assume_role_response}; +use super::{ ASSUME_ROLE_PATH, FORM_CONTENT_TYPE, RustfsAdminClient, RustfsClientError, STS_SIGNING_SERVICE, }; -use crate::credentials::StsAssumeRoleCredentials; -use crate::helpers::{build_form_body, parse_assume_role_response}; +use crate::sts::types::StsAssumeRoleCredentials; impl RustfsAdminClient { // STS duties: temporary credentials and AssumeRole API call path. diff --git a/src/sts/tests.rs b/src/sts/tests.rs index 52064a0e..bea76c42 100644 --- a/src/sts/tests.rs +++ b/src/sts/tests.rs @@ -12,11 +12,112 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Tests for the Tenant/kube-specific wrappers around `RustfsAdminClient`. -//! Wire-protocol tests (signing, hashing, response parsing) live in the -//! `rustfs-admin` crate. +//! Unit/integration tests for RustfsAdminClient split operation modules. -use super::tls_tenant_base_url; +use axum::{ + Router, + body::Body, + extract::State, + http::{Request, StatusCode}, + routing::{get, post, put}, +}; +use k8s_openapi::{ByteString, api::core::v1 as corev1}; +use serde_json::Value; +use std::{collections::BTreeMap, sync::Arc}; +use tokio::sync::Mutex; + +use super::{ + ADD_USER_PATH, ADMIN_SIGNING_SERVICE, CreateBucketResult, FORM_CONTENT_TYPE, JSON_CONTENT_TYPE, + LIST_CANNED_POLICIES_PATH, MAX_UPSTREAM_ERROR_BODY_BYTES, POOLS_DECOMMISSION_PATH, + POOLS_LIST_PATH, POOLS_STATUS_PATH, RustfsAdminClient, RustfsClientError, SERVER_INFO_PATH, + SET_POLICY_PATH, STS_SIGNING_SERVICE, USER_INFO_PATH, + helpers::{ + build_canonical_query, build_form_body, derive_signing_key, extract_canned_policy_document, + extract_credentials, hmac_sha256_hex, parse_assume_role_response, sha256_hex, + }, + tls_tenant_base_url, +}; + +const TEST_ACCESS_KEY: &str = "access"; +const TEST_SECRET_KEY: &str = "secret"; +const TEST_REGION: &str = "us-east-1"; + +#[test] +fn canonical_query_uses_sigv4_uri_encoding_and_encoded_sort_order() { + let query = build_canonical_query(&[ + ("z", "a b~c/雪"), + ("a~", "second"), + ("a ", "first"), + ("amp", "&="), + ("dup", "z"), + ("dup", "a"), + ("empty", ""), + ("雪", "key"), + ]); + + assert_eq!( + query, + "%E9%9B%AA=key&a%20=first&=%26%3D&a~=second&dup=a&dup=z&empty=&z=a%20b~c%2F%E9%9B%AA" + ); +} + +#[test] +fn form_body_keeps_html_form_encoding() { + assert_eq!( + build_form_body(&[("Policy", "a b~c/雪")]), + "Policy=a+b%7Ec%2F%E9%9B%AA" + ); +} + +#[test] +fn duplicate_query_values_match_independent_sigv4_verification() { + let query = + build_canonical_query(&[("dup", "z z"), ("dup", "a+a"), ("dup", "雪"), ("empty", "")]); + assert_eq!(query, "dup=%E9%9B%AA&dup=a%2Ba&dup=z%20z&empty="); + + let client = RustfsAdminClient::new_with_base_url( + "https://rustfs.example.test:9000", + TEST_ACCESS_KEY, + TEST_SECRET_KEY, + ); + let signed = client + .sign_request("GET", "/synthetic", &query, "", None, ADMIN_SIGNING_SERVICE) + .unwrap(); + let request = CapturedRequest { + method: "GET".to_string(), + path: "/synthetic".to_string(), + query, + body: String::new(), + host: "rustfs.example.test:9000".to_string(), + content_type: String::new(), + amz_date: signed.amz_date, + payload_hash: signed.payload_hash, + authorization: signed.authorization, + }; + + assert_sigv4_matches_wire(&request, ADMIN_SIGNING_SERVICE); +} + +fn secret_with_fields(fields: Vec<(&str, &[u8])>) -> corev1::Secret { + let mut data = BTreeMap::new(); + for (key, value) in fields { + data.insert(key.to_string(), ByteString(value.to_vec())); + } + + corev1::Secret { + data: Some(data), + ..Default::default() + } +} + +fn assert_oversized_upstream_body_hidden(err: RustfsClientError) { + assert_eq!( + err.to_string(), + format!( + "upstream returned 502 Bad Gateway: response body exceeded {MAX_UPSTREAM_ERROR_BODY_BYTES} bytes" + ) + ); +} #[test] fn tls_tenant_base_url_uses_custom_cluster_domain() { @@ -29,3 +130,1134 @@ fn tls_tenant_base_url_uses_custom_cluster_domain() { "https://prod-rustfs-hl.mse.svc.k8s.mse.cloud:9000" ); } + +#[test] +fn extract_credentials_reports_missing_access_key() { + let secret = secret_with_fields(vec![("secretkey", b"sekret")]); + + let err = extract_credentials(secret.data.as_ref()).expect_err("expected missing access key"); + assert!(matches!( + err, + RustfsClientError::MissingCredentialKey { key: "accesskey" } + )); +} + +#[test] +fn extract_credentials_reports_non_utf8_access_key() { + let secret = secret_with_fields(vec![("accesskey", &[0xff, 0xfe]), ("secretkey", b"sekret")]); + + let err = extract_credentials(secret.data.as_ref()).expect_err("expected invalid utf8"); + assert!(matches!( + err, + RustfsClientError::InvalidCredentialValue { key: "accesskey" } + )); +} + +#[test] +fn extract_credentials_reports_missing_secret_key() { + let secret = secret_with_fields(vec![("accesskey", b"access")]); + + let err = extract_credentials(secret.data.as_ref()).expect_err("expected missing secret key"); + assert!(matches!( + err, + RustfsClientError::MissingCredentialKey { key: "secretkey" } + )); +} + +#[test] +fn extract_credentials_reports_non_utf8_secret_key() { + let secret = secret_with_fields(vec![("accesskey", b"access"), ("secretkey", &[0xff, 0xfe])]); + + let err = extract_credentials(secret.data.as_ref()).expect_err("expected invalid utf8"); + assert!(matches!( + err, + RustfsClientError::InvalidCredentialValue { key: "secretkey" } + )); +} + +#[test] +fn extract_credentials_reports_empty_secret_key() { + let secret = secret_with_fields(vec![("accesskey", b"abc"), ("secretkey", b"")]); + + let err = extract_credentials(secret.data.as_ref()).expect_err("expected empty secret key"); + assert!(matches!( + err, + RustfsClientError::EmptyCredentialValue { key: "secretkey" } + )); +} + +#[test] +fn parse_assume_role_xml_success_and_failure() { + let body_ok = "AKISECTOKEN2026-01-01T00:00:00Z"; + let parsed = + parse_assume_role_response(body_ok).expect("valid assume role response should parse"); + + assert_eq!(parsed.access_key_id, "AKI"); + assert_eq!(parsed.secret_access_key, "SEC"); + assert_eq!(parsed.session_token, "TOKEN"); + assert_eq!(parsed.expiration, "2026-01-01T00:00:00Z"); + + assert!(parse_assume_role_response("").is_none()); +} + +#[test] +fn unexpected_status_includes_upstream_xml_error_summary() { + let err = RustfsClientError::unexpected_status_with_body( + StatusCode::BAD_REQUEST, + r#"InvalidRequestinvalid resource: unknown "*"abc"#, + ); + + let message = err.to_string(); + assert_eq!( + message, + r#"upstream returned 400 Bad Request: InvalidRequest: invalid resource: unknown "*""# + ); + assert!(!message.contains("")); +} + +#[test] +fn unexpected_status_includes_upstream_json_error_summary() { + let err = RustfsClientError::unexpected_status_with_body( + StatusCode::BAD_REQUEST, + r#"{"code":"InvalidRequest","message":"policy Resource must use ARN form"}"#, + ); + + assert_eq!( + err.to_string(), + "upstream returned 400 Bad Request: InvalidRequest: policy Resource must use ARN form" + ); +} + +#[test] +fn unexpected_status_redacts_sensitive_upstream_error_summary() { + let err = RustfsClientError::unexpected_status_with_body( + StatusCode::BAD_REQUEST, + r#"{"code":"InvalidRequest","message":"secretkey: SK_TEST clientSecret: oidc-secret SecretAccessKey: SK_STS AccessKeyId: AKIA_STS SK_XML AKIA_XML"}"#, + ); + + let message = err.to_string(); + assert!(message.contains("secretkey: ")); + assert!(message.contains("clientSecret: ")); + assert!(message.contains("SecretAccessKey: ")); + assert!(message.contains("AccessKeyId: ")); + assert!(message.contains("")); + assert!(message.contains("")); + assert!(!message.contains("SK_TEST")); + assert!(!message.contains("oidc-secret")); + assert!(!message.contains("SK_STS")); + assert!(!message.contains("AKIA_STS")); + assert!(!message.contains("SK_XML")); + assert!(!message.contains("AKIA_XML")); +} + +#[test] +fn unexpected_status_hides_truncated_unstructured_response_body() { + let retained_body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES); + let err = RustfsClientError::unexpected_status_with_limited_body( + StatusCode::BAD_GATEWAY, + &retained_body, + true, + ); + + assert_eq!( + err.to_string(), + format!( + "upstream returned 502 Bad Gateway: response body exceeded {MAX_UPSTREAM_ERROR_BODY_BYTES} bytes" + ) + ); +} + +#[tokio::test] +async fn unexpected_response_preserves_exact_limit_unstructured_response_body() { + let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES); + let router = Router::new().route( + ADD_USER_PATH, + put(move || { + let body = body.clone(); + async move { (StatusCode::BAD_GATEWAY, body) } + }), + ); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let err = client + .add_user("app-user", "secret123") + .await + .expect_err("exact limit body should still report the retained body"); + + let message = err.to_string(); + assert!(message.contains("upstream returned 502 Bad Gateway")); + assert!(!message.contains("response body exceeded")); + + server.abort(); +} + +#[tokio::test] +async fn unexpected_response_hides_over_limit_unstructured_response_body() { + let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES + 1); + let router = Router::new().route( + ADD_USER_PATH, + put(move || { + let body = body.clone(); + async move { (StatusCode::BAD_GATEWAY, body) } + }), + ); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let err = client + .add_user("app-user", "secret123") + .await + .expect_err("oversized body should be hidden"); + + assert_oversized_upstream_body_hidden(err); + + server.abort(); +} + +#[derive(Clone, Default)] +struct Capture { + method: Arc>, + path: Arc>, + query: Arc>, + body: Arc>, + host: Arc>, + content_type: Arc>, + amz_date: Arc>, + payload_hash: Arc>, + authorization: Arc>, + object_lock_header: Arc>, +} + +#[derive(Debug)] +struct CapturedRequest { + method: String, + path: String, + query: String, + body: String, + host: String, + content_type: String, + amz_date: String, + payload_hash: String, + authorization: String, +} + +impl Capture { + async fn request(&self) -> CapturedRequest { + CapturedRequest { + method: self.method.lock().await.clone(), + path: self.path.lock().await.clone(), + query: self.query.lock().await.clone(), + body: self.body.lock().await.clone(), + host: self.host.lock().await.clone(), + content_type: self.content_type.lock().await.clone(), + amz_date: self.amz_date.lock().await.clone(), + payload_hash: self.payload_hash.lock().await.clone(), + authorization: self.authorization.lock().await.clone(), + } + } +} + +fn request_header(req: &Request, name: &str) -> String { + req.headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string() +} + +async fn capture_signed_request(capture: &Capture, req: Request) { + let method = req.method().as_str().to_string(); + let path = req.uri().path().to_string(); + let query = req.uri().query().unwrap_or("").to_string(); + let host = request_header(&req, "host"); + let content_type = request_header(&req, "content-type"); + let amz_date = request_header(&req, "x-amz-date"); + let payload_hash = request_header(&req, "x-amz-content-sha256"); + let authorization = request_header(&req, "authorization"); + let body = axum::body::to_bytes(req.into_body(), usize::MAX) + .await + .unwrap(); + let body = String::from_utf8(body.to_vec()).unwrap(); + + *capture.method.lock().await = method; + *capture.path.lock().await = path; + *capture.query.lock().await = query; + *capture.body.lock().await = body; + *capture.host.lock().await = host; + *capture.content_type.lock().await = content_type; + *capture.amz_date.lock().await = amz_date; + *capture.payload_hash.lock().await = payload_hash; + *capture.authorization.lock().await = authorization; +} + +fn assert_sigv4_matches_wire(request: &CapturedRequest, service: &str) { + let calculated_payload_hash = sha256_hex(request.body.as_bytes()); + assert_eq!(request.payload_hash, calculated_payload_hash); + + let signed_header_names = if request.content_type.is_empty() { + "host;x-amz-content-sha256;x-amz-date" + } else { + "content-type;host;x-amz-content-sha256;x-amz-date" + }; + let mut canonical_headers = String::new(); + if !request.content_type.is_empty() { + canonical_headers.push_str("content-type:"); + canonical_headers.push_str(request.content_type.trim()); + canonical_headers.push('\n'); + } + canonical_headers.push_str("host:"); + canonical_headers.push_str(request.host.trim()); + canonical_headers.push_str("\nx-amz-content-sha256:"); + canonical_headers.push_str(request.payload_hash.trim()); + canonical_headers.push_str("\nx-amz-date:"); + canonical_headers.push_str(request.amz_date.trim()); + canonical_headers.push('\n'); + + let canonical_request = format!( + "{}\n{}\n{}\n{}\n{}\n{}", + request.method, + request.path, + request.query, + canonical_headers, + signed_header_names, + request.payload_hash + ); + let date_stamp = request + .amz_date + .get(..8) + .expect("x-amz-date must start with YYYYMMDD"); + let credential_scope = format!("{date_stamp}/{TEST_REGION}/{service}/aws4_request"); + let string_to_sign = format!( + "AWS4-HMAC-SHA256\n{}\n{}\n{}", + request.amz_date, + credential_scope, + sha256_hex(canonical_request.as_bytes()) + ); + let signing_key = + derive_signing_key(TEST_SECRET_KEY, date_stamp, TEST_REGION, service).unwrap(); + let signature = hmac_sha256_hex(&signing_key, &string_to_sign).unwrap(); + let expected_authorization = format!( + "AWS4-HMAC-SHA256 Credential={TEST_ACCESS_KEY}/{credential_scope}, SignedHeaders={signed_header_names}, Signature={signature}" + ); + + assert_eq!(request.authorization, expected_authorization); +} + +#[tokio::test] +async fn assume_role_request_targets_root_path_and_action_is_assume_role() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new().route( + "/", + post( + move |State(c): State, req: Request| async move { + capture_signed_request(&c, req).await; + + let response = + "AKISECTOKEN2026-01-01T00:00:00Z"; + (StatusCode::OK, response) + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + + let creds = client + .assume_role(Some(r#"{"Statement":[{"Resource":"a b~+/雪"}]}"#), 3600) + .await + .unwrap(); + assert_eq!(creds.access_key_id, "AKI"); + + let request = capture.request().await; + assert_eq!(request.path, "/"); + assert_eq!( + request.body, + "Action=AssumeRole&DurationSeconds=3600&Policy=%7B%22Statement%22%3A%5B%7B%22Resource%22%3A%22a+b%7E%2B%2F%E9%9B%AA%22%7D%5D%7D&Version=2011-06-15" + ); + assert!(request.query.is_empty()); + assert_eq!(request.content_type, FORM_CONTENT_TYPE); + assert_sigv4_matches_wire(&request, STS_SIGNING_SERVICE); + + server.abort(); +} + +#[tokio::test] +async fn info_canned_policy_uses_expected_path_and_query() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + "/rustfs/admin/v3/info-canned-policy", + get( + move |State(c): State, req: Request| async move { + let path = req.uri().path().to_string(); + let query = req.uri().query().unwrap_or("").to_string(); + let authorization = req + .headers() + .get("authorization") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + + *c.path.lock().await = path; + *c.query.lock().await = query; + *c.authorization.lock().await = authorization; + + ( + StatusCode::OK, + "{\"policy_name\":\"tenant-policy\",\"policy\":{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"allow\",\"Effect\":\"Allow\"}]}}", + ) + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + + let policy = client.get_canned_policy("tenant-policy").await.unwrap(); + let policy_value = serde_json::from_str::(&policy).unwrap(); + assert_eq!(policy_value["Version"], "2012-10-17"); + assert_eq!(policy_value["Statement"][0]["Sid"], "allow"); + + assert_eq!( + &*capture.path.lock().await, + "/rustfs/admin/v3/info-canned-policy" + ); + assert!(capture.query.lock().await.contains("name=tenant-policy")); + assert!( + capture + .authorization + .lock() + .await + .contains("/s3/aws4_request") + ); + + server.abort(); +} + +#[tokio::test] +async fn list_canned_policies_extracts_policy_document_and_canonicalizes_json() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + LIST_CANNED_POLICIES_PATH, + get( + move |State(c): State, req: Request| async move { + let path = req.uri().path().to_string(); + let query = req.uri().query().unwrap_or("").to_string(); + + *c.path.lock().await = path; + *c.query.lock().await = query; + + ( + StatusCode::OK, + serde_json::json!({ + "tenant-policy": { + "policy_name":"tenant-policy", + "policy":{ + "Statement": [{ + "Resource": "arn:aws:s3:::tenant", + "Effect": "Allow", + "Action": "s3:GetObject" + }], + "Version":"2012-10-17" + } + }, + "inline-policy": { + "Version": "2012-10-17", + "Statement": [{ + "Sid": "inline", + "Action": "s3:ListBucket", + "Effect": "Allow", + "Resource": ["arn:aws:s3:::tenant*"] + }] + } + }) + .to_string(), + ) + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let policies = client.list_canned_policies().await.unwrap(); + + let tenant_policy = serde_json::from_str::(&policies["tenant-policy"]).unwrap(); + assert_eq!(tenant_policy["Version"], "2012-10-17"); + assert_eq!(tenant_policy["Statement"][0]["Action"], "s3:GetObject"); + + let inline_policy = serde_json::from_str::(&policies["inline-policy"]).unwrap(); + assert_eq!(inline_policy["Version"], "2012-10-17"); + assert_eq!(inline_policy["Statement"][0]["Sid"], "inline"); + assert_eq!(&*capture.path.lock().await, LIST_CANNED_POLICIES_PATH); + assert!(capture.query.lock().await.is_empty()); + + server.abort(); +} + +#[tokio::test] +async fn add_canned_policy_uses_expected_path_query_body_and_admin_signing() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + "/rustfs/admin/v3/add-canned-policy", + put( + move |State(c): State, req: Request| async move { + let path = req.uri().path().to_string(); + let query = req.uri().query().unwrap_or("").to_string(); + let authorization = req + .headers() + .get("authorization") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + let body_bytes = axum::body::to_bytes(req.into_body(), usize::MAX) + .await + .unwrap(); + let body = String::from_utf8(body_bytes.to_vec()).unwrap(); + + *c.path.lock().await = path; + *c.query.lock().await = query; + *c.authorization.lock().await = authorization; + *c.body.lock().await = body; + + StatusCode::OK + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let policy = r#"{"Version":"2012-10-17","Statement":[]}"#; + + client + .add_canned_policy("tenant-policy", policy) + .await + .unwrap(); + + assert_eq!( + &*capture.path.lock().await, + "/rustfs/admin/v3/add-canned-policy" + ); + assert!(capture.query.lock().await.contains("name=tenant-policy")); + assert_eq!(&*capture.body.lock().await, policy); + assert!( + capture + .authorization + .lock() + .await + .contains("/s3/aws4_request") + ); + + server.abort(); +} + +#[tokio::test] +async fn add_canned_policy_reports_upstream_policy_parse_error() { + let router = Router::new().route( + "/rustfs/admin/v3/add-canned-policy", + put(|| async { + ( + StatusCode::BAD_REQUEST, + r#"InvalidRequestinvalid resource: unknown "*""#, + ) + }), + ); + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let policy = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}"#; + let err = client + .add_canned_policy("tenant-policy", policy) + .await + .expect_err("invalid RustFS policy should include upstream parse details"); + + let message = err.to_string(); + assert!(message.contains("upstream returned 400 Bad Request")); + assert!(message.contains(r#"InvalidRequest: invalid resource: unknown "*""#)); + assert!(!message.contains("")); + + server.abort(); +} + +#[tokio::test] +async fn server_info_uses_expected_path_and_parses_wrapped_health_fields() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + SERVER_INFO_PATH, + get( + move |State(c): State, req: Request| async move { + let path = req.uri().path().to_string(); + let authorization = req + .headers() + .get("authorization") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + + *c.path.lock().await = path; + *c.authorization.lock().await = authorization; + + ( + StatusCode::OK, + serde_json::json!({ + "info": { + "usage": {"size": 42}, + "backend": { + "onlineDisks": 3, + "offlineDisks": 1, + "standardSCParity": 2, + "totalSets": [1], + "totalDrivesPerSet": [4] + }, + "pools": { + "0": { + "0": { + "rawUsage": 100, + "rawCapacity": 400, + "usage": 50, + "objectsCount": 2, + "healDisks": 1 + } + } + } + }, + "admin_discovery": { + "runtimeCapabilities": "/rustfs/admin/v4/runtime/capabilities", + "clusterSnapshot": "/rustfs/admin/v4/cluster/snapshot", + "extensionsCatalog": "/rustfs/admin/v4/extensions/catalog" + }, + }) + .to_string(), + ) + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let info = client.server_info().await.unwrap(); + + let backend = info.backend.unwrap(); + assert_eq!(backend.online_disks, 3); + assert_eq!(backend.offline_disks, 1); + assert_eq!(backend.standard_sc_parity, Some(2)); + assert_eq!(info.usage.unwrap().size, 42); + assert_eq!(info.pools.unwrap()["0"]["0"].raw_capacity, 400); + assert_eq!(&*capture.path.lock().await, SERVER_INFO_PATH); + assert!( + capture + .authorization + .lock() + .await + .contains("/s3/aws4_request") + ); + + server.abort(); +} + +#[tokio::test] +async fn list_pools_parses_current_rustfs_pool_shape() { + let router = Router::new().route( + POOLS_LIST_PATH, + get(|| async { + ( + StatusCode::OK, + r#"[{"id":1,"cmdline":"http://tenant-pool-a-{0...3}.tenant-hl.ns.svc.cluster.local:9000/data/rustfs{0...3}","lastUpdate":"2026-05-20T00:00:00Z","totalSize":100,"currentSize":50,"usedSize":25,"used":25.0,"status":"running","decommissionInfo":{"startTime":"2026-05-20T00:00:00Z","complete":false,"failed":false,"canceled":false,"objectsDecommissioned":7,"objectsDecommissionedFailed":1,"bytesDecommissioned":9,"bytesDecommissionedFailed":2}}]"#, + ) + }), + ); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + + let pools = client.list_pools().await.unwrap(); + + assert_eq!(pools[0].id, 1); + assert_eq!(pools[0].status, "running"); + assert_eq!( + pools[0] + .decommission + .as_ref() + .and_then(|info| info.objects_decommissioned), + Some(7) + ); + + server.abort(); +} + +#[tokio::test] +async fn pool_decommission_start_uses_by_id_query_and_admin_signing() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + POOLS_DECOMMISSION_PATH, + post( + move |State(c): State, req: Request| async move { + *c.path.lock().await = req.uri().path().to_string(); + *c.query.lock().await = req.uri().query().unwrap_or("").to_string(); + *c.authorization.lock().await = req + .headers() + .get("authorization") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + + StatusCode::OK + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + + client.start_pool_decommission_by_id("1").await.unwrap(); + + assert_eq!(&*capture.path.lock().await, POOLS_DECOMMISSION_PATH); + assert_eq!(&*capture.query.lock().await, "by-id=true&pool=1"); + assert!( + capture + .authorization + .lock() + .await + .contains("/s3/aws4_request") + ); + + server.abort(); +} + +#[tokio::test] +async fn pool_status_uses_by_id_query_and_parses_decommission_info() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + POOLS_STATUS_PATH, + get( + move |State(c): State, req: Request| async move { + *c.path.lock().await = req.uri().path().to_string(); + *c.query.lock().await = req.uri().query().unwrap_or("").to_string(); + + ( + StatusCode::OK, + r#"{"id":1,"cmdline":"http://tenant-pool-a-{0...3}.tenant-hl.ns.svc.cluster.local:9000/data/rustfs{0...3}","lastUpdate":"2026-05-20T00:00:00Z","decommissionInfo":{"startTime":"2026-05-20T00:00:00Z","complete":true,"failed":false,"canceled":false,"objectsDecommissioned":10,"objectsDecommissionedFailed":0,"bytesDecommissioned":20,"bytesDecommissionedFailed":0}}"#, + ) + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + + let status = client.pool_status_by_id("1").await.unwrap(); + + assert_eq!(status.id, 1); + assert_eq!(&*capture.path.lock().await, POOLS_STATUS_PATH); + assert_eq!(&*capture.query.lock().await, "by-id=true&pool=1"); + assert_eq!( + status.decommission.and_then(|info| info.complete), + Some(true) + ); + + server.abort(); +} + +#[tokio::test] +async fn add_user_uses_expected_path_query_and_body() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + ADD_USER_PATH, + put( + move |State(c): State, req: Request| async move { + capture_signed_request(&c, req).await; + StatusCode::OK + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + client.add_user("app user~+/雪", "secret123").await.unwrap(); + + let request = capture.request().await; + assert_eq!(request.path, ADD_USER_PATH); + assert_eq!(request.query, "accessKey=app%20user~%2B%2F%E9%9B%AA"); + assert_eq!( + request.body, + r#"{"secretKey":"secret123","status":"enabled"}"# + ); + assert_eq!(request.content_type, JSON_CONTENT_TYPE); + assert_sigv4_matches_wire(&request, ADMIN_SIGNING_SERVICE); + + server.abort(); +} + +#[tokio::test] +async fn user_exists_limits_unexpected_error_response_body() { + let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES + 1); + let router = Router::new().route( + USER_INFO_PATH, + get(move || { + let body = body.clone(); + async move { (StatusCode::BAD_GATEWAY, body) } + }), + ); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let err = client + .user_exists("app-user") + .await + .expect_err("unexpected user lookup error should hide oversized body"); + + assert_oversized_upstream_body_hidden(err); + + server.abort(); +} + +#[tokio::test] +async fn get_user_info_parses_comma_separated_policy_names() { + let router = Router::new().route( + USER_INFO_PATH, + get(|| async { + ( + StatusCode::OK, + r#"{"status":"enabled","policyName":"cosi-mlflow,cosi-grant-ba-1"}"#, + ) + }), + ); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let info = client + .get_user_info("mlflow") + .await + .unwrap() + .expect("user should exist"); + assert_eq!( + info.policy_names, + vec!["cosi-mlflow".to_string(), "cosi-grant-ba-1".to_string()] + ); + assert!(client.user_exists("mlflow").await.unwrap()); + + server.abort(); +} + +#[tokio::test] +async fn get_user_info_returns_none_for_missing_user() { + let router = Router::new().route( + USER_INFO_PATH, + get(|| async { (StatusCode::NOT_FOUND, "NoSuchUser") }), + ); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + assert!(client.get_user_info("missing").await.unwrap().is_none()); + assert!(!client.user_exists("missing").await.unwrap()); + + server.abort(); +} + +#[tokio::test] +async fn set_user_policy_uses_single_authoritative_mapping_call() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + SET_POLICY_PATH, + put( + move |State(c): State, req: Request| async move { + *c.path.lock().await = req.uri().path().to_string(); + *c.query.lock().await = req.uri().query().unwrap_or("").to_string(); + StatusCode::OK + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + client + .set_user_policy( + "app-user", + &["app-readwrite".to_string(), "diagnostics".to_string()], + ) + .await + .unwrap(); + + assert_eq!(&*capture.path.lock().await, SET_POLICY_PATH); + assert_eq!( + &*capture.query.lock().await, + "isGroup=false&policyName=app-readwrite%2Cdiagnostics&userOrGroup=app-user" + ); + + server.abort(); +} + +#[tokio::test] +async fn set_user_policy_rejects_empty_policy_list() { + let client = RustfsAdminClient::new_with_base_url("http://127.0.0.1:1", "access", "secret"); + + let err = client + .set_user_policy("app-user", &[]) + .await + .expect_err("empty policy list should be rejected before request"); + + assert!(matches!(err, RustfsClientError::InvalidPolicyName)); +} + +#[tokio::test] +async fn bucket_object_lock_enabled_parses_enabled_response() { + let router = Router::new().route( + "/app-data", + get(|req: Request| async move { + assert_eq!(req.uri().query().unwrap_or(""), "object-lock="); + ( + StatusCode::OK, + "Enabled", + ) + }), + ); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + + assert!(client.bucket_object_lock_enabled("app-data").await.unwrap()); + + server.abort(); +} + +#[tokio::test] +async fn bucket_object_lock_enabled_limits_unexpected_error_response_body() { + let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES + 1); + let router = Router::new().route( + "/app-data", + get(move |req: Request| { + let body = body.clone(); + async move { + assert_eq!(req.uri().query().unwrap_or(""), "object-lock="); + (StatusCode::BAD_GATEWAY, body) + } + }), + ); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let err = client + .bucket_object_lock_enabled("app-data") + .await + .expect_err("unexpected object-lock error should hide oversized body"); + + assert_oversized_upstream_body_hidden(err); + + server.abort(); +} + +#[tokio::test] +async fn create_bucket_sends_object_lock_header_and_region_body() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + "/app-data", + put( + move |State(c): State, req: Request| async move { + *c.path.lock().await = req.uri().path().to_string(); + *c.object_lock_header.lock().await = req + .headers() + .get("x-amz-bucket-object-lock-enabled") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + let body_bytes = axum::body::to_bytes(req.into_body(), usize::MAX) + .await + .unwrap(); + *c.body.lock().await = String::from_utf8(body_bytes.to_vec()).unwrap(); + StatusCode::OK + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let result = client + .create_bucket("app-data", Some("us-west-2"), true) + .await + .unwrap(); + + assert_eq!(result, CreateBucketResult::Created); + assert_eq!(&*capture.path.lock().await, "/app-data"); + assert_eq!(&*capture.object_lock_header.lock().await, "true"); + assert!( + capture + .body + .lock() + .await + .contains("us-west-2") + ); + + server.abort(); +} + +#[tokio::test] +async fn create_bucket_limits_unexpected_error_response_body() { + let body = "x".repeat(MAX_UPSTREAM_ERROR_BODY_BYTES + 1); + let router = Router::new().route( + "/app-data", + put(move || { + let body = body.clone(); + async move { (StatusCode::BAD_GATEWAY, body) } + }), + ); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let err = client + .create_bucket("app-data", None, false) + .await + .expect_err("unexpected bucket create error should hide oversized body"); + + assert_oversized_upstream_body_hidden(err); + + server.abort(); +} + +#[test] +fn extract_canned_policy_document_accepts_raw_policy_document() { + let raw_policy = + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"raw\",\"Effect\":\"Allow\"}]}"; + + let policy = extract_canned_policy_document(raw_policy).unwrap(); + + let policy_value = serde_json::from_str::(&policy).unwrap(); + assert_eq!(policy_value["Version"], "2012-10-17"); + assert_eq!(policy_value["Statement"][0]["Sid"], "raw"); +} diff --git a/src/sts/types.rs b/src/sts/types.rs index 3769171c..ecf8beb8 100644 --- a/src/sts/types.rs +++ b/src/sts/types.rs @@ -113,7 +113,13 @@ pub fn parse_sts_form( }) } -pub use rustfs_admin::StsAssumeRoleCredentials; +#[derive(Debug, Clone)] +pub struct StsAssumeRoleCredentials { + pub access_key_id: String, + pub secret_access_key: String, + pub session_token: String, + pub expiration: String, +} #[derive(Debug, Clone)] pub struct StsWebIdentityResponseContext { diff --git a/src/tenant_monitor.rs b/src/tenant_monitor.rs index 00279a33..6b428e8f 100644 --- a/src/tenant_monitor.rs +++ b/src/tenant_monitor.rs @@ -14,10 +14,7 @@ use crate::{ metrics::{self, TenantStorageMetrics}, - sts::rustfs_client::{ - RustfsServerInfo, client_from_tenant, client_from_tls_tenant_for_sts, - load_tenant_credentials, - }, + sts::rustfs_client::{RustfsAdminClient, RustfsServerInfo}, types::v1alpha1::tenant::Tenant, }; use futures::{StreamExt, stream}; @@ -170,11 +167,12 @@ async fn poll_tenant_storage( tenant: &Tenant, cluster_domain: &str, ) -> Result> { - let credentials = load_tenant_credentials(client, tenant).await?; + let credentials = RustfsAdminClient::load_tenant_credentials(client, tenant).await?; let rustfs_client = if tenant.spec.tls.as_ref().is_some_and(|tls| tls.is_enabled()) { - client_from_tls_tenant_for_sts(client, tenant, credentials, cluster_domain).await? + RustfsAdminClient::from_tls_tenant_for_sts(client, tenant, credentials, cluster_domain) + .await? } else { - client_from_tenant(tenant, credentials)? + RustfsAdminClient::from_tenant(tenant, credentials)? }; let info = rustfs_client.server_info().await?; From 3de8b3715cbb68060118f23347d80c79c337b0b6 Mon Sep 17 00:00:00 2001 From: benjamin fuentes Date: Tue, 4 Aug 2026 14:23:42 +0200 Subject: [PATCH 3/4] chore: update .gitignore to include .kubeconfig --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 183907c3..e43b1908 100755 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ docs/* .claude/ .cursor/ .gemini/ +.kubeconfig \ No newline at end of file From 2bbfe287082bfcf92477efc0f45788bd967221d1 Mon Sep 17 00:00:00 2001 From: benjamin fuentes Date: Tue, 4 Aug 2026 14:24:00 +0200 Subject: [PATCH 4/4] chore: correct .gitignore entry for kubeconfig --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e43b1908..c2e48faa 100755 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,4 @@ docs/* .claude/ .cursor/ .gemini/ -.kubeconfig \ No newline at end of file +kubeconfig \ No newline at end of file