diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78ec06f..9d06be6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,6 +103,7 @@ jobs: run: | set -euo pipefail helm lint deploy/rustfs-operator + helm lint deploy/rustfs-tenant # Default: cosiDriver disabled, nothing COSI-related should render. helm template test deploy/rustfs-operator > /tmp/helm-default.yaml diff --git a/.github/workflows/helm-package.yml b/.github/workflows/helm-package.yml index 6946ad1..e6ec290 100644 --- a/.github/workflows/helm-package.yml +++ b/.github/workflows/helm-package.yml @@ -84,15 +84,18 @@ jobs: run: | set -eux helm package ./deploy/rustfs-operator \ - --destination deploy/rustfs-operator/ \ + --destination helm-packages/ \ --version "${{ steps.version.outputs.chart_version }}" \ --app-version "${{ steps.version.outputs.chart_version }}" + helm package ./deploy/rustfs-tenant \ + --destination helm-packages/ \ + --version "${{ steps.version.outputs.chart_version }}" - name: Upload helm package as artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: helm-package - path: deploy/rustfs-operator/*.tgz + path: helm-packages/*.tgz retention-days: 1 publish-helm-package: diff --git a/.gitignore b/.gitignore index c2e48fa..6091ecd 100755 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ CONSOLE-INTEGRATION-SUMMARY.md SCRIPTS-UPDATE.md AGENTS.md docs/* +!docs/tenant-networking.md !docs/operator-user-guide.md !docs/operator-user-guide.zh-CN.md .codex diff --git a/CHANGELOG.md b/CHANGELOG.md index e7e020e..82b1270 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,10 @@ All notable changes to RustFS Operator are documented in this file. The format i ### Added +- Optional `rustfs-tenant` Helm chart for a Tenant and separate API/Console Ingress + or HTTPRoute resources, with backend TLS policies, native YAML/Kustomize examples, + and networking lifecycle documentation. Existing CRDs and controller behavior are unchanged. + - Tenant `spec.oidc.extraCaCertSecretRef` for validated, reloadable custom CA trust used by RustFS OIDC outbound connections. - Tenant `spec.additionalVolumes` and `spec.additionalVolumeMounts` for Kubernetes-native file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 158429d..2cec430 100755 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,6 +10,10 @@ ## 📋 Code Quality Requirements +Helm 3 must be on `PATH` for `cargo test --all` and `make pre-commit`: the Tenant +chart contract tests render the chart and verify its backends against Operator +Service generation. CI installs Helm before running these tests. + ### 🔧 Code Formatting Rules **MANDATORY**: All code must be properly formatted before committing. This project enforces strict formatting standards to maintain code consistency and readability. diff --git a/README.md b/README.md index bc35c64..9591b7e 100755 --- a/README.md +++ b/README.md @@ -174,6 +174,7 @@ Then use `http://127.0.0.1:19000` for the Tenant S3 API and `http://127.0.0.1:19 |-----|---------| | [CONTRIBUTING.md](CONTRIBUTING.md) | Quality gates, `make pre-commit`, PR rules. | | [examples/README.md](examples/README.md) | Tenant manifests and usage notes. | +| [docs/tenant-networking.md](docs/tenant-networking.md) | Tenant networking with native YAML, Kustomize, or the optional Tenant chart. | | [deploy/README.md](deploy/README.md) | Helm and Kubernetes deployment entry point. | | [deploy/rustfs-operator/README.md](deploy/rustfs-operator/README.md) | Helm chart values and examples. | | [console-web/README.md](console-web/README.md) | Operator console frontend development and deployment. | diff --git a/deploy/README.md b/deploy/README.md index be7e1a3..849096b 100755 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,6 +1,6 @@ # RustFS Operator Deployment -This directory contains the Helm chart for deploying the RustFS Kubernetes operator. +This directory contains separate charts for installing the RustFS Operator and deploying Tenants. ## Quick Start @@ -20,6 +20,15 @@ helm install rustfs-operator deploy/rustfs-operator/ \ - CRD installation - Example Tenant resources +- **rustfs-tenant/** - Optional Tenant deployment chart with API/Console Ingress or HTTPRoute. + It requires an existing Operator; it does not change existing Tenant YAML or install a Gateway. + +## Tenant networking + +Use [plain Kubernetes YAML/Kustomize](../docs/tenant-networking.md) or the +[separate Tenant chart](rustfs-tenant/README.md). All external endpoints are disabled +by default in the chart. Existing deployments do not need to migrate. + ## Documentation See the [Helm chart README](rustfs-operator/README.md) for detailed configuration options and usage examples. diff --git a/deploy/rustfs-tenant/Chart.yaml b/deploy/rustfs-tenant/Chart.yaml new file mode 100644 index 0000000..22ed5b6 --- /dev/null +++ b/deploy/rustfs-tenant/Chart.yaml @@ -0,0 +1,26 @@ +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: rustfs-tenant +description: A RustFS Tenant with optional S3 and Console ingress resources +type: application +version: 0.0.6 +home: https://rustfs.com +sources: + - https://github.com/rustfs/operator +keywords: + - storage + - s3 + - rustfs diff --git a/deploy/rustfs-tenant/README.md b/deploy/rustfs-tenant/README.md new file mode 100644 index 0000000..5dfc8c0 --- /dev/null +++ b/deploy/rustfs-tenant/README.md @@ -0,0 +1,145 @@ + + +# RustFS Tenant chart + +Deploy a Tenant and optional S3 API / Tenant Console networking as one Helm release, +or render ordinary Kubernetes YAML for GitOps and `kubectl apply -f`. +Install the RustFS Operator and its CRD separately; this chart has no Operator dependency. +See the [networking guide](../../docs/tenant-networking.md) for plain YAML, TLS, +virtual-hosted S3, verification, and lifecycle details. + +## Resource ownership + +| Owner | Resources | +| --- | --- | +| This chart / your GitOps application | Tenant, optional Ingress / HTTPRoute / BackendTLSPolicy | +| RustFS Operator | Tenant Services, StatefulSets and other existing managed resources | +| Platform administrator | GatewayClass, Gateway, controllers, DNS and external certificates | +| Secret management system / administrator | Credentials, referenced TLS Secrets and CA ConfigMaps | + +The chart does not change the Tenant CRD, controller RBAC, or existing Operator chart. +`ingress` and `httpRoute` are **chart values**, not new Tenant.spec fields. +The Operator Console's `console.ingress` setting belongs to the Operator chart and +is independent of the Tenant Console configured here. + +## Install from this repository + +1. Install the Operator and CRD using the [Operator guide](../../docs/operator-user-guide.md). +2. Create `rustfs-storage` and a `rustfs-credentials` Secret in that namespace, + containing UTF-8 `accesskey` and `secretkey` values (minimum 8 characters each). + Use your secret manager; do not commit credentials in values files. +3. Copy [values.yaml](values.yaml) to `tenant-values.yaml`. Set a tested image, + storage class, pool layout and capacity for your environment. The default is + a **single-node, single-disk development** Tenant, not an HA layout. +4. Install without external access: + +```bash +helm upgrade --install rustfs ./deploy/rustfs-tenant \ + --namespace rustfs-storage --values tenant-values.yaml +``` + +To enable networking, edit the hostnames and platform references in one example: + +```bash +# Choose Ingress OR Gateway API per endpoint. +helm upgrade --install rustfs ./deploy/rustfs-tenant \ + --namespace rustfs-storage --values tenant-values.yaml \ + --values deploy/rustfs-tenant/examples/gateway.yaml +``` + +The Ingress alternative is [examples/ingress.yaml](examples/ingress.yaml). +The example requires an installed IngressClass and a pre-created edge TLS Secret. +The Gateway example requires Gateway API CRDs, a controller, and an existing HTTPS +Gateway listener allowing routes from `rustfs-storage`. It does not create them. + +These commands use the source chart. Use a published repository chart only after +a release containing `rustfs-tenant` is available; do not assume the existing +published `0.0.6` release already contains this addition. The release workflow +packages both charts under the release version. The Tenant chart deliberately +has no `appVersion`: it does not select a RustFS image through chart metadata. + +## Values contract + +| Value | Meaning | +| --- | --- | +| `tenant.metadata.name` | Tenant name; defaults to release name; DNS-1035, max 55 characters | +| `tenant.metadata.labels`, `annotations` | Native metadata maps for the Tenant | +| `tenant.spec` | Native Tenant spec, rendered without `tpl` evaluation or environment injection | +| `ingress.api`, `ingress.console` | Independent Ingress settings; disabled by default | +| `ingress.*.hosts` | Explicit DNS hostnames; no catch-all rule | +| `ingress.*.ingressClassName`, `tls` | Native Ingress fields; TLS uses existing Secrets | +| `ingress.*.labels`, `annotations` | Resource metadata, including controller-specific settings | +| `httpRoute.api`, `httpRoute.console` | Independent HTTPRoute settings; disabled by default | +| `httpRoute.*.parentRefs`, `hostnames` | Native Gateway references and explicit hostnames | +| `httpRoute.*.timeouts` | Native rule timeouts; depends on controller support | +| `httpRoute.*.labels`, `annotations` | HTTPRoute metadata | +| `httpRoute.*.backendTLS` | Optional native BackendTLSPolicy `spec.validation`; requires v1.4+ CRDs and controller support | + +Namespace comes from `--namespace`. Services are always `-io:9000` and +`-console:9001`, matching the existing Operator contract. Each route uses +`/` with prefix matching. The chart does not add rewrites, shared Gateways, DNS, +certificates, arbitrary extra resources, or controller-specific policies. +Use separate standard manifests for advanced platform policies. + +Helm merges maps but **replaces lists**, including `tenant.spec.pools` and `env`. +Keep the complete desired lists in your values file. Clearing a default map uses +`null`, e.g. `tenant.spec.credsSecret: null` when using another supported credential +source. The installed Tenant CRD remains the authority for validating native spec +fields; chart schema validation covers the chart-owned settings. + +## Upgrade and remove + +Keep the release name, namespace and Tenant name stable. Renaming a Tenant creates +a different storage cluster; this chart does not implement data migration. +Do not adopt an existing Tenant or user-managed networking by adding Helm ownership +annotations or using `--take-ownership`. Existing installations can keep using +plain YAML unchanged. Plan any ownership migration separately. + +Set an endpoint's `enabled: false` and apply a Helm upgrade to remove its chart-owned +Ingress/HTTPRoute and its optional backend TLS policy. Disabled endpoints may +retain their configuration for later reuse. Reverting to an internal-only +deployment must disable all previously +enabled endpoints. Use complete desired values; avoid `--reuse-values` when removing +old exposure settings. Switching Ingress to HTTPRoute can interrupt traffic and +needs a planned cutover. + +`helm uninstall rustfs -n rustfs-storage` deletes the chart-owned Tenant and routes; +Tenant deletion triggers the Operator/Kubernetes workload cleanup. Treat it as a +storage teardown, not a networking-only operation. Check PVC/PV retention and +backups before teardown. Referenced credential/certificate resources and shared +Gateways are not deleted by this chart. + +Deleting only the Tenant CR does **not** garbage-collect the chart-owned routes: +there are intentionally no synthetic ownerReferences or lookup hooks. Use the +Helm release lifecycle, or explicitly manage every resource with GitOps. +Helm is not a continuous reconciler; use a GitOps controller if drift correction +of networking resources is required. + +## Validation + +From the repository root (Helm 3 required): + +```bash +helm lint deploy/rustfs-tenant +cargo test --test tenant_chart +kubectl kustomize examples/networking/ingress +kubectl kustomize examples/networking/gateway +``` + +Rendering does not verify the installed CRDs, Gateway attachment, DNS, certificates, +or S3 traffic. Follow the [runtime verification steps](../../docs/tenant-networking.md#verify-the-data-path) +on your target platform. diff --git a/deploy/rustfs-tenant/examples/backend-tls.yaml b/deploy/rustfs-tenant/examples/backend-tls.yaml new file mode 100644 index 0000000..43114cd --- /dev/null +++ b/deploy/rustfs-tenant/examples/backend-tls.yaml @@ -0,0 +1,50 @@ +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Layer over examples/gateway.yaml. Requires cert-manager, an existing Issuer, +# a trusted CA ConfigMap (ca.crt) and Gateway API v1.4+ with BackendTLSPolicy support. +tenant: + spec: + tls: + mode: certManager + enableInternodeHttps: true + certManager: + manageCertificate: true + secretName: rustfs-server-tls + issuerRef: + group: cert-manager.io + kind: Issuer + name: rustfs-issuer + includeGeneratedDnsNames: true + dnsNames: + - rustfs-io.rustfs-storage.svc.cluster.local + - rustfs-console.rustfs-storage.svc.cluster.local + caTrust: + source: CertificateSecretCa + trustSystemCa: false +httpRoute: + api: + backendTLS: + hostname: rustfs-io.rustfs-storage.svc.cluster.local + caCertificateRefs: + - group: "" + kind: ConfigMap + name: rustfs-backend-ca + console: + backendTLS: + hostname: rustfs-console.rustfs-storage.svc.cluster.local + caCertificateRefs: + - group: "" + kind: ConfigMap + name: rustfs-backend-ca diff --git a/deploy/rustfs-tenant/examples/gateway.yaml b/deploy/rustfs-tenant/examples/gateway.yaml new file mode 100644 index 0000000..f2b111d --- /dev/null +++ b/deploy/rustfs-tenant/examples/gateway.yaml @@ -0,0 +1,34 @@ +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The existing Gateway owns the public certificate and HTTPS listener. +httpRoute: + api: + enabled: true + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: shared-gateway + namespace: gateway-system + sectionName: https + hostnames: [s3.example.com] + console: + enabled: true + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: shared-gateway + namespace: gateway-system + sectionName: https + hostnames: [console.example.com] diff --git a/deploy/rustfs-tenant/examples/ingress.yaml b/deploy/rustfs-tenant/examples/ingress.yaml new file mode 100644 index 0000000..cec2666 --- /dev/null +++ b/deploy/rustfs-tenant/examples/ingress.yaml @@ -0,0 +1,30 @@ +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Values file for: helm upgrade --install rustfs deploy/rustfs-tenant -n rustfs-storage -f ... +ingress: + api: + enabled: true + ingressClassName: traefik + hosts: [s3.example.com] + tls: + - hosts: [s3.example.com] + secretName: rustfs-edge-tls + console: + enabled: true + ingressClassName: traefik + hosts: [console.example.com] + tls: + - hosts: [console.example.com] + secretName: rustfs-edge-tls diff --git a/deploy/rustfs-tenant/examples/virtual-hosted.yaml b/deploy/rustfs-tenant/examples/virtual-hosted.yaml new file mode 100644 index 0000000..a978c31 --- /dev/null +++ b/deploy/rustfs-tenant/examples/virtual-hosted.yaml @@ -0,0 +1,24 @@ +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Layer over examples/gateway.yaml. DNS and edge TLS must cover BOTH names. +# Requires a RustFS image supporting RUSTFS_SERVER_DOMAINS. +tenant: + spec: + env: + - name: RUSTFS_SERVER_DOMAINS + value: s3.example.com +httpRoute: + api: + hostnames: [s3.example.com, "*.s3.example.com"] diff --git a/deploy/rustfs-tenant/templates/_helpers.tpl b/deploy/rustfs-tenant/templates/_helpers.tpl new file mode 100644 index 0000000..69a19b0 --- /dev/null +++ b/deploy/rustfs-tenant/templates/_helpers.tpl @@ -0,0 +1,39 @@ +{{/* +Copyright 2026 RustFS Team + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- define "rustfs-tenant.name" -}} +{{- $name := default .Release.Name .Values.tenant.metadata.name -}} +{{- if or (gt (len $name) 55) (not (regexMatch "^[a-z]([-a-z0-9]*[a-z0-9])?$" $name)) -}} +{{- fail "tenant.metadata.name (or release name) must be a DNS-1035 label of at most 55 characters" -}} +{{- end -}} +{{- $name -}} +{{- end -}} + +{{/* Validate offline too: helm template must not depend on cluster discovery. */}} +{{- define "rustfs-tenant.validate" -}} +{{- range $endpoint := list "api" "console" -}} +{{- $ingress := index $.Values.ingress $endpoint -}} +{{- $route := index $.Values.httpRoute $endpoint -}} +{{- if and $ingress.enabled $route.enabled -}} +{{- fail (printf "%s: enable either ingress or httpRoute, not both" $endpoint) -}} +{{- end -}} +{{- if and $ingress.enabled (empty $ingress.hosts) -}} +{{- fail (printf "ingress.%s.hosts must not be empty when enabled" $endpoint) -}} +{{- end -}} +{{- if and $route.enabled (or (empty $route.parentRefs) (empty $route.hostnames)) -}} +{{- fail (printf "httpRoute.%s requires parentRefs and hostnames when enabled" $endpoint) -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/deploy/rustfs-tenant/templates/httproute.yaml b/deploy/rustfs-tenant/templates/httproute.yaml new file mode 100644 index 0000000..f2319ed --- /dev/null +++ b/deploy/rustfs-tenant/templates/httproute.yaml @@ -0,0 +1,73 @@ +{{/* +Copyright 2026 RustFS Team + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- $name := include "rustfs-tenant.name" . -}} +{{- range $endpoint := list "api" "console" }} +{{- $config := index $.Values.httpRoute $endpoint }} +{{- if $config.enabled }} +{{- $service := printf "%s-console" $name }} +{{- $port := 9001 }} +{{- if eq $endpoint "api" }} +{{- $service = printf "%s-io" $name }} +{{- $port = 9000 }} +{{- end }} +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ $name }}-{{ $endpoint }} + namespace: {{ $.Release.Namespace }} + {{- with $config.labels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $config.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + parentRefs: + {{- toYaml $config.parentRefs | nindent 4 }} + hostnames: + {{- toYaml $config.hostnames | nindent 4 }} + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: {{ $service }} + port: {{ $port }} + {{- with $config.timeouts }} + timeouts: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- with $config.backendTLS }} +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: BackendTLSPolicy +metadata: + name: {{ $name }}-{{ $endpoint }} + namespace: {{ $.Release.Namespace }} +spec: + targetRefs: + - group: "" + kind: Service + name: {{ $service }} + validation: + {{- toYaml . | nindent 4 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/deploy/rustfs-tenant/templates/ingress.yaml b/deploy/rustfs-tenant/templates/ingress.yaml new file mode 100644 index 0000000..aefcf21 --- /dev/null +++ b/deploy/rustfs-tenant/templates/ingress.yaml @@ -0,0 +1,62 @@ +{{/* +Copyright 2026 RustFS Team + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- $name := include "rustfs-tenant.name" . -}} +{{- range $endpoint := list "api" "console" }} +{{- $config := index $.Values.ingress $endpoint }} +{{- if $config.enabled }} +{{- $service := printf "%s-console" $name }} +{{- $port := 9001 }} +{{- if eq $endpoint "api" }} +{{- $service = printf "%s-io" $name }} +{{- $port = 9000 }} +{{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $name }}-{{ $endpoint }} + namespace: {{ $.Release.Namespace }} + {{- with $config.labels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $config.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with $config.ingressClassName }} + ingressClassName: {{ . | quote }} + {{- end }} + {{- with $config.tls }} + tls: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + {{- range $config.hosts }} + - host: {{ . | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ $service }} + port: + number: {{ $port }} + {{- end }} +{{- end }} +{{- end }} diff --git a/deploy/rustfs-tenant/templates/tenant.yaml b/deploy/rustfs-tenant/templates/tenant.yaml new file mode 100644 index 0000000..8dc8c4c --- /dev/null +++ b/deploy/rustfs-tenant/templates/tenant.yaml @@ -0,0 +1,31 @@ +{{/* +Copyright 2026 RustFS Team + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- include "rustfs-tenant.validate" . -}} +apiVersion: rustfs.com/v1alpha1 +kind: Tenant +metadata: + name: {{ include "rustfs-tenant.name" . }} + namespace: {{ .Release.Namespace }} + {{- with .Values.tenant.metadata.labels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.tenant.metadata.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- toYaml .Values.tenant.spec | nindent 2 }} diff --git a/deploy/rustfs-tenant/values.schema.json b/deploy/rustfs-tenant/values.schema.json new file mode 100644 index 0000000..51f2dfd --- /dev/null +++ b/deploy/rustfs-tenant/values.schema.json @@ -0,0 +1,315 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$comment": "Copyright 2026 RustFS Team\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.", + "type": "object", + "additionalProperties": false, + "properties": { + "tenant": { + "type": "object", + "additionalProperties": false, + "properties": { + "metadata": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "labels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "annotations": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, + "spec": { + "type": "object", + "required": [ + "pools" + ], + "properties": { + "pools": { + "type": "array", + "minItems": 1, + "items": { + "type": "object" + } + } + } + } + }, + "required": [ + "metadata", + "spec" + ] + }, + "ingress": { + "type": "object", + "additionalProperties": false, + "properties": { + "api": { + "$ref": "#/definitions/ingress" + }, + "console": { + "$ref": "#/definitions/ingress" + } + }, + "required": [ + "api", + "console" + ] + }, + "httpRoute": { + "type": "object", + "additionalProperties": false, + "properties": { + "api": { + "$ref": "#/definitions/httpRoute" + }, + "console": { + "$ref": "#/definitions/httpRoute" + } + }, + "required": [ + "api", + "console" + ] + }, + "global": { + "type": "object", + "description": "Reserved for Helm parent-chart globals; unused by this chart." + } + }, + "required": [ + "tenant", + "ingress", + "httpRoute" + ], + "definitions": { + "ingress": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "labels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "annotations": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "ingressClassName": { + "type": "string" + }, + "hosts": { + "type": "array", + "items": { + "type": "string", + "maxLength": 253, + "pattern": "^(\\*\\.)?[a-z0-9]([a-z0-9.-]*[a-z0-9])?$" + } + }, + "tls": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "hosts": { + "type": "array", + "items": { + "type": "string", + "maxLength": 253, + "pattern": "^(\\*\\.)?[a-z0-9]([a-z0-9.-]*[a-z0-9])?$" + } + }, + "secretName": { + "type": "string", + "minLength": 1 + } + } + } + } + }, + "required": [ + "enabled", + "hosts" + ] + }, + "httpRoute": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "labels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "annotations": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "parentRefs": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "group": { + "type": "string", + "enum": [ + "gateway.networking.k8s.io" + ] + }, + "kind": { + "type": "string", + "enum": [ + "Gateway" + ] + }, + "name": { + "type": "string", + "minLength": 1 + }, + "namespace": { + "type": "string", + "minLength": 1 + }, + "sectionName": { + "type": "string", + "minLength": 1 + }, + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + } + }, + "required": [ + "name" + ] + } + }, + "hostnames": { + "type": "array", + "items": { + "type": "string", + "maxLength": 253, + "pattern": "^(\\*\\.)?[a-z0-9]([a-z0-9.-]*[a-z0-9])?$" + } + }, + "timeouts": { + "type": "object", + "additionalProperties": false, + "properties": { + "request": { + "type": "string", + "minLength": 1 + }, + "backendRequest": { + "type": "string", + "minLength": 1 + } + } + }, + "backendTLS": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "object", + "properties": { + "hostname": { + "type": "string", + "minLength": 1 + }, + "caCertificateRefs": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "group": { + "type": "string" + }, + "kind": { + "type": "string", + "minLength": 1 + }, + "name": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "group", + "kind", + "name" + ] + } + }, + "wellKnownCACertificates": { + "enum": [ + "System" + ] + } + }, + "required": [ + "hostname" + ], + "oneOf": [ + { + "required": [ + "caCertificateRefs" + ], + "not": { + "required": [ + "wellKnownCACertificates" + ] + } + }, + { + "required": [ + "wellKnownCACertificates" + ], + "not": { + "required": [ + "caCertificateRefs" + ] + } + } + ] + } + ] + } + }, + "required": [ + "enabled", + "parentRefs", + "hostnames" + ] + } + } +} diff --git a/deploy/rustfs-tenant/values.yaml b/deploy/rustfs-tenant/values.yaml new file mode 100644 index 0000000..7033874 --- /dev/null +++ b/deploy/rustfs-tenant/values.yaml @@ -0,0 +1,78 @@ +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Install the RustFS Operator and CRD separately before using this chart. +# This default pool is for development, not a highly available deployment. +tenant: + metadata: + # Defaults to the Helm release name. Keep this stable across upgrades. + name: "" + labels: {} + annotations: {} + # Native Tenant.spec, passed through without tpl evaluation or env injection. + # The referenced Secret must already exist in the release namespace. + spec: + credsSecret: + name: rustfs-credentials + pools: + - name: pool-0 + servers: 1 + persistence: + volumesPerServer: 1 + volumeClaimTemplate: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 10Gi + +# Ingress and HTTPRoute are alternative frontends for each endpoint. +# All paths are / (Prefix), with no Host or URI rewriting. +ingress: + api: + enabled: false + labels: {} + annotations: {} + ingressClassName: "" + hosts: [] + # Native Ingress.spec.tls entries; Secrets are externally managed. + tls: [] + console: + enabled: false + labels: {} + annotations: {} + ingressClassName: "" + hosts: [] + tls: [] + +httpRoute: + api: + enabled: false + labels: {} + annotations: {} + # Native HTTPRoute.spec.parentRefs. Reference an existing Gateway. + parentRefs: [] + hostnames: [] + # Native HTTPRoute rule timeouts; support depends on the Gateway controller. + timeouts: {} + # Optional native BackendTLSPolicy.spec.validation (Gateway API v1.4+). + # Set when the backend uses HTTPS; never disables certificate verification. + backendTLS: null + console: + enabled: false + labels: {} + annotations: {} + parentRefs: [] + hostnames: [] + timeouts: {} + backendTLS: null diff --git a/docs/operator-user-guide.md b/docs/operator-user-guide.md index bb00d17..1f40b9d 100644 --- a/docs/operator-user-guide.md +++ b/docs/operator-user-guide.md @@ -20,6 +20,10 @@ This guide is a technical manual for installing, configuring, and operating the Chinese version: [operator-user-guide.zh-CN.md](operator-user-guide.zh-CN.md) +For Tenant external access, see [Tenant networking](tenant-networking.md): native +Kubernetes YAML/Kustomize and an optional separate Tenant Helm chart. The existing +Tenant CRD is unchanged; these routes are managed by your deployment tooling. + ## 1. Overview RustFS Operator manages RustFS object storage clusters on Kubernetes. Users describe the desired storage cluster with a namespaced `Tenant` custom resource, and the operator reconciles Kubernetes resources needed to run RustFS. diff --git a/docs/tenant-networking.md b/docs/tenant-networking.md new file mode 100644 index 0000000..ab90b9c --- /dev/null +++ b/docs/tenant-networking.md @@ -0,0 +1,271 @@ + + +# Tenant networking: Kubernetes YAML and Helm + +The Operator creates `-io:9000` for S3 and `-console:9001` for the +Tenant Console. Attach your ingress or gateway to these ClusterIP Services, not +the internal headless Service. All pools of a Tenant belong to one cluster and +share these endpoints. Existing Tenant YAML and Operator installations are unchanged. + +The optional [Tenant chart](../deploy/rustfs-tenant/README.md) packages a Tenant +and networking into one release. Networking is not part of the Tenant CRD and +is not reconciled by the Operator. Both deployment methods below use the same +Service contract. Choose **one owner** for each resource: Helm or YAML/GitOps. + +## Prerequisites + +- An installed RustFS Operator and `rustfs.com/v1alpha1` Tenant CRD. +- A namespace (`rustfs-storage` in these examples) and an existing credential + Secret named `rustfs-credentials` with UTF-8 `accesskey` and `secretkey`, each at + least 8 characters. Provision secrets through your normal secret management. +- A storage class and a tested RustFS image suitable for your deployment. The + examples use a single-node/single-disk **development** pool. Choose an appropriate + HA pool layout and capacity for production. +- For Ingress: an installed controller / IngressClass and edge TLS Secret. +- For HTTPRoute: Gateway API `v1` CRDs, a controller, and an existing Gateway with + an HTTPS listener and certificate covering the requested domains. +- DNS pointing the public names at the chosen ingress/gateway address. + +Create the namespace before provisioning the Secret: + +```bash +kubectl create namespace rustfs-storage +``` + +## Apply ordinary Kubernetes YAML + +Copy and edit [the base Tenant](../examples/networking/base/tenant.yaml), including +namespace, name and pool settings. Then choose either +[Ingress](../examples/networking/ingress/ingress.yaml) or +[HTTPRoute](../examples/networking/gateway/httproute.yaml). The manifests contain +normal `apiVersion`, `kind`, `metadata` and `spec` fields with no Helm expressions. +If you rename the Tenant, update every route backend reference as well. + +```bash +kubectl apply -f examples/networking/base/tenant.yaml + +# Choose one: +kubectl apply -f examples/networking/ingress/ingress.yaml +# kubectl apply -f examples/networking/gateway/httproute.yaml +``` + +Or apply the equivalent Kustomize overlay after editing it: + +```bash +kubectl apply -k examples/networking/ingress +# Alternative: kubectl apply -k examples/networking/gateway +``` + +The base creates no namespace, Secret, Gateway, or controller. Changing between +overlays does not prune the old overlay's resources with plain `kubectl apply`. +Delete obsolete Ingress/HTTPRoute objects explicitly after a planned cutover, or +use your GitOps tool's pruning configuration. Do not run both alternatives against +the same public endpoint inadvertently. + +An HTTPRoute backend is an ordinary Service reference: + +```yaml +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: rustfs-api + namespace: rustfs-storage +spec: + parentRefs: + - name: shared-gateway + namespace: gateway-system + sectionName: https + hostnames: + - s3.example.com + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: rustfs-io + port: 9000 +``` + +The Gateway owner must allow the route's namespace through the listener's +`allowedRoutes`. A cross-namespace Gateway parent reference uses this attachment +permission, not ReferenceGrant. Route-to-Service references here stay in the Tenant +namespace. The chart never changes the shared Gateway's listeners or permissions. +See [Gateway API cross-namespace routing](https://gateway-api.sigs.k8s.io/guides/multiple-ns/). + +## Use one Helm values file + +Copy [values.yaml](../deploy/rustfs-tenant/values.yaml), put your existing Tenant +`spec` under `tenant.spec`, and configure `ingress` or `httpRoute` in the same file. +`tenant.metadata` accepts name, labels and annotations; namespace is the release +namespace. These wrapper keys belong to Helm only, not to the Kubernetes Tenant. + +```bash +helm upgrade --install rustfs ./deploy/rustfs-tenant \ + --namespace rustfs-storage --values tenant-values.yaml +``` + +To review/render regular Kubernetes manifests without installing a Helm release: + +```bash +helm template rustfs ./deploy/rustfs-tenant \ + --namespace rustfs-storage --values tenant-values.yaml > tenant-stack.yaml +kubectl apply --dry-run=server -f tenant-stack.yaml +kubectl apply -f tenant-stack.yaml +``` + +In this mode there is **no Helm release** to upgrade/uninstall. Track the rendered +resources with YAML/GitOps; removing a document from a file does not delete its +previously applied resource. Direct `kubectl delete tenant` also leaves the +separately managed routes behind. + +## S3 and Console routing + +Prefer separate domains, e.g. `s3.example.com` and `console.example.com`, both +forwarding `/` unchanged. Preserve the original Host, path encoding and query +parameters: they participate in S3 signing. Do not mount S3 behind `/s3` with a +rewrite or put a browser-login redirect in front of the S3 endpoint. Configure +body-size limits, streaming/buffering and timeouts for your object sizes using +the chosen controller's documented policies; these are not portable Ingress settings. + +The Console domain forwards to port 9001, including its UI and management API +paths. Do not forward only the static `/rustfs/console` subtree. If your image +supports a custom `RUSTFS_CONSOLE_PREFIX`, configure it through `Tenant.spec.env`; +it does not relocate all admin endpoints. Do not change the listener ports away +from the Operator Service contract. + +For browser/OIDC deployments, supported RustFS images accept +`RUSTFS_BROWSER_REDIRECT_URL=https://console.example.com` (origin only, no path). +Use `spec.env` in YAML or `tenant.spec.env` in chart values. Validate the OIDC +provider callback and proxy headers. Node-local in-flight OIDC state may require +controller-specific session affinity; an external redirect URL alone does not +provide affinity. The chart does not silently inject env vars or affinity policies. + +### Path-style and virtual-hosted-style S3 + +Path-style (`https://s3.example.com/bucket/key`) uses the base route examples. +Configure your SDK/client to use path-style addressing when needed. + +For `https://bucket.s3.example.com/key`, configure all of: + +1. A RustFS image supporting `RUSTFS_SERVER_DOMAINS`, set to `s3.example.com`. +2. Both `s3.example.com` and `*.s3.example.com` in HTTPRoute `hostnames`, or separate + Ingress `rules` (chart values: `ingress.api.hosts`). +3. DNS and edge certificates covering the base and wildcard domain. A wildcard + certificate does not cover the base domain or arbitrary multi-label bucket names. + +The chart [virtual-hosted overlay](../deploy/rustfs-tenant/examples/virtual-hosted.yaml) +can be layered over its Gateway example. Merge any other required env entries: +Helm replaces the entire `env` list. For plain YAML, edit `spec.env` and route +hostnames directly; no Operator feature switch is required. + +## TLS termination and backend encryption + +Public HTTPS and backend TLS are separate connections. The default examples +terminate TLS at the edge and forward HTTP to the Tenant. Configure network +isolation appropriate to your environment; use backend encryption when required. + +For encrypted Gateway-to-RustFS connections: + +1. Configure the existing Tenant `spec.tls` and certificate/CA resources. The + [chart TLS overlay](../deploy/rustfs-tenant/examples/backend-tls.yaml) shows + cert-manager with an existing Issuer. Keep the generated internal SANs and + include the validation names for both Services. Adjust cluster domain, namespace + and Tenant name together. The Operator remains responsible for this existing + Tenant TLS lifecycle. +2. Supply a trusted CA ConfigMap with key `ca.crt` in the Tenant namespace, using + the CA that actually issued the backend certificate. Do not disable verification. +3. Use a Gateway implementation supporting `BackendTLSPolicy` and install Gateway + API v1.4+ CRDs (the policy uses `gateway.networking.k8s.io/v1`). Set + `httpRoute.api.backendTLS` and `httpRoute.console.backendTLS`, or apply the + [plain policies](../examples/networking/gateway/backend-tls.yaml) after editing. + Both RustFS listeners inherit the Tenant TLS settings; configure both exposed + backends accordingly. Do not infer HTTP/HTTPS solely from a Service port's name. + +```bash +helm template rustfs ./deploy/rustfs-tenant -n rustfs-storage \ + -f deploy/rustfs-tenant/examples/gateway.yaml \ + -f deploy/rustfs-tenant/examples/backend-tls.yaml +``` + +`backendTLS` is the native policy `spec.validation` object (hostname and CA +references or `wellKnownCACertificates: System`). It configures the gateway, not +the RustFS listener. Explicit CA references are recommended for private PKI; +controller support for system CAs varies. See [Gateway TLS configuration](https://gateway-api.sigs.k8s.io/guides/tls/). + +For Ingress with an HTTPS backend, set the controller-specific upstream protocol, +CA trust and TLS verification name through annotations or separately managed policy +resources. Merely setting `ingress.*.tls` only configures edge TLS. There is no +portable Ingress field that replaces BackendTLSPolicy. Keep controller-specific +resources outside the storage Operator. + +## Verify the data path + +First validate and inspect Kubernetes resources: + +```bash +kubectl apply --dry-run=server -f tenant-stack.yaml +kubectl -n rustfs-storage get tenants,pods,svc +kubectl -n rustfs-storage describe ingress rustfs-api +# Gateway alternative: +kubectl -n rustfs-storage get httproute rustfs-api -o yaml +kubectl -n rustfs-storage get httproute rustfs-console -o yaml +``` + +For HTTPRoute, check the intended parent/listener reports `Accepted=True` and +`ResolvedRefs=True` for the current generation; also inspect Gateway and any +BackendTLSPolicy conditions. This does not by itself prove DNS or TLS reachability. +Use the command appropriate to your chosen Ingress/Gateway deployment. + +With an AWS CLI profile containing test credentials, verify against a disposable +bucket (choose a unique name and configure the intended addressing style): + +```bash +aws --profile rustfs --endpoint-url https://s3.example.com s3 mb s3://network-check-unique +printf 'network check\n' > /tmp/rustfs-network-check.txt +aws --profile rustfs --endpoint-url https://s3.example.com s3 cp \ + /tmp/rustfs-network-check.txt s3://network-check-unique/probe.txt +aws --profile rustfs --endpoint-url https://s3.example.com s3 cp \ + s3://network-check-unique/probe.txt /tmp/rustfs-network-check-downloaded.txt +cmp /tmp/rustfs-network-check.txt /tmp/rustfs-network-check-downloaded.txt +aws --profile rustfs --endpoint-url https://s3.example.com s3 presign \ + s3://network-check-unique/probe.txt --expires-in 60 +``` + +Download the returned presigned URL and compare the contents. Then test a large +multipart upload/download using your workload's part sizes and timeouts, and +verify checksums. Repeat with virtual-hosted addressing if enabled. Verify the +Console login and, when used, the complete OIDC callback flow through the public +domain. Remove the disposable objects and bucket afterward. + +The chart contract tests validate rendered resource boundaries, Service references, +TLS policy targets and invalid configurations. They do not replace these runtime +tests. Helm `--wait` is not a guarantee of Tenant or external endpoint readiness. + +## Compatibility and lifecycle + +Existing Tenant YAML, Operator chart values, CRD fields, RBAC and controller +behavior are unchanged. Default chart installation creates no external route and +no Gateway API resource, so Gateway CRDs are not required for internal-only or +Ingress deployments. Plain YAML remains a first-class option. + +Keep existing resources under their existing owner. This chart does not implement +in-place adoption, renaming, data migration or seamless ingress-controller cutovers. +Do not use `helm --take-ownership` as a migration shortcut. Route ownership follows +the Helm release / GitOps application, not Tenant garbage collection. See the +[chart lifecycle instructions](../deploy/rustfs-tenant/README.md#upgrade-and-remove) +before disabling networking or uninstalling a release. diff --git a/examples/README.md b/examples/README.md index 3173cb0..b285cca 100755 --- a/examples/README.md +++ b/examples/README.md @@ -2,6 +2,13 @@ This directory contains example Tenant configurations for the RustFS Kubernetes Operator, covering various use cases from development to production. +## Tenant networking + +[Networking examples](../docs/tenant-networking.md) provide ordinary Tenant, Ingress +and HTTPRoute YAML, plus Kustomize overlays. They reference existing platform +Gateways/certificates and do not require Helm. For one Helm release, see the +[Tenant chart](../deploy/rustfs-tenant/README.md). + ## Quick Start Guide **Choose the right example for your needs:** diff --git a/examples/networking/base/kustomization.yaml b/examples/networking/base/kustomization.yaml new file mode 100644 index 0000000..f7e8346 --- /dev/null +++ b/examples/networking/base/kustomization.yaml @@ -0,0 +1,18 @@ +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - tenant.yaml diff --git a/examples/networking/base/tenant.yaml b/examples/networking/base/tenant.yaml new file mode 100644 index 0000000..9d95a78 --- /dev/null +++ b/examples/networking/base/tenant.yaml @@ -0,0 +1,33 @@ +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Development layout. Create the namespace and credential Secret first. +apiVersion: rustfs.com/v1alpha1 +kind: Tenant +metadata: + name: rustfs + namespace: rustfs-storage +spec: + credsSecret: + name: rustfs-credentials + pools: + - name: pool-0 + servers: 1 + persistence: + volumesPerServer: 1 + volumeClaimTemplate: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 10Gi diff --git a/examples/networking/gateway/backend-tls.yaml b/examples/networking/gateway/backend-tls.yaml new file mode 100644 index 0000000..e23f9b3 --- /dev/null +++ b/examples/networking/gateway/backend-tls.yaml @@ -0,0 +1,51 @@ +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Optional companion to httproute.yaml, NOT included in the default overlay. +# First configure Tenant.spec.tls (see chart examples/backend-tls.yaml). +# Both listeners use the Tenant TLS configuration. The CA ConfigMap must exist +# in rustfs-storage and contain ca.crt; the certificate must cover each hostname. +apiVersion: gateway.networking.k8s.io/v1 +kind: BackendTLSPolicy +metadata: + name: rustfs-api + namespace: rustfs-storage +spec: + targetRefs: + - group: "" + kind: Service + name: rustfs-io + validation: + hostname: rustfs-io.rustfs-storage.svc.cluster.local + caCertificateRefs: + - group: "" + kind: ConfigMap + name: rustfs-backend-ca +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: BackendTLSPolicy +metadata: + name: rustfs-console + namespace: rustfs-storage +spec: + targetRefs: + - group: "" + kind: Service + name: rustfs-console + validation: + hostname: rustfs-console.rustfs-storage.svc.cluster.local + caCertificateRefs: + - group: "" + kind: ConfigMap + name: rustfs-backend-ca diff --git a/examples/networking/gateway/httproute.yaml b/examples/networking/gateway/httproute.yaml new file mode 100644 index 0000000..5bed7d4 --- /dev/null +++ b/examples/networking/gateway/httproute.yaml @@ -0,0 +1,57 @@ +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: rustfs-api + namespace: rustfs-storage +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: shared-gateway + namespace: gateway-system + sectionName: https + hostnames: [s3.example.com] + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: rustfs-io + port: 9000 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: rustfs-console + namespace: rustfs-storage +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: shared-gateway + namespace: gateway-system + sectionName: https + hostnames: [console.example.com] + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: rustfs-console + port: 9001 diff --git a/examples/networking/gateway/kustomization.yaml b/examples/networking/gateway/kustomization.yaml new file mode 100644 index 0000000..19fafe8 --- /dev/null +++ b/examples/networking/gateway/kustomization.yaml @@ -0,0 +1,19 @@ +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../base + - httproute.yaml diff --git a/examples/networking/ingress/ingress.yaml b/examples/networking/ingress/ingress.yaml new file mode 100644 index 0000000..9a4a397 --- /dev/null +++ b/examples/networking/ingress/ingress.yaml @@ -0,0 +1,59 @@ +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: rustfs-api + namespace: rustfs-storage +spec: + # Replace with an installed IngressClass. + ingressClassName: traefik + tls: + - hosts: [s3.example.com] + secretName: rustfs-edge-tls + rules: + - host: s3.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: rustfs-io + port: + number: 9000 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: rustfs-console + namespace: rustfs-storage +spec: + # Replace with an installed IngressClass. + ingressClassName: traefik + tls: + - hosts: [console.example.com] + secretName: rustfs-edge-tls + rules: + - host: console.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: rustfs-console + port: + number: 9001 diff --git a/examples/networking/ingress/kustomization.yaml b/examples/networking/ingress/kustomization.yaml new file mode 100644 index 0000000..7f53f4d --- /dev/null +++ b/examples/networking/ingress/kustomization.yaml @@ -0,0 +1,19 @@ +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../base + - ingress.yaml diff --git a/scripts/check-release-metadata.sh b/scripts/check-release-metadata.sh index 5e2faff..7310681 100755 --- a/scripts/check-release-metadata.sh +++ b/scripts/check-release-metadata.sh @@ -121,6 +121,10 @@ if [[ ! "$cargo_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+([-.][0-9A-Za-z.-]+)?$ ]]; th exit 1 fi +tenant_chart_version="$(awk '$1 == "version:" { gsub(/"/, "", $2); print $2; exit }' deploy/rustfs-tenant/Chart.yaml)" +require_value "Tenant chart version" "$tenant_chart_version" +require_equal "Tenant chart version" "$tenant_chart_version" "$cargo_version" + require_equal "Helm chart version" "$chart_version" "$cargo_version" require_equal "Helm appVersion" "$app_version" "$cargo_version" require_equal "setup action Rust version" "$setup_version" "$toolchain_version" diff --git a/tests/tenant_chart.rs b/tests/tenant_chart.rs new file mode 100644 index 0000000..ab1cd16 --- /dev/null +++ b/tests/tenant_chart.rs @@ -0,0 +1,359 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Deployment contracts: render the public values interface and compare backends +//! with Services generated by the real Tenant implementation. Helm is required. +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use operator::types::v1alpha1::tenant::Tenant; +use serde::Deserialize; +use serde_yaml_ng::Value; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Output, Stdio}; + +fn render(values: &str, files: &[&str]) -> Output { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut command = Command::new("helm"); + command.current_dir(root).args([ + "template", + "rustfs", + "deploy/rustfs-tenant", + "--namespace", + "rustfs-storage", + ]); + for file in files { + command.args(["--values", file]); + } + let mut child = command + .args(["--values", "-"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("install Helm 3 to run Tenant chart contract tests"); + child + .stdin + .take() + .unwrap() + .write_all(values.as_bytes()) + .unwrap(); + child.wait_with_output().unwrap() +} + +fn documents(output: Output) -> Vec { + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + serde_yaml_ng::Deserializer::from_slice(&output.stdout) + .map(|doc| Value::deserialize(doc).unwrap()) + .filter(|doc| !doc.is_null()) + .collect() +} + +fn tenant(docs: &[Value]) -> Tenant { + let doc = docs.iter().find(|doc| doc["kind"] == "Tenant").unwrap(); + let tenant: Tenant = serde_yaml_ng::from_value(doc.clone()).unwrap(); + tenant.validate_name().unwrap(); + tenant.validate_pools().unwrap(); + tenant +} + +fn assert_resource_boundary(docs: &[Value]) { + for doc in docs { + assert!( + ["Tenant", "Ingress", "HTTPRoute", "BackendTLSPolicy"] + .contains(&doc["kind"].as_str().unwrap()) + ); + assert_eq!(doc["metadata"]["namespace"], "rustfs-storage"); + assert!(doc["metadata"]["ownerReferences"].is_null()); + } +} + +#[test] +fn default_chart_only_creates_a_tenant() { + let docs = documents(render("{}", &[])); + assert_eq!(docs.len(), 1); + let tenant = tenant(&docs); + assert_eq!(tenant.metadata.name.as_deref(), Some("rustfs")); + assert_eq!(tenant.spec.creds_secret.unwrap().name, "rustfs-credentials"); + assert_resource_boundary(&docs); +} + +#[test] +fn native_tenant_fields_and_metadata_survive_without_template_evaluation() { + let docs = documents(render( + r#" +tenant: + metadata: + name: warehouse + labels: {team: storage} + annotations: {literal: '{{ .Release.Name }}'} + spec: + image: registry.example.com/rustfs:tested + env: + - name: RUSTFS_SERVER_DOMAINS + value: s3.example.com + - name: CUSTOM_TOKEN + valueFrom: + secretKeyRef: {name: application-token, key: token} + network: + ipFamilyPolicy: PreferDualStack + ipFamilies: [IPv4, IPv6] + pools: + - name: primary + servers: 4 + persistence: + volumesPerServer: 4 +"#, + &["deploy/rustfs-tenant/examples/ingress.yaml"], + )); + let tenant = tenant(&docs); + assert_eq!(tenant.metadata.name.as_deref(), Some("warehouse")); + assert_eq!( + tenant.metadata.annotations.as_ref().unwrap()["literal"], + "{{ .Release.Name }}" + ); + assert_eq!(tenant.spec.env.len(), 2); + assert!(tenant.spec.env[1].value_from.is_some()); + assert_eq!(tenant.spec.pools[0].servers, 4); + assert!(tenant.spec.network.is_some()); + let ingress = docs + .iter() + .find(|d| d["metadata"]["name"] == "warehouse-api") + .unwrap(); + assert_eq!( + ingress["spec"]["rules"][0]["http"]["paths"][0]["backend"]["service"]["name"], + "warehouse-io" + ); + assert_resource_boundary(&docs); +} + +#[test] +fn ingress_backends_match_operator_services_and_preserve_tls() { + let docs = documents(render( + "{}", + &["deploy/rustfs-tenant/examples/ingress.yaml"], + )); + assert_eq!(docs.len(), 3); + let tenant = tenant(&docs); + for (endpoint, service) in [ + ("api", tenant.new_io_service()), + ("console", tenant.new_console_service()), + ] { + let name = format!("rustfs-{endpoint}"); + let ingress = docs.iter().find(|d| d["metadata"]["name"] == name).unwrap(); + let path = &ingress["spec"]["rules"][0]["http"]["paths"][0]; + assert_eq!(path["path"], "/"); + assert_eq!(path["pathType"], "Prefix"); + assert_eq!( + path["backend"]["service"]["name"], + service.metadata.name.unwrap() + ); + assert_eq!( + path["backend"]["service"]["port"]["number"] + .as_i64() + .unwrap(), + i64::from(service.spec.unwrap().ports.unwrap()[0].port) + ); + assert_eq!(ingress["spec"]["tls"][0]["secretName"], "rustfs-edge-tls"); + } + assert_resource_boundary(&docs); +} + +#[test] +fn gateway_preserves_parent_refs_timeouts_and_virtual_hostnames() { + let docs = documents(render( + r#" +httpRoute: + api: + timeouts: {request: 600s, backendRequest: 600s} +"#, + &[ + "deploy/rustfs-tenant/examples/gateway.yaml", + "deploy/rustfs-tenant/examples/virtual-hosted.yaml", + ], + )); + assert_eq!(docs.len(), 3); + let tenant = tenant(&docs); + assert_eq!(tenant.spec.env[0].value.as_deref(), Some("s3.example.com")); + for (endpoint, service) in [ + ("api", tenant.new_io_service()), + ("console", tenant.new_console_service()), + ] { + let name = format!("rustfs-{endpoint}"); + let route = docs.iter().find(|d| d["metadata"]["name"] == name).unwrap(); + let parent = &route["spec"]["parentRefs"][0]; + assert_eq!(parent["namespace"], "gateway-system"); + assert_eq!(parent["sectionName"], "https"); + assert_eq!(parent["kind"], "Gateway"); + let rule = &route["spec"]["rules"][0]; + assert!( + rule["filters"].is_null(), + "S3 signatures require unmodified paths and Host" + ); + assert_eq!(rule["matches"][0]["path"]["value"], "/"); + assert_eq!( + rule["backendRefs"][0]["name"], + service.metadata.name.unwrap() + ); + assert_eq!( + rule["backendRefs"][0]["port"].as_i64().unwrap(), + i64::from(service.spec.unwrap().ports.unwrap()[0].port) + ); + if endpoint == "api" { + assert_eq!(route["spec"]["hostnames"][1], "*.s3.example.com"); + assert_eq!(rule["timeouts"]["request"], "600s"); + } + } + assert_resource_boundary(&docs); +} + +#[test] +fn backend_tls_policies_target_each_service_with_explicit_trust() { + let docs = documents(render( + "{}", + &[ + "deploy/rustfs-tenant/examples/gateway.yaml", + "deploy/rustfs-tenant/examples/backend-tls.yaml", + ], + )); + assert_eq!(docs.len(), 5); + assert!(tenant(&docs).spec.tls.is_some()); + let policies: Vec<_> = docs + .iter() + .filter(|d| d["kind"] == "BackendTLSPolicy") + .collect(); + assert_eq!(policies.len(), 2); + for policy in policies { + let service = policy["spec"]["targetRefs"][0]["name"].as_str().unwrap(); + assert!(["rustfs-io", "rustfs-console"].contains(&service)); + assert_eq!( + policy["spec"]["validation"]["hostname"], + format!("{service}.rustfs-storage.svc.cluster.local") + ); + assert_eq!( + policy["spec"]["validation"]["caCertificateRefs"][0]["name"], + "rustfs-backend-ca" + ); + } + assert_resource_boundary(&docs); +} + +#[test] +fn endpoints_can_be_disabled_without_leaving_route_or_policy_resources() { + let docs = documents(render( + r#" +httpRoute: + api: + enabled: false + console: + enabled: false +"#, + &[ + "deploy/rustfs-tenant/examples/gateway.yaml", + "deploy/rustfs-tenant/examples/backend-tls.yaml", + ], + )); + assert_eq!(docs.len(), 1); + assert_eq!(docs[0]["kind"], "Tenant"); +} + +#[test] +fn endpoints_can_choose_different_frontends_and_preserve_native_metadata() { + let docs = documents(render( + r#" +ingress: + api: + enabled: true + hosts: [s3.example.com, "*.s3.example.com"] + labels: {team: storage} + annotations: {example.com/policy: streaming} +httpRoute: + console: + enabled: true + parentRefs: [{name: shared, sectionName: https, port: 443}] + hostnames: [console.example.com] +"#, + &[], + )); + assert_eq!(docs.len(), 3); + let ingress = docs.iter().find(|d| d["kind"] == "Ingress").unwrap(); + assert_eq!(ingress["metadata"]["labels"]["team"], "storage"); + assert_eq!( + ingress["metadata"]["annotations"]["example.com/policy"], + "streaming" + ); + assert_eq!(ingress["spec"]["rules"][1]["host"], "*.s3.example.com"); + let route = docs.iter().find(|d| d["kind"] == "HTTPRoute").unwrap(); + assert_eq!(route["metadata"]["name"], "rustfs-console"); + assert_eq!(route["spec"]["parentRefs"][0]["port"].as_i64(), Some(443)); + assert_resource_boundary(&docs); +} + +#[test] +fn invalid_networking_values_fail_offline_instead_of_silently_exposing() { + for values in [ + "ingress: {api: {enabled: true}}", + "httpRoute: {api: {enabled: true}}", + "ingress: {api: {enabled: true, hosts: [s3.example.com]}}\nhttpRoute: {api: {enabled: true}}", + "tenant: {metadata: {name: 9invalid}}", + "tenant: {spec: {pools: []}}", + "httpRoute: {api: {enabled: true, parentRefs: [{name: shared, kind: Service}], hostnames: [s3.example.com]}}", + "httpRoute: {api: {backendTLS: {hostname: s3.example.com}}}", + "ingress: {api: {enable: true}}", + ] { + let output = render(values, &[]); + assert!( + !output.status.success(), + "invalid values unexpectedly rendered: {values}" + ); + assert!(!output.stderr.is_empty()); + } +} + +#[test] +fn plain_yaml_examples_use_the_same_service_contract() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let base = std::fs::read_to_string(root.join("examples/networking/base/tenant.yaml")).unwrap(); + let tenant: Tenant = serde_yaml_ng::from_str(&base).unwrap(); + tenant.validate_pools().unwrap(); + for file in [ + "ingress/ingress.yaml", + "gateway/httproute.yaml", + "gateway/backend-tls.yaml", + ] { + let yaml = std::fs::read_to_string(root.join("examples/networking").join(file)).unwrap(); + for doc in serde_yaml_ng::Deserializer::from_str(&yaml) { + let value = Value::deserialize(doc).unwrap(); + let name = value["metadata"]["name"].as_str().unwrap(); + let service = if name == "rustfs-api" { + tenant.new_io_service() + } else { + tenant.new_console_service() + }; + let target = match value["kind"].as_str().unwrap() { + "Ingress" => { + &value["spec"]["rules"][0]["http"]["paths"][0]["backend"]["service"]["name"] + } + "HTTPRoute" => &value["spec"]["rules"][0]["backendRefs"][0]["name"], + "BackendTLSPolicy" => &value["spec"]["targetRefs"][0]["name"], + other => panic!("unexpected example resource: {other}"), + }; + assert_eq!(target.as_str(), service.metadata.name.as_deref()); + } + } +}