fix(storage): 支持 MinIO 集群容量查询

docs-site/docs/features/storage-backends.md

@@ -27,7 +27,11 @@ New local-disk targets enable **Relay remote backups through Master** by default
 
 Physical storage usage is shown only when the backend returns total capacity and used bytes. S3-compatible storage, including MinIO, does not provide this through the standard S3 API or rclone About capability. Missing capacity is not treated as zero.
 
-For these targets, BackupX displays the size tracked by backup records. Set a **backup quota** on the storage target to display a backup quota percentage. This soft quota is enforced by BackupX; it is not the bucket quota or the total disk capacity of the MinIO cluster. The tracked size does not include objects created by other applications or untracked object versions. Use the MinIO console or its monitoring interfaces for physical cluster and bucket capacity.
+For these targets, BackupX displays the size tracked by backup records. Set a **backup quota** on the storage target to display a backup quota percentage. This soft quota is enforced by BackupX; it is not the bucket quota or the total disk capacity of the MinIO cluster. The tracked size does not include objects created by other applications or untracked object versions. For MinIO, enable **Query MinIO cluster capacity** on the S3 target and supply the monitoring `bearer_token` generated by `mc admin prometheus generate <alias>`. The token is optional only for an existing anonymous metrics endpoint. BackupX reads `/minio/v2/metrics/cluster` on the configured S3 Endpoint, including any reverse-proxy path prefix. The page explicitly displays the entire cluster’s physical capacity, calculated from its raw total and free bytes; it is not bucket usage or the usable capacity after erasure coding.
+
+The monitoring token is encrypted and masked like other secrets, and remains on the Master when backup or restore configurations are sent to Agents. Capacity queries have a timeout and reject redirects. If permission is denied, the endpoint is unavailable, or metrics are invalid, the page reports a query failure rather than zero capacity. Ordinary S3 targets retain the backup-record/quota display unless this MinIO option is enabled. Bucket usage and bucket quotas still require MinIO’s console or other monitoring tools.
+
+See the [MinIO metrics reference](https://github.com/minio/minio/blob/master/docs/metrics/prometheus/list.md) and [monitoring authentication guide](https://github.com/minio/minio/blob/master/docs/metrics/prometheus/README.md).
 
 ## Rclone backends
 

docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/features/storage-backends.md

@@ -27,7 +27,11 @@ BackupX 的目标是接入任何你想放置备份文件的地方。
 
 只有后端返回总容量和已用字节时，才显示真实存储空间使用率。包括 MinIO 在内的 S3 兼容存储不通过标准 S3 API 或 rclone About 能力提供这些数据；未获取到容量不代表容量为零。
 
-此类目标会显示备份记录统计的大小。可在存储目标中设置 **备份配额**，显示备份配额使用率。这是 BackupX 执行的软限额，不是桶配额，也不是 MinIO 集群磁盘总容量；统计不包含其他应用写入的对象或未被记录的对象版本。集群物理容量与桶容量请使用 MinIO 控制台或监控接口查询。
+此类目标会显示备份记录统计的大小。可在存储目标中设置 **备份配额**，显示备份配额使用率。这是 BackupX 执行的软限额，不是桶配额，也不是 MinIO 集群磁盘总容量；统计不包含其他应用写入的对象或未被记录的对象版本。使用 MinIO 时，可在 S3 存储目标中开启 **查询 MinIO 集群容量**，填写 `mc admin prometheus generate <alias>` 生成的监控 `bearer_token`。只有监控接口已允许匿名访问时才可留空。BackupX 会访问所配置 S3 Endpoint 下的 `/minio/v2/metrics/cluster`，保留反向代理的路径前缀。页面明确标为整个集群的物理容量，使用原始总容量与剩余容量计算；不代表当前桶用量，也不代表扣除纠删码开销后的可用容量。
+
+监控 Token 会加密保存并脱敏展示，仅留在 Master，不随备份或恢复配置下发给 Agent。容量请求有超时限制，并拒绝重定向。权限不足、接口不可达或指标无效时，页面会提示查询失败，不会显示为零容量。未开启此选项的普通 S3 目标仍显示备份记录大小及配额。桶用量和桶配额仍需使用 MinIO 控制台或其他监控工具查询。
+
+接口与鉴权见 [MinIO 指标参考](https://github.com/minio/minio/blob/master/docs/metrics/prometheus/list.md) 和 [监控鉴权说明](https://github.com/minio/minio/blob/master/docs/metrics/prometheus/README.md)。
 
 ## Rclone 后端
 

server/go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/natefinch/lumberjack v2.0.0+incompatible
 	github.com/pquerna/otp v1.5.0
 	github.com/prometheus/client_golang v1.24.1
+	github.com/prometheus/common v0.70.1
 	github.com/rclone/rclone v1.75.0
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/shirou/gopsutil/v4 v4.26.6
@@ -190,7 +191,6 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
-	github.com/prometheus/common v0.70.1 // indirect
 	github.com/prometheus/procfs v0.21.1 // indirect
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
 	github.com/quic-go/qpack v0.6.0 // indirect

server/internal/service/agent_service.go

@@ -239,6 +239,10 @@ func (s *AgentService) GetTaskSpec(ctx context.Context, node *model.Node, taskID
 		if err != nil {
 			return nil, fmt.Errorf("decrypt storage config: %w", err)
 		}
+		configRaw, err = storage.ConfigForAgent(configRaw)
+		if err != nil {
+			return nil, err
+		}
 		transferMode := storage.TransferModeDirect
 		if strings.EqualFold(target.Type, storage.TypeLocalDisk) {
 			var localConfig storage.LocalDiskConfig

server/internal/service/restore_service.go

@@ -647,6 +647,10 @@ func (s *RestoreService) GetAgentRestoreSpec(ctx context.Context, node *model.No
 		if err != nil {
 			return nil, fmt.Errorf("decrypt storage config: %w", err)
 		}
+		configRaw, err = storage.ConfigForAgent(configRaw)
+		if err != nil {
+			return nil, err
+		}
 	}
 	return &AgentRestoreSpec{
 		RestoreRecordID: restore.ID,

server/internal/service/storage_target_service.go

@@ -355,6 +355,9 @@ func (s *StorageTargetService) runCapacityCheckOnce(ctx context.Context, dispatc
 
 func (s *StorageTargetService) dispatchCapacityWarning(ctx context.Context, dispatcher EventDispatcher, target *model.StorageTarget, info *storage.StorageUsageInfo, usage float64) {
 	title := "BackupX 存储容量预警"
+	if info.Scope == "minio_cluster" {
+		title = "BackupX MinIO 集群物理容量预警"
+	}
 	usedGB := float64(*info.Used) / (1 << 30)
 	totalGB := float64(*info.Total) / (1 << 30)
 	body := fmt.Sprintf("存储目标：%s (类型: %s)\n使用率：%.1f%%\n已用：%.2f GB / 总量：%.2f GB\n建议清理旧备份或扩容。",
@@ -367,6 +370,9 @@ func (s *StorageTargetService) dispatchCapacityWarning(ctx context.Context, disp
 		"usedBytes":         *info.Used,
 		"totalBytes":        *info.Total,
 	}
+	if info.Scope != "" {
+		fields["capacityScope"] = info.Scope
+	}
 	_ = dispatcher.DispatchEvent(ctx, model.NotificationEventStorageCapacity, title, body, fields)
 }
 
@@ -738,11 +744,12 @@ func cloneMap(source map[string]any) map[string]any {
 }
 
 type StorageTargetUsage struct {
-	TargetID    uint                      `json:"targetId"`
-	TargetName  string                    `json:"targetName"`
-	RecordCount int64                     `json:"recordCount"`
-	TotalSize   int64                     `json:"totalSize"`
-	DiskUsage   *storage.StorageUsageInfo `json:"diskUsage,omitempty"`
+	TargetID      uint                      `json:"targetId"`
+	TargetName    string                    `json:"targetName"`
+	RecordCount   int64                     `json:"recordCount"`
+	TotalSize     int64                     `json:"totalSize"`
+	DiskUsage     *storage.StorageUsageInfo `json:"diskUsage,omitempty"`
+	CapacityError string                    `json:"capacityError,omitempty"`
 }
 
 func (s *StorageTargetService) GetUsage(ctx context.Context, id uint) (*StorageTargetUsage, error) {
@@ -772,6 +779,8 @@ func (s *StorageTargetService) GetUsage(ctx context.Context, id uint) (*StorageT
 			if abouter, ok := provider.(storage.StorageAbout); ok {
 				if diskUsage, aboutErr := abouter.About(ctx); aboutErr == nil {
 					result.DiskUsage = diskUsage
+				} else if storage.ParseProviderType(target.Type) == storage.TypeS3 && configMap["minioCapacity"] == true {
+					result.CapacityError = "MinIO 容量查询失败，请检查监控 Token、访问权限和网络连接。"
 				}
 			}
 		}

server/internal/service/storage_target_service_test.go

@@ -0,0 +1,150 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"backupx/server/internal/model"
+	"backupx/server/internal/storage"
+	storageRclone "backupx/server/internal/storage/rclone"
+)
+
+func TestStorageTargetMinIOCapacityAndCredentials(t *testing.T) {
+	svc, _, _, _, owner, _ := newAgentServicePoolTestHarness(t)
+	var denied atomic.Bool
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/minio/v2/metrics/cluster" || r.Header.Get("Authorization") != "Bearer monitoring-token" {
+			t.Error("invalid monitoring request")
+		}
+		if denied.Load() {
+			w.WriteHeader(http.StatusForbidden)
+			return
+		}
+		if _, err := fmt.Fprint(w, "# TYPE minio_cluster_capacity_raw_total_bytes gauge\nminio_cluster_capacity_raw_total_bytes 1024\n# TYPE minio_cluster_capacity_raw_free_bytes gauge\nminio_cluster_capacity_raw_free_bytes 128\n"); err != nil {
+			t.Errorf("write metrics: %v", err)
+		}
+	}))
+	defer server.Close()
+	ctx := context.Background()
+	target, err := svc.storageRepo.FindByID(ctx, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	target.Type = storage.TypeS3
+	target.ConfigCiphertext, err = svc.cipher.EncryptJSON(map[string]any{"endpoint": server.URL, "bucket": "backups", "accessKeyId": "backup-key", "secretAccessKey": "backup-secret", "minioCapacity": true, "minioMetricsToken": "monitoring-token"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := svc.storageRepo.Update(ctx, target); err != nil {
+		t.Fatal(err)
+	}
+	registry := storage.NewRegistry(storageRclone.NewS3Factory())
+	targets := NewStorageTargetService(svc.storageRepo, nil, registry, svc.cipher)
+	detail, err := targets.Get(ctx, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if detail.Config["minioMetricsToken"] != "********" {
+		t.Fatal("monitoring token exposed in target details")
+	}
+	usage, err := targets.GetUsage(ctx, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if usage.DiskUsage == nil || *usage.DiskUsage.Total != 1024 || *usage.DiskUsage.Used != 896 || usage.DiskUsage.Scope != "minio_cluster" {
+		t.Fatalf("unexpected usage: %+v", usage)
+	}
+	warning := &minioCapacityWarningCapture{}
+	var mu sync.Mutex
+	targets.runCapacityCheckOnce(ctx, warning, &mu, map[uint]bool{})
+	if warning.title != "BackupX MinIO 集群物理容量预警" || warning.fields["capacityScope"] != "minio_cluster" {
+		t.Fatalf("capacity warning omitted cluster scope: %+v", warning)
+	}
+	denied.Store(true)
+	usage, err = targets.GetUsage(ctx, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if usage.DiskUsage != nil || usage.CapacityError == "" || strings.Contains(usage.CapacityError, "monitoring-token") {
+		t.Fatalf("invalid failure result: %+v", usage)
+	}
+	// 备份下发只能包含 S3 传输凭据，不能带上 Master 的监控权限。
+	spec, err := svc.GetTaskSpec(ctx, owner, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assertTransferConfig := func(raw json.RawMessage) {
+		t.Helper()
+		var config map[string]any
+		if err := json.Unmarshal(raw, &config); err != nil {
+			t.Fatal(err)
+		}
+		if _, ok := config["minioMetricsToken"]; ok {
+			t.Fatal("monitoring token forwarded to Agent")
+		}
+		if _, ok := config["minioCapacity"]; ok {
+			t.Fatal("Agent should not query cluster capacity")
+		}
+		if config["accessKeyId"] != "backup-key" || config["secretAccessKey"] != "backup-secret" {
+			t.Fatal("S3 transfer credentials were lost")
+		}
+	}
+	assertTransferConfig(spec.StorageTargets[0].Config)
+	// 恢复下发使用相同的过滤规则。
+	h := newRestoreTestHarness(t, true)
+	restoreTarget, err := h.service.targets.FindByID(ctx, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	restoreTarget.Type = target.Type
+	var config map[string]any
+	if err := svc.cipher.DecryptJSON(target.ConfigCiphertext, &config); err != nil {
+		t.Fatal(err)
+	}
+	restoreTarget.ConfigCiphertext, err = h.service.cipher.EncryptJSON(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := h.service.targets.Update(ctx, restoreTarget); err != nil {
+		t.Fatal(err)
+	}
+	task, err := h.tasks.FindByID(ctx, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	restoreOwner, err := h.nodes.FindByID(ctx, task.NodeID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	record := &model.BackupRecord{TaskID: task.ID, StorageTargetID: 1, NodeID: restoreOwner.ID, Status: model.BackupRecordStatusSuccess, FileName: "backup.tar.gz", StoragePath: "backup.tar.gz", StartedAt: time.Now().UTC()}
+	if err := h.records.Create(ctx, record); err != nil {
+		t.Fatal(err)
+	}
+	restore := &model.RestoreRecord{BackupRecordID: record.ID, TaskID: task.ID, NodeID: restoreOwner.ID, Status: model.RestoreRecordStatusRunning, StartedAt: time.Now().UTC()}
+	if err := h.restores.Create(ctx, restore); err != nil {
+		t.Fatal(err)
+	}
+	restoreSpec, err := h.service.GetAgentRestoreSpec(ctx, restoreOwner, restore.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assertTransferConfig(restoreSpec.Storage.Config)
+}
+
+type minioCapacityWarningCapture struct {
+	title  string
+	fields map[string]any
+}
+
+func (c *minioCapacityWarningCapture) DispatchEvent(_ context.Context, _, title, _ string, fields map[string]any) error {
+	c.title, c.fields = title, fields
+	return nil
+}

server/internal/storage/agent_config.go

@@ -0,0 +1,21 @@
+package storage
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// ConfigForAgent 保留传输配置，MinIO 监控 Token 和容量查询只留在 Master。
+func ConfigForAgent(raw []byte) ([]byte, error) {
+	var cfg map[string]json.RawMessage
+	if err := json.Unmarshal(raw, &cfg); err != nil {
+		return nil, fmt.Errorf("decode agent storage config: %w", err)
+	}
+	delete(cfg, "minioCapacity")
+	delete(cfg, "minioMetricsToken")
+	result, err := json.Marshal(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("encode agent storage config: %w", err)
+	}
+	return result, nil
+}

server/internal/storage/agent_config_test.go

@@ -0,0 +1,27 @@
+package storage
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestConfigForAgentPreservesTransferFieldsAndRemovesMonitoring(t *testing.T) {
+	raw := []byte(`{"root":"/backups","masterRelay":true,"customNumber":9007199254740993,"secretAccessKey":"transfer-secret","minioCapacity":true,"minioMetricsToken":"monitoring-token"}`)
+	result, err := ConfigForAgent(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var config map[string]json.RawMessage
+	if err := json.Unmarshal(result, &config); err != nil {
+		t.Fatal(err)
+	}
+	if config["minioCapacity"] != nil || config["minioMetricsToken"] != nil {
+		t.Fatal("monitoring configuration was forwarded")
+	}
+	if string(config["root"]) != `"/backups"` || string(config["masterRelay"]) != "true" || string(config["customNumber"]) != "9007199254740993" || string(config["secretAccessKey"]) != `"transfer-secret"` {
+		t.Fatal("transfer configuration changed")
+	}
+	if _, err := ConfigForAgent([]byte("invalid")); err == nil {
+		t.Fatal("invalid storage configuration was accepted")
+	}
+}

server/internal/storage/rclone/factory.go

@@ -66,7 +66,9 @@ type S3Factory struct{}
 func NewS3Factory() S3Factory { return S3Factory{} }
 
 func (S3Factory) Type() storage.ProviderType { return storage.ProviderTypeS3 }
-func (S3Factory) SensitiveFields() []string  { return []string{"accessKeyId", "secretAccessKey"} }
+func (S3Factory) SensitiveFields() []string {
+	return []string{"accessKeyId", "secretAccessKey", "minioMetricsToken"}
+}
 
 func (S3Factory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
 	cfg, err := storage.DecodeConfig[storage.S3Config](rawConfig)
@@ -79,7 +81,20 @@ func (S3Factory) New(ctx context.Context, rawConfig map[string]any) (storage.Sto
 	if strings.TrimSpace(cfg.AccessKeyID) == "" || strings.TrimSpace(cfg.SecretAccessKey) == "" {
 		return nil, fmt.Errorf("s3 credentials are required")
 	}
-	return newFs(ctx, storage.ProviderTypeS3, buildS3Remote("Other", cfg.AccessKeyID, cfg.SecretAccessKey, cfg.Endpoint, cfg.Region, cfg.Bucket, cfg.ForcePathStyle))
+	metricsURL := ""
+	if cfg.MinIOCapacity {
+		metricsURL, err = minioMetricsURL(cfg.Endpoint, cfg.MinIOMetricsToken)
+		if err != nil {
+			return nil, err
+		}
+	}
+	p, err := newFs(ctx, storage.ProviderTypeS3, buildS3Remote("Other", cfg.AccessKeyID, cfg.SecretAccessKey, cfg.Endpoint, cfg.Region, cfg.Bucket, cfg.ForcePathStyle))
+	if err != nil {
+		return nil, err
+	}
+	p.minioMetricsURL = metricsURL
+	p.minioMetricsToken = strings.TrimSpace(cfg.MinIOMetricsToken)
+	return p, nil
 }
 
 // buildS3Remote 构建 S3 兼容存储的 rclone 连接字符串。