feat(observability): attribute dashboard panels per cluster and server

core/cmd/cnpgi/instance.go

@@ -56,6 +56,10 @@ var instanceCmd = &cobra.Command{
 		clusterName, _ := cmd.Flags().GetString("cluster-name")
 		clusterNamespace, _ := cmd.Flags().GetString("cluster-namespace")
 
+		// Seed the backup counter series at 0 so a first failure (or success)
+		// is a visible increment for rate()/increase()-based panels.
+		opentelemetry.InitPluginBackupSeries(cmd.Context(), clusterName)
+
 		capabilities := func(server *cnpgi.CNPGI) {
 			server.AddBackupCapability(cnpgi.BackupCapabilityOptions{
 				Tier2: configuration.Tier2BackupEnabled,

core/internal/cnpgi/backup.go

@@ -117,16 +117,16 @@ func (b backupServiceImplementation) Backup(
 	)
 
 	backupStart := time.Now()
-	recordBackupStart(ctx)
-	defer recordBackupFinished(ctx)
+	recordBackupStart(ctx, cluster.Name)
+	defer recordBackupFinished(ctx, cluster.Name)
 
 	metadata, err := b.runBackup(
 		ctx,
 		backupName,
 		isPrimary,
 	)
 	if err != nil {
-		recordBackupFailure(ctx, time.Since(backupStart), err)
+		recordBackupFailure(ctx, cluster.Name, time.Since(backupStart), err)
 		span.RecordError(err)
 		span.SetStatus(codes.Error, "backup failed")
 
@@ -138,14 +138,14 @@ func (b backupServiceImplementation) Backup(
 	// verification is folded into the successful-backup recording below.
 	corruption, verifyErr := b.runVerify(ctx, backupName)
 	if corruption {
-		recordBackupFailure(ctx, time.Since(backupStart), verifyErr)
+		recordBackupFailure(ctx, cluster.Name, time.Since(backupStart), verifyErr)
 		span.RecordError(verifyErr)
 		span.SetStatus(codes.Error, "verification detected corruption")
 
 		return nil, verifyErr
 	}
 
-	recordBackupSuccess(ctx, time.Since(backupStart))
+	recordBackupSuccess(ctx, cluster.Name, time.Since(backupStart))
 
 	return &backup.BackupResult{
 		BackupName:        backupName,

core/internal/cnpgi/metrics.go

@@ -23,44 +23,63 @@ import (
 	"context"
 	"time"
 
+	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
 
 	"github.com/cloudnative-pg/klio/core/internal/opentelemetry"
 )
 
+// clusterAttr returns the `cluster_name` attribute every plugin backup metric
+// carries, so panels can attribute backup activity to a specific PostgreSQL
+// cluster even when several clusters share one namespace.
+func clusterAttr(clusterName string) attribute.KeyValue {
+	return opentelemetry.AttributeKeyClusterName.Of(clusterName)
+}
+
 // recordBackupStart records that a backup has started. Callers must pair this
 // with a deferred recordBackupFinished so the in-progress counter decrements
 // on every exit path, including panics.
-func recordBackupStart(ctx context.Context) {
-	opentelemetry.PluginBackup.LatestStartTime.Record(ctx, time.Now().Unix())
-	opentelemetry.PluginBackup.InProgress.Add(ctx, 1)
+func recordBackupStart(ctx context.Context, clusterName string) {
+	cluster := clusterAttr(clusterName)
+	opentelemetry.PluginBackup.LatestStartTime.Record(ctx, time.Now().Unix(),
+		metric.WithAttributes(cluster))
+	opentelemetry.PluginBackup.InProgress.Add(ctx, 1, metric.WithAttributes(cluster))
 }
 
 // recordBackupFinished decrements the in-progress counter. Always invoke via
 // defer immediately after recordBackupStart so concurrent backup accounting
-// stays correct even when a backup panics or returns early.
-func recordBackupFinished(ctx context.Context) {
-	opentelemetry.PluginBackup.InProgress.Add(ctx, -1)
+// stays correct even when a backup panics or returns early. It must pass the
+// same clusterName as recordBackupStart so the up/down counter cancels out per
+// cluster.
+func recordBackupFinished(ctx context.Context, clusterName string) {
+	opentelemetry.PluginBackup.InProgress.Add(ctx, -1,
+		metric.WithAttributes(clusterAttr(clusterName)))
 }
 
 // recordBackupSuccess records a successful backup completion.
-func recordBackupSuccess(ctx context.Context, duration time.Duration) {
-	opentelemetry.PluginBackup.LatestCompletionTime.Record(ctx, time.Now().Unix())
-	opentelemetry.PluginBackup.LatestDuration.Record(ctx, duration.Seconds())
+func recordBackupSuccess(ctx context.Context, clusterName string, duration time.Duration) {
+	cluster := clusterAttr(clusterName)
+	opentelemetry.PluginBackup.LatestCompletionTime.Record(ctx, time.Now().Unix(),
+		metric.WithAttributes(cluster))
+	opentelemetry.PluginBackup.LatestDuration.Record(ctx, duration.Seconds(),
+		metric.WithAttributes(cluster))
 	opentelemetry.PluginBackup.Duration.Record(ctx, duration.Seconds(),
-		metric.WithAttributes(opentelemetry.OutcomeSuccess.Attribute()))
+		metric.WithAttributes(cluster, opentelemetry.OutcomeSuccess.Attribute()))
 	opentelemetry.PluginBackup.Runs.Add(ctx, 1,
-		metric.WithAttributes(opentelemetry.OutcomeSuccess.Attribute()))
+		metric.WithAttributes(cluster, opentelemetry.OutcomeSuccess.Attribute()))
 }
 
 // recordBackupFailure records a failed backup.
-func recordBackupFailure(ctx context.Context, duration time.Duration, err error) {
+func recordBackupFailure(ctx context.Context, clusterName string, duration time.Duration, err error) {
+	cluster := clusterAttr(clusterName)
 	category := classifyRunBackupError(ctx, err)
-	opentelemetry.PluginBackup.LatestFailureTime.Record(ctx, time.Now().Unix())
+	opentelemetry.PluginBackup.LatestFailureTime.Record(ctx, time.Now().Unix(),
+		metric.WithAttributes(cluster))
 	opentelemetry.PluginBackup.Duration.Record(ctx, duration.Seconds(),
-		metric.WithAttributes(opentelemetry.OutcomeFailure.Attribute()))
+		metric.WithAttributes(cluster, opentelemetry.OutcomeFailure.Attribute()))
 	opentelemetry.PluginBackup.Runs.Add(ctx, 1,
 		metric.WithAttributes(
+			cluster,
 			opentelemetry.OutcomeFailure.Attribute(),
 			opentelemetry.AttributeKeyFailureCategory.Of(category.Name),
 		))