fix(cni): Keep NAT64 tenants off the standby egress shard

config/monitoring/prometheusrule.yaml

@@ -546,17 +546,20 @@ spec:
               is dropped. galactic_cni_egress_pool_members shows whether its shards are unreachable (no route to their
               SID over an SRv6 uplink) or draining.
         # Fewer shards take tenants than the operator said the class should
-        # have, on a node with routes on the group. A class with no eligible
-        # shard at all is GalacticEgressClassUnserved instead. A node with no
-        # route on the group is left out for the same reason as in
-        # GalacticEgressPoolEmpty: an edge node running a shard counts one
-        # shard fewer than the rest of the cluster, its own.
+        # have, on a node with routes on the group. A NAT64 standby held out
+        # of the class while a primary serves it counts toward the minimum:
+        # it takes the class over as soon as no primary resolves, so a site
+        # with one primary and one standby still has its redundancy. A class
+        # with no eligible shard at all is GalacticEgressClassUnserved
+        # instead. A node with no route on the group is left out for the same
+        # reason as in GalacticEgressPoolEmpty: an edge node running a shard
+        # counts one shard fewer than the rest of the cluster, its own.
         - alert: GalacticEgressPoolBelowMinimum
           expr: |
-            sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state="active"})
+            sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state=~"active|standby"})
               < on (node) group_left max by (node) (galactic_cni_egress_pool_min_active{job="galactic-cni"})
             and on (node, class)
-            sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state=~"active|draining|unreachable"}) > 0
+            sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state=~"active|standby|draining|unreachable"}) > 0
             and on (node, class)
             max by (node, class) (galactic_cni_egress_pool_enabled{job="galactic-cni"}) == 1
             and on (node, class)
@@ -570,7 +573,7 @@ spec:
             summary: "{{ $labels.node }} has fewer active {{ $labels.class }} egress shards than the pool minimum"
             runbook_url: "https://github.com/datum-cloud/infra/blob/main/docs/runbooks/galactic/README.md#galacticegresspoolbelowminimum"
             description: >-
-              {{ $labels.node }} has had {{ $value }} active {{ $labels.class }} egress shards for 10
+              {{ $labels.node }} has had {{ $value }} active or standby {{ $labels.class }} egress shards for 10
               minutes, below the GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE the cluster is configured for. The remaining
               shards carry the metro's whole egress load for that class. galactic_cni_egress_pool_members shows
               which shards are draining or unreachable.
@@ -580,7 +583,7 @@ spec:
         # for the class, so its traffic has no egress.
         - alert: GalacticEgressClassUnserved
           expr: |
-            sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state=~"active|draining|unreachable|local"}) == 0
+            sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state=~"active|standby|draining|unreachable|local"}) == 0
             and on (node, class)
             sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state="ineligible"}) > 0
           for: 15m

config/monitoring/tests/rules-test.yaml

@@ -738,7 +738,52 @@ tests:
               summary: "dfw-worker has fewer active nat66 egress shards than the pool minimum"
               runbook_url: "https://github.com/datum-cloud/infra/blob/main/docs/runbooks/galactic/README.md#galacticegresspoolbelowminimum"
               description: >-
-                dfw-worker has had 0 active nat66 egress shards for 10
+                dfw-worker has had 0 active or standby nat66 egress shards for 10
+                minutes, below the GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE the cluster is configured for. The remaining
+                shards carry the metro's whole egress load for that class. galactic_cni_egress_pool_members shows
+                which shards are draining or unreachable.
+
+  # A NAT64 class with one primary and one held standby meets a pool
+  # minimum of 2 and fires nothing. When the standby stops resolving at 5m
+  # the class has no failover left, and BelowMinimum fires after 10m.
+  - interval: 30s
+    input_series:
+      - series: 'galactic_cni_egress_pool_enabled{job="galactic-cni",node="dfw-worker",class="nat64:64:ff9b::/96"}'
+        values: "1x40"
+      - series: 'galactic_cni_egress_group_routes{job="galactic-cni",node="dfw-worker",class="nat64:64:ff9b::/96"}'
+        values: "3x40"
+      - series: 'galactic_cni_egress_pool_min_active{job="galactic-cni",node="dfw-worker"}'
+        values: "2x40"
+      - series: 'galactic_cni_egress_pool_members{job="galactic-cni",node="dfw-worker",class="nat64:64:ff9b::/96",state="active"}'
+        values: "1x40"
+      - series: 'galactic_cni_egress_pool_members{job="galactic-cni",node="dfw-worker",class="nat64:64:ff9b::/96",state="standby"}'
+        values: "1x10 0x30"
+      - series: 'galactic_cni_egress_pool_members{job="galactic-cni",node="dfw-worker",class="nat64:64:ff9b::/96",state="unreachable"}'
+        values: "0x10 1x30"
+    alert_rule_test:
+      - eval_time: 4m
+        alertname: GalacticEgressPoolBelowMinimum
+        exp_alerts: []
+      - eval_time: 4m
+        alertname: GalacticEgressPoolEmpty
+        exp_alerts: []
+      - eval_time: 14m
+        alertname: GalacticEgressPoolBelowMinimum
+        exp_alerts: []
+      - eval_time: 16m
+        alertname: GalacticEgressPoolBelowMinimum
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              service: galactic
+              team: connect
+              node: dfw-worker
+              class: nat64:64:ff9b::/96
+            exp_annotations:
+              summary: "dfw-worker has fewer active nat64:64:ff9b::/96 egress shards than the pool minimum"
+              runbook_url: "https://github.com/datum-cloud/infra/blob/main/docs/runbooks/galactic/README.md#galacticegresspoolbelowminimum"
+              description: >-
+                dfw-worker has had 1 active or standby nat64:64:ff9b::/96 egress shards for 10
                 minutes, below the GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE the cluster is configured for. The remaining
                 shards carry the metro's whole egress load for that class. galactic_cni_egress_pool_members shows
                 which shards are draining or unreachable.

docs/nat/active-active-rollout.md

@@ -57,7 +57,8 @@ are deployed by infra.
 - [ ] On every compute node: `galactic_cni_egress_mode{mode="hashed"}` is 1,
       `galactic_cni_egress_shard_watch_synced` is 1,
       `galactic_cni_egress_pool_members{state="active"}` equals the shard
-      count for every class, `ineligible` is what you expect, and
+      count for every class (for a NAT64 class, less the shards marked
+      `spec.nat64Standby`, which count as `standby` instead), `ineligible` is what you expect, and
       `galactic_cni_egress_sweep_errors_total` does not move. Missing pool
       series on a node are a failure, not an idle pool.
 - [ ] On every edge node that runs a shard: its own shard reads `local`,

docs/nat/configuration.md

@@ -521,6 +521,21 @@ hashed mode with more than three NAT64 prefixes in
   candidate once its status carries a `shardSID` and both `Programmed` and
   `Ready` are `True`, and it is not being deleted.
   `GALACTIC_CNI_EGRESS_SHARD_SIDS` is not used for placement in hashed mode.
+- **A NAT64 standby serves NAT64 only when no primary can.** A site whose
+  shards share one IPv4 unicast prefix announces it from every edge, with
+  the standby's announcement prepended, so replies to the standby's
+  `shardAddressIPv4` arrive at the primary's node, which holds no session
+  for them and drops them. Mark that shard `spec.nat64Standby: true`. In
+  every NAT64 class, a standby is held out of the group while any shard
+  without the flag resolves, draining or not, and its pins stop being
+  honoured, so tenants pinned to it during a failover move back to the
+  primary on their next packet. Once no such shard resolves, the standby
+  takes the class. A standby's NAT66 placement is unchanged, since each
+  shard's IPv6 address is advertised from its own node. A held standby is
+  counted as `standby`, and `GalacticEgressPoolBelowMinimum` counts it
+  toward the pool minimum, since it takes over without anyone acting. A
+  primary running on this node is local and never resolves, so on the
+  primary's own node the standby still takes NAT64 tenants.
 - **A shard counts only while it is reachable.** A dead node's `EgressShard`
   status goes stale, but its BGP session drops and the route to its SID is
   withdrawn. A candidate takes tenants only while its SID resolves to a
@@ -672,17 +687,17 @@ route as a redirect to a nonexistent interface and drops the traffic.
 
 ### Observability
 
-| Metric                                                     | Meaning                                                                                                                              |
-| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ |
-| `galactic_cni_egress_mode{mode}`                           | 1 for the configured mode. Always present, so a node configured for hashed mode that never published a group still says so.          |
-| `galactic_cni_egress_shard_watch_synced`                   | 1 once the `EgressShard` watch has listed the cluster.                                                                               |
-| `galactic_cni_egress_group_routes{class}`                  | Routes naming each class's group after the last successful sweep. Must be 0 before rolling back to an older release.                 |
-| `galactic_cni_egress_sweep_errors_total{stage}`            | Sweep failures: `apply`, `refresh`, `watch`, `close`, `disable`.                                                                     |
-| `galactic_cni_egress_pool_members{class,state}`            | Each class's shards: `active`, `draining`, `unreachable`, `local` (runs on this node), and `ineligible` (cannot translate it).       |
-| `galactic_cni_egress_pool_enabled{class}`                  | 1 while the class's group takes new routes; 0 while it is being retired.                                                             |
-| `galactic_cni_egress_pool_min_active`                      | `GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE`.                                                                                               |
-| `galactic_cni_egress_shard_packets_total{shard_sid,class}` | Tenant packets this node sent to each shard for each class, since the shard took its slot. `..._bytes_total` is the same in bytes.   |
-| `galactic_cni_egress_shard_selections_total{result}`       | How each packet was placed: `maglev`, `pin_hit`, `pin_stale` (re-placed), or dropped as `group_empty`, `shard_dead`, `parse_failed`. |
+| Metric                                                     | Meaning                                                                                                                                                                                               |
+| ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `galactic_cni_egress_mode{mode}`                           | 1 for the configured mode. Always present, so a node configured for hashed mode that never published a group still says so.                                                                           |
+| `galactic_cni_egress_shard_watch_synced`                   | 1 once the `EgressShard` watch has listed the cluster.                                                                                                                                                |
+| `galactic_cni_egress_group_routes{class}`                  | Routes naming each class's group after the last successful sweep. Must be 0 before rolling back to an older release.                                                                                  |
+| `galactic_cni_egress_sweep_errors_total{stage}`            | Sweep failures: `apply`, `refresh`, `watch`, `close`, `disable`.                                                                                                                                      |
+| `galactic_cni_egress_pool_members{class,state}`            | Each class's shards: `active`, `draining`, `unreachable`, `local` (runs on this node), `standby` (a NAT64 standby held out while a primary serves the class), and `ineligible` (cannot translate it). |
+| `galactic_cni_egress_pool_enabled{class}`                  | 1 while the class's group takes new routes; 0 while it is being retired.                                                                                                                              |
+| `galactic_cni_egress_pool_min_active`                      | `GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE`.                                                                                                                                                                |
+| `galactic_cni_egress_shard_packets_total{shard_sid,class}` | Tenant packets this node sent to each shard for each class, since the shard took its slot. `..._bytes_total` is the same in bytes.                                                                    |
+| `galactic_cni_egress_shard_selections_total{result}`       | How each packet was placed: `maglev`, `pin_hit`, `pin_stale` (re-placed), or dropped as `group_empty`, `shard_dead`, `parse_failed`.                                                                  |
 
 A node that never published a group exports no `pool_*` series. That is not
 a healthy empty pool: read it together with `galactic_cni_egress_mode` and

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/spf13/viper v1.21.0
 	github.com/vishvananda/netlink v1.3.2-0.20261002201547-1d1c62507e3c
 	go.datum.net/cloud v0.1.2
-	go.datum.net/network v0.4.1
+	go.datum.net/network v0.4.2-0.20261014202400-2c27b50b5385
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0
 	go.opentelemetry.io/otel v1.47.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0

go.sum

@@ -204,8 +204,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 go.datum.net/cloud v0.1.2 h1:Bs8qR+ZCxf70bZGvjscyXeqoIIy8uNo2zjfdrWzIlGE=
 go.datum.net/cloud v0.1.2/go.mod h1:HTeaepXL0EqiqNsQ13K4/kGEwv+VLJ//BGrUfVIFfEA=
-go.datum.net/network v0.4.1 h1:SreFO3bMZBLSwsum+mYljYyB6g0BUXrZA6GXzHNxO3w=
-go.datum.net/network v0.4.1/go.mod h1:LawI34k+ddtv3kRbmjiss0Lb6QKpSr60YP3fX1IG5RU=
+go.datum.net/network v0.4.2-0.20261014202400-2c27b50b5385 h1:jenPo4QQ9eedLlwImONtHb8MXRpUFoMkbU314RGjFYY=
+go.datum.net/network v0.4.2-0.20261014202400-2c27b50b5385/go.mod h1:LawI34k+ddtv3kRbmjiss0Lb6QKpSr60YP3fX1IG5RU=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 h1:Tq+E65HaNbZTeqcLw6QUhsV+/EIWdkfUH2CQzcrlE5M=

internal/config/cni.go

@@ -124,7 +124,8 @@ const (
 
 	// EnvCNIEgressPoolMinActive is the fewest active egress shards the cluster
 	// is expected to have. It is exported as a metric for the pool alert to
-	// compare against, and does not change forwarding. Default 1.
+	// compare against, and does not change forwarding. A NAT64 standby held
+	// out of a class counts toward it there. Default 1.
 	EnvCNIEgressPoolMinActive = "GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE"
 )
 

internal/installer/egressgroup.go

@@ -631,6 +631,9 @@ func (g *egressGroup) logApplied(class egressroutemap.TranslationClass, result e
 		if s.Local {
 			slog.Info("Egress shard runs on this node; it takes no tenant from here",
 				"class", class.String(), "sid", s.SID, "slot", s.Slot)
+		} else if s.Held {
+			slog.Info("NAT64 standby egress shard held out while a sibling serves this class",
+				"class", class.String(), "sid", s.SID, "slot", s.Slot)
 		} else if s.Err != nil {
 			slog.Warn("Egress shard unreachable; it takes no tenant until its SID resolves",
 				"class", class.String(), "sid", s.SID, "slot", s.Slot, "err", s.Err)
@@ -712,7 +715,10 @@ func (g *egressGroup) refresh(
 // One with spec.drain set is draining. One being deleted is no candidate at
 // all: deletion withdraws its advertisement and clears its translation at
 // once, so it has nothing to drain with. One whose target is nodeName is
-// local.
+// local. For a NAT64 class, one with spec.nat64Standby set is a standby, which
+// the group holds out of the class while a sibling without it is alive; see
+// egressroutemap.ShardGroupTable.Apply. NAT66 ignores the flag, since every
+// shard's IPv6 address is attracted to its own node.
 func egressShardCandidates(
 	shards []bgpv1alpha1.EgressShard, class egressroutemap.TranslationClass, nodeName string,
 ) (eligible []egressroutemap.ShardCandidate, ineligible []string) {
@@ -735,6 +741,7 @@ func egressShardCandidates(
 		}
 		eligible = append(eligible, egressroutemap.ShardCandidate{
 			SID: sid, Draining: shard.Spec.Drain, Local: targetsNode(shard.Spec.TargetRef, nodeName),
+			Standby: class.NAT64 && shard.Spec.NAT64Standby,
 		})
 	}
 	return eligible, ineligible