diff --git a/config/monitoring/prometheusrule.yaml b/config/monitoring/prometheusrule.yaml index 52b1b9f2..8b1726b8 100644 --- a/config/monitoring/prometheusrule.yaml +++ b/config/monitoring/prometheusrule.yaml @@ -546,17 +546,20 @@ spec: is dropped. galactic_cni_egress_pool_members shows whether its shards are unreachable (no route to their SID over an SRv6 uplink) or draining. # Fewer shards take tenants than the operator said the class should - # have, on a node with routes on the group. A class with no eligible - # shard at all is GalacticEgressClassUnserved instead. A node with no - # route on the group is left out for the same reason as in - # GalacticEgressPoolEmpty: an edge node running a shard counts one - # shard fewer than the rest of the cluster, its own. + # have, on a node with routes on the group. A NAT64 standby held out + # of the class while a primary serves it counts toward the minimum: + # it takes the class over as soon as no primary resolves, so a site + # with one primary and one standby still has its redundancy. A class + # with no eligible shard at all is GalacticEgressClassUnserved + # instead. A node with no route on the group is left out for the same + # reason as in GalacticEgressPoolEmpty: an edge node running a shard + # counts one shard fewer than the rest of the cluster, its own. - alert: GalacticEgressPoolBelowMinimum expr: | - sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state="active"}) + sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state=~"active|standby"}) < on (node) group_left max by (node) (galactic_cni_egress_pool_min_active{job="galactic-cni"}) and on (node, class) - sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state=~"active|draining|unreachable"}) > 0 + sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state=~"active|standby|draining|unreachable"}) > 0 and on (node, class) max by (node, class) (galactic_cni_egress_pool_enabled{job="galactic-cni"}) == 1 and on (node, class) @@ -570,7 +573,7 @@ spec: summary: "{{ $labels.node }} has fewer active {{ $labels.class }} egress shards than the pool minimum" runbook_url: "https://github.com/datum-cloud/infra/blob/main/docs/runbooks/galactic/README.md#galacticegresspoolbelowminimum" description: >- - {{ $labels.node }} has had {{ $value }} active {{ $labels.class }} egress shards for 10 + {{ $labels.node }} has had {{ $value }} active or standby {{ $labels.class }} egress shards for 10 minutes, below the GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE the cluster is configured for. The remaining shards carry the metro's whole egress load for that class. galactic_cni_egress_pool_members shows which shards are draining or unreachable. @@ -580,7 +583,7 @@ spec: # for the class, so its traffic has no egress. - alert: GalacticEgressClassUnserved expr: | - sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state=~"active|draining|unreachable|local"}) == 0 + sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state=~"active|standby|draining|unreachable|local"}) == 0 and on (node, class) sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state="ineligible"}) > 0 for: 15m diff --git a/config/monitoring/tests/rules-test.yaml b/config/monitoring/tests/rules-test.yaml index c7140d6b..523f1397 100644 --- a/config/monitoring/tests/rules-test.yaml +++ b/config/monitoring/tests/rules-test.yaml @@ -738,7 +738,52 @@ tests: summary: "dfw-worker has fewer active nat66 egress shards than the pool minimum" runbook_url: "https://github.com/datum-cloud/infra/blob/main/docs/runbooks/galactic/README.md#galacticegresspoolbelowminimum" description: >- - dfw-worker has had 0 active nat66 egress shards for 10 + dfw-worker has had 0 active or standby nat66 egress shards for 10 + minutes, below the GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE the cluster is configured for. The remaining + shards carry the metro's whole egress load for that class. galactic_cni_egress_pool_members shows + which shards are draining or unreachable. + + # A NAT64 class with one primary and one held standby meets a pool + # minimum of 2 and fires nothing. When the standby stops resolving at 5m + # the class has no failover left, and BelowMinimum fires after 10m. + - interval: 30s + input_series: + - series: 'galactic_cni_egress_pool_enabled{job="galactic-cni",node="dfw-worker",class="nat64:64:ff9b::/96"}' + values: "1x40" + - series: 'galactic_cni_egress_group_routes{job="galactic-cni",node="dfw-worker",class="nat64:64:ff9b::/96"}' + values: "3x40" + - series: 'galactic_cni_egress_pool_min_active{job="galactic-cni",node="dfw-worker"}' + values: "2x40" + - series: 'galactic_cni_egress_pool_members{job="galactic-cni",node="dfw-worker",class="nat64:64:ff9b::/96",state="active"}' + values: "1x40" + - series: 'galactic_cni_egress_pool_members{job="galactic-cni",node="dfw-worker",class="nat64:64:ff9b::/96",state="standby"}' + values: "1x10 0x30" + - series: 'galactic_cni_egress_pool_members{job="galactic-cni",node="dfw-worker",class="nat64:64:ff9b::/96",state="unreachable"}' + values: "0x10 1x30" + alert_rule_test: + - eval_time: 4m + alertname: GalacticEgressPoolBelowMinimum + exp_alerts: [] + - eval_time: 4m + alertname: GalacticEgressPoolEmpty + exp_alerts: [] + - eval_time: 14m + alertname: GalacticEgressPoolBelowMinimum + exp_alerts: [] + - eval_time: 16m + alertname: GalacticEgressPoolBelowMinimum + exp_alerts: + - exp_labels: + severity: warning + service: galactic + team: connect + node: dfw-worker + class: nat64:64:ff9b::/96 + exp_annotations: + summary: "dfw-worker has fewer active nat64:64:ff9b::/96 egress shards than the pool minimum" + runbook_url: "https://github.com/datum-cloud/infra/blob/main/docs/runbooks/galactic/README.md#galacticegresspoolbelowminimum" + description: >- + dfw-worker has had 1 active or standby nat64:64:ff9b::/96 egress shards for 10 minutes, below the GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE the cluster is configured for. The remaining shards carry the metro's whole egress load for that class. galactic_cni_egress_pool_members shows which shards are draining or unreachable. diff --git a/docs/nat/active-active-rollout.md b/docs/nat/active-active-rollout.md index 11dae7d2..4eb64a08 100644 --- a/docs/nat/active-active-rollout.md +++ b/docs/nat/active-active-rollout.md @@ -57,7 +57,8 @@ are deployed by infra. - [ ] On every compute node: `galactic_cni_egress_mode{mode="hashed"}` is 1, `galactic_cni_egress_shard_watch_synced` is 1, `galactic_cni_egress_pool_members{state="active"}` equals the shard - count for every class, `ineligible` is what you expect, and + count for every class (for a NAT64 class, less the shards marked + `spec.nat64Standby`, which count as `standby` instead), `ineligible` is what you expect, and `galactic_cni_egress_sweep_errors_total` does not move. Missing pool series on a node are a failure, not an idle pool. - [ ] On every edge node that runs a shard: its own shard reads `local`, diff --git a/docs/nat/configuration.md b/docs/nat/configuration.md index 5db8fc34..54dde224 100644 --- a/docs/nat/configuration.md +++ b/docs/nat/configuration.md @@ -521,6 +521,21 @@ hashed mode with more than three NAT64 prefixes in candidate once its status carries a `shardSID` and both `Programmed` and `Ready` are `True`, and it is not being deleted. `GALACTIC_CNI_EGRESS_SHARD_SIDS` is not used for placement in hashed mode. +- **A NAT64 standby serves NAT64 only when no primary can.** A site whose + shards share one IPv4 unicast prefix announces it from every edge, with + the standby's announcement prepended, so replies to the standby's + `shardAddressIPv4` arrive at the primary's node, which holds no session + for them and drops them. Mark that shard `spec.nat64Standby: true`. In + every NAT64 class, a standby is held out of the group while any shard + without the flag resolves, draining or not, and its pins stop being + honoured, so tenants pinned to it during a failover move back to the + primary on their next packet. Once no such shard resolves, the standby + takes the class. A standby's NAT66 placement is unchanged, since each + shard's IPv6 address is advertised from its own node. A held standby is + counted as `standby`, and `GalacticEgressPoolBelowMinimum` counts it + toward the pool minimum, since it takes over without anyone acting. A + primary running on this node is local and never resolves, so on the + primary's own node the standby still takes NAT64 tenants. - **A shard counts only while it is reachable.** A dead node's `EgressShard` status goes stale, but its BGP session drops and the route to its SID is withdrawn. A candidate takes tenants only while its SID resolves to a @@ -672,17 +687,17 @@ route as a redirect to a nonexistent interface and drops the traffic. ### Observability -| Metric | Meaning | -| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | -| `galactic_cni_egress_mode{mode}` | 1 for the configured mode. Always present, so a node configured for hashed mode that never published a group still says so. | -| `galactic_cni_egress_shard_watch_synced` | 1 once the `EgressShard` watch has listed the cluster. | -| `galactic_cni_egress_group_routes{class}` | Routes naming each class's group after the last successful sweep. Must be 0 before rolling back to an older release. | -| `galactic_cni_egress_sweep_errors_total{stage}` | Sweep failures: `apply`, `refresh`, `watch`, `close`, `disable`. | -| `galactic_cni_egress_pool_members{class,state}` | Each class's shards: `active`, `draining`, `unreachable`, `local` (runs on this node), and `ineligible` (cannot translate it). | -| `galactic_cni_egress_pool_enabled{class}` | 1 while the class's group takes new routes; 0 while it is being retired. | -| `galactic_cni_egress_pool_min_active` | `GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE`. | -| `galactic_cni_egress_shard_packets_total{shard_sid,class}` | Tenant packets this node sent to each shard for each class, since the shard took its slot. `..._bytes_total` is the same in bytes. | -| `galactic_cni_egress_shard_selections_total{result}` | How each packet was placed: `maglev`, `pin_hit`, `pin_stale` (re-placed), or dropped as `group_empty`, `shard_dead`, `parse_failed`. | +| Metric | Meaning | +| ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `galactic_cni_egress_mode{mode}` | 1 for the configured mode. Always present, so a node configured for hashed mode that never published a group still says so. | +| `galactic_cni_egress_shard_watch_synced` | 1 once the `EgressShard` watch has listed the cluster. | +| `galactic_cni_egress_group_routes{class}` | Routes naming each class's group after the last successful sweep. Must be 0 before rolling back to an older release. | +| `galactic_cni_egress_sweep_errors_total{stage}` | Sweep failures: `apply`, `refresh`, `watch`, `close`, `disable`. | +| `galactic_cni_egress_pool_members{class,state}` | Each class's shards: `active`, `draining`, `unreachable`, `local` (runs on this node), `standby` (a NAT64 standby held out while a primary serves the class), and `ineligible` (cannot translate it). | +| `galactic_cni_egress_pool_enabled{class}` | 1 while the class's group takes new routes; 0 while it is being retired. | +| `galactic_cni_egress_pool_min_active` | `GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE`. | +| `galactic_cni_egress_shard_packets_total{shard_sid,class}` | Tenant packets this node sent to each shard for each class, since the shard took its slot. `..._bytes_total` is the same in bytes. | +| `galactic_cni_egress_shard_selections_total{result}` | How each packet was placed: `maglev`, `pin_hit`, `pin_stale` (re-placed), or dropped as `group_empty`, `shard_dead`, `parse_failed`. | A node that never published a group exports no `pool_*` series. That is not a healthy empty pool: read it together with `galactic_cni_egress_mode` and diff --git a/go.mod b/go.mod index c7bf7deb..fcb07e4c 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/spf13/viper v1.21.0 github.com/vishvananda/netlink v1.3.2-0.20261002201547-1d1c62507e3c go.datum.net/cloud v0.1.2 - go.datum.net/network v0.4.1 + go.datum.net/network v0.4.2-0.20261014202400-2c27b50b5385 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 go.opentelemetry.io/otel v1.47.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 diff --git a/go.sum b/go.sum index fda06dcf..f440fca0 100644 --- a/go.sum +++ b/go.sum @@ -204,8 +204,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= go.datum.net/cloud v0.1.2 h1:Bs8qR+ZCxf70bZGvjscyXeqoIIy8uNo2zjfdrWzIlGE= go.datum.net/cloud v0.1.2/go.mod h1:HTeaepXL0EqiqNsQ13K4/kGEwv+VLJ//BGrUfVIFfEA= -go.datum.net/network v0.4.1 h1:SreFO3bMZBLSwsum+mYljYyB6g0BUXrZA6GXzHNxO3w= -go.datum.net/network v0.4.1/go.mod h1:LawI34k+ddtv3kRbmjiss0Lb6QKpSr60YP3fX1IG5RU= +go.datum.net/network v0.4.2-0.20261014202400-2c27b50b5385 h1:jenPo4QQ9eedLlwImONtHb8MXRpUFoMkbU314RGjFYY= +go.datum.net/network v0.4.2-0.20261014202400-2c27b50b5385/go.mod h1:LawI34k+ddtv3kRbmjiss0Lb6QKpSr60YP3fX1IG5RU= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 h1:Tq+E65HaNbZTeqcLw6QUhsV+/EIWdkfUH2CQzcrlE5M= diff --git a/internal/config/cni.go b/internal/config/cni.go index 10dd07b1..6d49132f 100644 --- a/internal/config/cni.go +++ b/internal/config/cni.go @@ -124,7 +124,8 @@ const ( // EnvCNIEgressPoolMinActive is the fewest active egress shards the cluster // is expected to have. It is exported as a metric for the pool alert to - // compare against, and does not change forwarding. Default 1. + // compare against, and does not change forwarding. A NAT64 standby held + // out of a class counts toward it there. Default 1. EnvCNIEgressPoolMinActive = "GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE" ) diff --git a/internal/installer/egressgroup.go b/internal/installer/egressgroup.go index 4fbbde40..e3a5a2f0 100644 --- a/internal/installer/egressgroup.go +++ b/internal/installer/egressgroup.go @@ -631,6 +631,9 @@ func (g *egressGroup) logApplied(class egressroutemap.TranslationClass, result e if s.Local { slog.Info("Egress shard runs on this node; it takes no tenant from here", "class", class.String(), "sid", s.SID, "slot", s.Slot) + } else if s.Held { + slog.Info("NAT64 standby egress shard held out while a sibling serves this class", + "class", class.String(), "sid", s.SID, "slot", s.Slot) } else if s.Err != nil { slog.Warn("Egress shard unreachable; it takes no tenant until its SID resolves", "class", class.String(), "sid", s.SID, "slot", s.Slot, "err", s.Err) @@ -712,7 +715,10 @@ func (g *egressGroup) refresh( // One with spec.drain set is draining. One being deleted is no candidate at // all: deletion withdraws its advertisement and clears its translation at // once, so it has nothing to drain with. One whose target is nodeName is -// local. +// local. For a NAT64 class, one with spec.nat64Standby set is a standby, which +// the group holds out of the class while a sibling without it is alive; see +// egressroutemap.ShardGroupTable.Apply. NAT66 ignores the flag, since every +// shard's IPv6 address is attracted to its own node. func egressShardCandidates( shards []bgpv1alpha1.EgressShard, class egressroutemap.TranslationClass, nodeName string, ) (eligible []egressroutemap.ShardCandidate, ineligible []string) { @@ -735,6 +741,7 @@ func egressShardCandidates( } eligible = append(eligible, egressroutemap.ShardCandidate{ SID: sid, Draining: shard.Spec.Drain, Local: targetsNode(shard.Spec.TargetRef, nodeName), + Standby: class.NAT64 && shard.Spec.NAT64Standby, }) } return eligible, ineligible diff --git a/internal/installer/egressgroup_test.go b/internal/installer/egressgroup_test.go index 07d2a628..29ac2ea9 100644 --- a/internal/installer/egressgroup_test.go +++ b/internal/installer/egressgroup_test.go @@ -945,3 +945,148 @@ func groupRoutes(t *testing.T, reg *prometheus.Registry) map[string]float64 { } return got } + +// TestEgressShardCandidates_NAT64Standby checks that spec.nat64Standby marks a +// shard a standby for every NAT64 class it translates and for no NAT66 one. +func TestEgressShardCandidates_NAT64Standby(t *testing.T) { + primary, standby := dual("primary", 0x10), dual("standby", 0x11) + primary.Status.TranslatesWellKnownPrefix = true + standby.Status.TranslatesWellKnownPrefix = true + standby.Spec.NAT64Standby = true + shards := []bgpv1alpha1.EgressShard{primary, standby} + + for _, tt := range []struct { + class egressroutemap.TranslationClass + want map[string]bool + }{ + {class: egressroutemap.NAT66Class, want: map[string]bool{shardSIDFor(0x10): false, shardSIDFor(0x11): false}}, + {class: nat64NSP, want: map[string]bool{shardSIDFor(0x10): false, shardSIDFor(0x11): true}}, + {class: nat64WKP, want: map[string]bool{shardSIDFor(0x10): false, shardSIDFor(0x11): true}}, + } { + t.Run(tt.class.String(), func(t *testing.T) { + eligible, _ := egressShardCandidates(shards, tt.class, "") + got := map[string]bool{} + for _, c := range eligible { + got[c.SID.String()] = c.Standby + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("standby = %v, want %v", got, tt.want) + } + }) + } +} + +// The pool states poolStates reports, as galactic_cni_egress_pool_members +// names them. +const ( + poolActive = "active" + poolStandby = "standby" + poolUnreachable = "unreachable" +) + +// poolStates is each class group's members by base SID, as the pool metric +// would classify them. +func poolStates(t *testing.T, f *fixture) map[egressroutemap.TranslationClass]map[string]string { + t.Helper() + out := map[egressroutemap.TranslationClass]map[string]string{} + for id := range uint32(egressroutemap.MaxShardGroups) { + s := status(t, f.groups, id) + if !s.Present { + continue + } + members, err := f.groups.Members(id) + if err != nil { + t.Fatal(err) + } + states := map[string]string{} + for _, m := range members { + switch { + case m.Held: + states[m.SID.String()] = poolStandby + case !m.Alive: + states[m.SID.String()] = poolUnreachable + default: + states[m.SID.String()] = poolActive + } + } + out[s.Class] = states + } + return out +} + +// TestSweepHashed_NAT64Standby checks the sweep end to end: a NAT64 standby is +// held out of every NAT64 class while a primary resolves, serves it when none +// does, and is unaffected in NAT66; with no standby flags nothing changes. +func TestSweepHashed_NAT64Standby(t *testing.T) { + primary, standby, other := staticSIDFor(0x10), staticSIDFor(0x11), staticSIDFor(0x12) + flagged := func(name string, nodeID uint16, isStandby bool) bgpv1alpha1.EgressShard { + s := dual(name, nodeID) + s.Spec.NAT64Standby = isStandby + return s + } + tests := []struct { + name string + shards []bgpv1alpha1.EgressShard + reachable []string + nat66 map[string]string + nat64 map[string]string + }{ + { + name: "both up", + shards: []bgpv1alpha1.EgressShard{flagged("primary", 0x10, false), flagged(poolStandby, 0x11, true)}, + reachable: []string{primary, standby}, + nat66: map[string]string{primary: poolActive, standby: poolActive}, + nat64: map[string]string{primary: poolActive, standby: poolStandby}, + }, + { + name: "primary unreachable", + shards: []bgpv1alpha1.EgressShard{flagged("primary", 0x10, false), flagged(poolStandby, 0x11, true)}, + reachable: []string{standby}, + nat66: map[string]string{primary: poolUnreachable, standby: poolActive}, + nat64: map[string]string{primary: poolUnreachable, standby: poolActive}, + }, + { + name: "only the standby exists", + shards: []bgpv1alpha1.EgressShard{flagged(poolStandby, 0x11, true)}, + reachable: []string{standby}, + nat66: map[string]string{standby: poolActive}, + nat64: map[string]string{standby: poolActive}, + }, + { + name: "no standby flags", + shards: []bgpv1alpha1.EgressShard{flagged("primary", 0x10, false), flagged("other", 0x12, false)}, + reachable: []string{primary, other}, + nat66: map[string]string{primary: poolActive, other: poolActive}, + nat64: map[string]string{primary: poolActive, other: poolActive}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := newFixture(t, testHashed, nil, []netip.Prefix{testNSP}) + f.g.shards = &fakeLister{shards: tt.shards, synced: true} + up := map[string]bool{} + for _, s := range tt.reachable { + up[s] = true + } + t.Cleanup(egressroutemap.OverrideResolverForTest( + func(sid net.IP) (int, net.HardwareAddr, net.HardwareAddr, error) { + base, _ := egressroutemap.BaseShardSID(sid) + if !up[base.String()] { + return 0, nil, nil, errInjected + } + return 7, net.HardwareAddr{2, 0, 0, 0, 0, 1}, net.HardwareAddr{2, 0, 0, 0, 0, 2}, nil + })) + + if err := f.sweep(t); err != nil { + t.Fatal(err) + } + got := poolStates(t, f) + if !reflect.DeepEqual(got[egressroutemap.NAT66Class], tt.nat66) { + t.Errorf("NAT66 members = %v, want %v", got[egressroutemap.NAT66Class], tt.nat66) + } + if !reflect.DeepEqual(got[nat64NSP], tt.nat64) { + t.Errorf("NAT64 members = %v, want %v", got[nat64NSP], tt.nat64) + } + }) + } +} diff --git a/internal/plumbing/ebpf/egressroutemap/shardgroup.go b/internal/plumbing/ebpf/egressroutemap/shardgroup.go index a9848687..8cd1cda4 100644 --- a/internal/plumbing/ebpf/egressroutemap/shardgroup.go +++ b/internal/plumbing/ebpf/egressroutemap/shardgroup.go @@ -53,6 +53,8 @@ const ( shardFlagAlive uint32 = 1 << 0 shardFlagDraining uint32 = 1 << 1 shardFlagLocal uint32 = 1 << 2 + shardFlagStandby uint32 = 1 << 3 + shardFlagHeld uint32 = 1 << 4 classKindNAT66 uint8 = 1 classKindNAT64 uint8 = 2 @@ -181,6 +183,12 @@ type ShardCandidate struct { // takes no tenant from this node, the same as in ordered mode, but it is // reported as local rather than unreachable, since nothing is wrong. Local bool + // Standby marks a NAT64 standby shard: its IPv4 address is attracted to + // a sibling shard's node while that sibling is up, so replies to it never + // reach it. In a NAT64 group, Apply holds a standby out of the class while + // any shard without the flag is alive, and lets it serve only when none + // is. A NAT66 group ignores it. + Standby bool } // ShardState reports one group member. @@ -197,6 +205,12 @@ type ShardState struct { Draining bool // Local echoes the candidate's Local. Local bool + // Standby echoes the candidate's Standby. + Standby bool + // Held reports that a standby shard is held out of the class because a + // shard without Standby is alive to serve it. A held shard resolves but + // is not alive: it takes no new tenant and no pin to it is honoured. + Held bool // Err is why the SID did not resolve, when it did not. Err error // Assigned reports that this Apply gave the shard its slot, so nothing @@ -204,7 +218,8 @@ type ShardState struct { Assigned bool } -// Active reports whether the shard takes new tenant addresses. +// Active reports whether the shard takes new tenant addresses. A held shard +// is never alive, so it is never active. func (s ShardState) Active() bool { return s.Alive && !s.Draining } // GroupResult reports what Apply found and published. @@ -405,6 +420,8 @@ func (g *ShardGroupTable) Members(groupID uint32) ([]ShardState, error) { Alive: s.Flags&shardFlagAlive != 0, Draining: s.Flags&shardFlagDraining != 0, Local: s.Flags&shardFlagLocal != 0, + Standby: s.Flags&shardFlagStandby != 0, + Held: s.Flags&shardFlagHeld != 0, }) } return members, nil @@ -445,6 +462,14 @@ func (a *allocator) take() uint32 { // Every node computes the same table from the same set, whatever slots the // shards occupy locally. // +// In a NAT64 group, a standby candidate that resolves is held rather than +// alive while any candidate without Standby is alive, draining or not: a +// draining sibling still attracts the standby's replies. A held shard keeps +// its slot and generation but is not alive, so it gets no share of the Maglev +// table and the datapath stops honouring every pin to it. Each Apply decides +// afresh, so a standby takes tenants as soon as its last alive sibling stops +// resolving and is held again once one resolves. +// // A shard keeps its slot, and the slot keeps its generation, for as long as // the shard stays a candidate of the same class, so a rebuild leaves its pins // intact. A new shard takes the lowest free slot under a fresh generation. @@ -476,6 +501,7 @@ func (g *ShardGroupTable) Apply(groupID uint32, policy GroupPolicy, candidates [ assignment := assignSlots(previous, wanted) resolved := resolveShards(wanted) + hold := policy.Class.NAT64 && anyPrimaryAlive(wanted, resolved) var ( value prog.UsidEgressShardGroupValue @@ -491,7 +517,9 @@ func (g *ShardGroupTable) Apply(groupID uint32, policy GroupPolicy, candidates [ if assigned { member.Generation = alloc.take() } - if !c.local && hop.err == nil { + held := hold && c.standby && !c.local && hop.err == nil + alive := !c.local && hop.err == nil && !held + if alive { member.Flags |= shardFlagAlive member.LinkIfindex = uint32(hop.hop.link) //nolint:gosec // a kernel ifindex copy(member.Dmac[:], hop.hop.dmac) @@ -503,11 +531,18 @@ func (g *ShardGroupTable) Apply(groupID uint32, policy GroupPolicy, candidates [ if c.local { member.Flags |= shardFlagLocal } + if c.standby && policy.Class.NAT64 { + member.Flags |= shardFlagStandby + } + if held { + member.Flags |= shardFlagHeld + } value.Shards[slot] = member state := ShardState{ - SID: c.sid, Slot: slot, Generation: member.Generation, Alive: !c.local && hop.err == nil, - Draining: c.draining, Local: c.local, Err: hop.err, Assigned: assigned, + SID: c.sid, Slot: slot, Generation: member.Generation, Alive: alive, + Draining: c.draining, Local: c.local, Standby: member.Flags&shardFlagStandby != 0, Held: held, + Err: hop.err, Assigned: assigned, } result.Shards = append(result.Shards, state) if state.Active() { @@ -562,6 +597,7 @@ func (g *ShardGroupTable) Close(groupID uint32) (bool, error) { SID: net.IP(netip.AddrFrom16(s.Sid).AsSlice()), Draining: s.Flags&shardFlagDraining != 0, Local: s.Flags&shardFlagLocal != 0, + Standby: s.Flags&shardFlagStandby != 0, }) } } @@ -608,10 +644,12 @@ type candidate struct { sid netip.Addr draining bool local bool + standby bool } // canonicalCandidates reduces candidates to base SIDs, ordered by SID. A -// shard listed twice is draining, or local, if either listing says so. +// shard listed twice is draining, local, or a standby, if either listing says +// so. func canonicalCandidates(candidates []ShardCandidate) ([]candidate, error) { bySID := map[netip.Addr]candidate{} for _, c := range candidates { @@ -620,7 +658,10 @@ func canonicalCandidates(candidates []ShardCandidate) ([]candidate, error) { return nil, fmt.Errorf("egressroutemap: %w", err) } prev := bySID[base] - bySID[base] = candidate{sid: base, draining: prev.draining || c.Draining, local: prev.local || c.Local} + bySID[base] = candidate{ + sid: base, draining: prev.draining || c.Draining, local: prev.local || c.Local, + standby: prev.standby || c.Standby, + } } out := make([]candidate, 0, len(bySID)) for _, c := range bySID { @@ -662,6 +703,18 @@ func assignSlots(slots [MaxShardsPerGroup]prog.UsidEgressShardMember, wanted []c return assignment } +// anyPrimaryAlive reports whether a wanted shard without Standby resolved, so +// it can serve the class and the standbys are held. A local shard does not +// count: this node cannot send it tenants. +func anyPrimaryAlive(wanted []candidate, resolved map[netip.Addr]resolution) bool { + for _, c := range wanted { + if !c.standby && !c.local && resolved[c.sid].err == nil { + return true + } + } + return false +} + // resolveShards resolves every wanted shard's next hop, except a local one's. func resolveShards(wanted []candidate) map[netip.Addr]resolution { out := make(map[netip.Addr]resolution, len(wanted)) diff --git a/internal/plumbing/ebpf/egressroutemap/shardgroup_datapath_test.go b/internal/plumbing/ebpf/egressroutemap/shardgroup_datapath_test.go index 36350832..6ed6ce28 100644 --- a/internal/plumbing/ebpf/egressroutemap/shardgroup_datapath_test.go +++ b/internal/plumbing/ebpf/egressroutemap/shardgroup_datapath_test.go @@ -599,3 +599,55 @@ func TestGroupDatapath_CountsVRFEgress(t *testing.T) { t.Errorf("vrf_traffic_counters = %+v, want %+v", got, want) } } + +// TestGroupDatapath_HeldStandbyLosesItsPins checks that a tenant pinned to a +// NAT64 standby while the primary was down is re-placed on the primary once it +// returns and the standby is held, rather than kept on a shard whose replies +// the primary's node attracts, and that it goes back to the standby when the +// primary fails again. +func TestGroupDatapath_HeldStandbyLosesItsPins(t *testing.T) { + dp := loadDatapath(t) + primary, standby := shardSID(0x10), shardSID(0x11) + nat64 := netip.MustParsePrefix("64:ff9b::/96") + const nat64Group = NAT66ShardGroup + 1 + route := &net.IPNet{IP: nat64.Addr().AsSlice(), Mask: net.CIDRMask(nat64.Bits(), 128)} + if err := dp.routes.RegisterGroup(dpTableID, route, nat64Group, dpArgument); err != nil { + t.Fatalf("RegisterGroup(%s): %v", nat64, err) + } + policy := GroupPolicy{Class: NAT64Class(nat64), PinIdle: time.Hour} + in := []ShardCandidate{{SID: net.IP(primary.AsSlice())}, {SID: net.IP(standby.AsSlice()), Standby: true}} + apply := func() { + t.Helper() + if _, err := dp.groups.Apply(nat64Group, policy, in); err != nil { + t.Fatalf("Apply: %v", err) + } + } + pkt := func(port uint16) []byte { + return udp6(tenant(1), netip.MustParseAddr("64:ff9b::c000:201"), port, 443) + } + + reachable(t, standby) // the primary is down + apply() + if verdict, got, _ := dp.send(t, pkt(1)); verdict != tcActRedirect || got != standby { + t.Fatalf("with the primary down: verdict %d toward %s, want redirect to the standby %s", verdict, got, standby) + } + + reachable(t, primary, standby) + apply() + if verdict, got, _ := dp.send(t, pkt(2)); verdict != tcActRedirect || got != primary { + t.Fatalf("after the primary returned: verdict %d toward %s, want the pinned tenant re-placed on %s", + verdict, got, primary) + } + if got := stat(t, dp.objs.EgressShardStats, prog.EgressShardStatPinStale); got != 1 { + t.Errorf("egress_shard_stats[pin_stale] = %d, want 1: the pin to the held standby must be rejected", got) + } + if _, got, _ := dp.send(t, pkt(3)); got != primary { + t.Errorf("next packet went to %s, want it to follow its new pin to %s", got, primary) + } + + reachable(t, standby) + apply() + if verdict, got, _ := dp.send(t, pkt(4)); verdict != tcActRedirect || got != standby { + t.Errorf("after the primary failed again: verdict %d toward %s, want the standby %s", verdict, got, standby) + } +} diff --git a/internal/plumbing/ebpf/egressroutemap/shardgroup_standby_test.go b/internal/plumbing/ebpf/egressroutemap/shardgroup_standby_test.go new file mode 100644 index 00000000..4ccacf51 --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/shardgroup_standby_test.go @@ -0,0 +1,229 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "net" + "net/netip" + "testing" + "time" +) + +var standbyTestNAT64 = NAT64Class(netip.MustParsePrefix("64:ff9b::/96")) + +// standbyShard describes one candidate in a standby test case. +type standbyShard struct { + sid netip.Addr + standby bool + draining bool + local bool + reachable bool +} + +// wantShard is the state Apply must report, and publish, for one shard. +type wantShard struct { + active bool + held bool + // unreachable is a shard that is neither alive nor held because its SID + // did not resolve. + unreachable bool +} + +func standbyCandidates(shards []standbyShard) (in []ShardCandidate, resolves []netip.Addr) { + for _, s := range shards { + in = append(in, ShardCandidate{ + SID: net.IP(s.sid.AsSlice()), Standby: s.standby, Draining: s.draining, Local: s.local, + }) + if s.reachable { + resolves = append(resolves, s.sid) + } + } + return in, resolves +} + +// TestApplyNAT64Standby checks which shards a group spreads tenants over when +// some are NAT64 standbys: a NAT64 group holds every standby out while a shard +// without the flag is alive, and a NAT66 group ignores the flag. +func TestApplyNAT64Standby(t *testing.T) { + primary, standby, other := shardSID(0x10), shardSID(0x11), shardSID(0x12) + tests := []struct { + name string + class TranslationClass + shards []standbyShard + want map[netip.Addr]wantShard + }{ + { + name: "both up: the standby is held out of NAT64", + class: standbyTestNAT64, + shards: []standbyShard{ + {sid: primary, reachable: true}, {sid: standby, standby: true, reachable: true}, + }, + want: map[netip.Addr]wantShard{primary: {active: true}, standby: {held: true}}, + }, + { + name: "both up: NAT66 ignores the flag", + class: NAT66Class, + shards: []standbyShard{ + {sid: primary, reachable: true}, {sid: standby, standby: true, reachable: true}, + }, + want: map[netip.Addr]wantShard{primary: {active: true}, standby: {active: true}}, + }, + { + name: "primary unreachable: the standby serves NAT64", + class: standbyTestNAT64, + shards: []standbyShard{ + {sid: primary}, {sid: standby, standby: true, reachable: true}, + }, + want: map[netip.Addr]wantShard{primary: {unreachable: true}, standby: {active: true}}, + }, + { + name: "only a standby exists: it serves NAT64", + class: standbyTestNAT64, + shards: []standbyShard{{sid: standby, standby: true, reachable: true}}, + want: map[netip.Addr]wantShard{standby: {active: true}}, + }, + { + name: "no standby flags: unchanged", + class: standbyTestNAT64, + shards: []standbyShard{ + {sid: primary, reachable: true}, {sid: other, reachable: true}, + }, + want: map[netip.Addr]wantShard{primary: {active: true}, other: {active: true}}, + }, + { + name: "draining primary still attracts the standby's replies", + class: standbyTestNAT64, + shards: []standbyShard{ + {sid: primary, draining: true, reachable: true}, {sid: standby, standby: true, reachable: true}, + }, + want: map[netip.Addr]wantShard{primary: {}, standby: {held: true}}, + }, + { + name: "a local primary takes no tenant from here, so the standby serves", + class: standbyTestNAT64, + shards: []standbyShard{ + {sid: primary, local: true}, {sid: standby, standby: true, reachable: true}, + }, + want: map[netip.Addr]wantShard{primary: {}, standby: {active: true}}, + }, + { + name: "an unreachable standby reports unreachable, not held", + class: standbyTestNAT64, + shards: []standbyShard{ + {sid: primary, reachable: true}, {sid: standby, standby: true}, + }, + want: map[netip.Addr]wantShard{primary: {active: true}, standby: {unreachable: true}}, + }, + { + name: "one live primary of two holds the standby", + class: standbyTestNAT64, + shards: []standbyShard{ + {sid: primary}, {sid: other, reachable: true}, {sid: standby, standby: true, reachable: true}, + }, + want: map[netip.Addr]wantShard{primary: {unreachable: true}, other: {active: true}, standby: {held: true}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + in, resolves := standbyCandidates(tt.shards) + reachable(t, resolves...) + + result := applyOK(t, g, GroupPolicy{Class: tt.class}, in) + v := published(t, store) + activeSlots := map[uint8]bool{} + for sid, want := range tt.want { + s := slotOf(t, result, sid) + if s.Active() != want.active || s.Held != want.held || (s.Err != nil) != want.unreachable { + t.Errorf("%s: state %+v, want active %v held %v unreachable %v", + sid, s, want.active, want.held, want.unreachable) + } + if s.Held && s.Alive { + t.Errorf("%s: held and alive at once", sid) + } + flags := v.Shards[s.Slot].Flags + if (flags&shardFlagHeld != 0) != want.held || (flags&shardFlagAlive != 0 && want.held) { + t.Errorf("%s: published flags %#x, want held %v and never alive while held", sid, flags, want.held) + } + wantStandbyFlag := tt.class.NAT64 && sid == standby + if (flags&shardFlagStandby != 0) != wantStandbyFlag { + t.Errorf("%s: published flags %#x, want standby flag %v", sid, flags, wantStandbyFlag) + } + if want.active { + activeSlots[uint8(s.Slot)] = true //nolint:gosec // < 32 + } + } + if int(v.Active) != len(activeSlots) || int(v.Candidates) != len(tt.shards) { + t.Errorf("snapshot active %d candidates %d, want %d and %d", + v.Active, v.Candidates, len(activeSlots), len(tt.shards)) + } + for i, slot := range v.Maglev { + if len(activeSlots) == 0 { + if slot != shardSlotNone { + t.Fatalf("Maglev slot %d = %d, want empty", i, slot) + } + continue + } + if !activeSlots[slot] { + t.Fatalf("Maglev slot %d = %d, want one of the active slots %v", i, slot, activeSlots) + } + } + }) + } +} + +// TestApplyNAT64StandbyFollowsPrimaryLiveness checks that each Apply decides +// again: the standby serves once the primary stops resolving and is held again +// when it returns, keeping its slot and generation throughout, and that Close +// keeps the standby flag so a retired group still holds it. +func TestApplyNAT64StandbyFollowsPrimaryLiveness(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + primary, standby := shardSID(0x10), shardSID(0x11) + policy := GroupPolicy{Class: standbyTestNAT64, PinIdle: time.Hour} + in := []ShardCandidate{{SID: net.IP(primary.AsSlice())}, {SID: net.IP(standby.AsSlice()), Standby: true}} + + reachable(t, primary, standby) + first := slotOf(t, applyOK(t, g, policy, in), standby) + if !first.Held { + t.Fatalf("standby %+v, want held while the primary resolves", first) + } + + reachable(t, standby) + failover := applyOK(t, g, policy, in) + if s := slotOf(t, failover, standby); !s.Active() || s.Held || s.Slot != first.Slot || + s.Generation != first.Generation { + t.Errorf("standby after the primary failed %+v, want active in slot %d generation %d", + s, first.Slot, first.Generation) + } + if !failover.Changed { + t.Error("losing the primary published nothing") + } + + reachable(t, primary, standby) + back := applyOK(t, g, policy, in) + if s := slotOf(t, back, standby); !s.Held || s.Active() || s.Slot != first.Slot || + s.Generation != first.Generation { + t.Errorf("standby after the primary returned %+v, want held in slot %d generation %d", + s, first.Slot, first.Generation) + } + if back.Active() != 1 || !slotOf(t, back, primary).Active() { + t.Errorf("after the primary returned, active = %d, want the primary alone", back.Active()) + } + + if _, err := g.Close(NAT66ShardGroup); err != nil { + t.Fatal(err) + } + members, err := g.Members(NAT66ShardGroup) + if err != nil { + t.Fatal(err) + } + for _, m := range members { + if (m.SID == standby) != m.Standby || (m.SID == standby) != m.Held { + t.Errorf("member after Close %+v: standby and held = %v, want %v", m, m.Standby, m.SID == standby) + } + } +} diff --git a/internal/plumbing/ebpf/metrics/egressshard.go b/internal/plumbing/ebpf/metrics/egressshard.go index f79d8abf..7339dc6e 100644 --- a/internal/plumbing/ebpf/metrics/egressshard.go +++ b/internal/plumbing/ebpf/metrics/egressshard.go @@ -27,6 +27,7 @@ const ( poolStateDraining = "draining" poolStateUnreachable = "unreachable" poolStateLocal = "local" + poolStateStandby = "standby" poolStateIneligible = "ineligible" ) @@ -54,8 +55,9 @@ var ( prometheus.BuildFQName(egressNamespace, "egress_pool", "members"), "The cluster's egress shards as this node's group for each translation class sees them: active shards "+ "take new tenant addresses, draining ones serve only the tenants pinned to them, unreachable ones "+ - "have no route from this node, local ones run on this node and take no tenant from it, and "+ - "ineligible ones cannot translate the class (no address in its family, or another NAT64 prefix).", + "have no route from this node, local ones run on this node and take no tenant from it, standby ones "+ + "are NAT64 standbys held out of the class while a sibling serves it, and ineligible ones cannot "+ + "translate the class (no address in its family, or another NAT64 prefix).", []string{labelClass, "state"}, nil, ) egressPoolMinActiveDesc = prometheus.NewDesc( @@ -138,12 +140,14 @@ func (c *EgressShardCollector) Collect(ch chan<- prometheus.Metric) { } states := map[string]int{ poolStateActive: 0, poolStateDraining: 0, poolStateUnreachable: 0, poolStateLocal: 0, - poolStateIneligible: status.Ineligible, + poolStateStandby: 0, poolStateIneligible: status.Ineligible, } for _, member := range members { switch { case member.Local: states[poolStateLocal]++ + case member.Held: + states[poolStateStandby]++ case !member.Alive: states[poolStateUnreachable]++ case member.Draining: diff --git a/internal/plumbing/ebpf/metrics/egressshard_test.go b/internal/plumbing/ebpf/metrics/egressshard_test.go index fbe37a4a..d154ff1a 100644 --- a/internal/plumbing/ebpf/metrics/egressshard_test.go +++ b/internal/plumbing/ebpf/metrics/egressshard_test.go @@ -54,7 +54,12 @@ func TestEgressShardCollector(t *testing.T) { // not open. nat64 := prog.UsidEgressShardGroupValue{Flags: 1 | 4, Ineligible: 3, ClassKind: 2, ClassPrefix: sidBytes("2001:db8:64::")} - store := snapshotStore{0: nat66, 1: nat64, 2: {ClassKind: 2}} + // A NAT64 class its primary serves, with the standby held out. + wkp := prog.UsidEgressShardGroupValue{Flags: 1, Candidates: 2, Active: 1, ClassKind: 2, + ClassPrefix: sidBytes("64:ff9b::")} + wkp.Shards[0] = prog.UsidEgressShardMember{Sid: sidBytes("2001:db8:ff09:20:e000::"), Flags: 1, Generation: 5} + wkp.Shards[1] = prog.UsidEgressShardMember{Sid: sidBytes("2001:db8:ff09:21:e000::"), Flags: 8 | 16, Generation: 6} + store := snapshotStore{0: nat66, 1: nat64, 2: {ClassKind: 2}, 3: wkp} counters := perCPUTable{ uint32(0): []prog.UsidEgressShardCounter{{Packets: 3, Bytes: 300}, {Packets: 4, Bytes: 400}}, } @@ -69,6 +74,7 @@ func TestEgressShardCollector(t *testing.T) { got[m.name] = m.value } const nat64Class = `class="nat64:2001:db8:64::/96"` + const wkpClass = `class="nat64:64:ff9b::/96"` want := map[string]float64{ `galactic_cni_egress_pool_min_active`: 2, `galactic_cni_egress_pool_enabled{class="nat66"}`: 1, @@ -78,6 +84,10 @@ func TestEgressShardCollector(t *testing.T) { `galactic_cni_egress_pool_members{class="nat66",state="unreachable"}`: 1, `galactic_cni_egress_pool_members{class="nat66",state="local"}`: 1, `galactic_cni_egress_pool_members{class="nat66",state="ineligible"}`: 0, + `galactic_cni_egress_pool_members{class="nat66",state="standby"}`: 0, + `galactic_cni_egress_pool_members{` + wkpClass + `,state="active"}`: 1, + `galactic_cni_egress_pool_members{` + wkpClass + `,state="standby"}`: 1, + `galactic_cni_egress_pool_members{` + wkpClass + `,state="unreachable"}`: 0, `galactic_cni_egress_pool_members{` + nat64Class + `,state="active"}`: 0, `galactic_cni_egress_pool_members{` + nat64Class + `,state="ineligible"}`: 3, `galactic_cni_egress_shard_packets_total{class="nat66",shard_sid="2001:db8:ff09:10:e000::"}`: 7, diff --git a/internal/plumbing/ebpf/prog/usid.c b/internal/plumbing/ebpf/prog/usid.c index 8c3893ca..adcc2b22 100644 --- a/internal/plumbing/ebpf/prog/usid.c +++ b/internal/plumbing/ebpf/prog/usid.c @@ -723,9 +723,16 @@ struct egress_route_value { // // LOCAL means the shard runs on this node. The control plane sets it for the // metrics alone and never sets ALIVE with it, so the datapath ignores it. +// +// STANDBY marks a NAT64 standby shard, and HELD one the control plane is +// keeping out of the class because a sibling without STANDBY is alive. Both +// are for the metrics alone: the control plane never sets ALIVE with HELD, so +// a held shard is absent from the Maglev table and its pins are stale. #define USID_EGRESS_SHARD_ALIVE (1u << 0) #define USID_EGRESS_SHARD_DRAINING (1u << 1) #define USID_EGRESS_SHARD_LOCAL (1u << 2) +#define USID_EGRESS_SHARD_STANDBY (1u << 3) +#define USID_EGRESS_SHARD_HELD (1u << 4) // struct egress_shard_member is one group member, with the link and L2 // addresses resolved ahead of time for the same reason egress_route_value