From 4460e53e6f8d315ab8b02a1b7ff8083e5bafcd7e Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Wed, 14 Oct 2026 16:33:49 -0400 Subject: [PATCH] fix(cni): Keep NAT64 tenants off the standby egress shard Each edge site runs two egress shards, and both announce the site's IPv4 unicast prefix upstream. The standby prepends its announcement, so every IPv4 reply, including replies to the standby's own shardAddressIPv4, arrives at the primary, which holds no session for it and drops it. In hashed egress mode the standby is a member of every NAT64 class it translates, so tenants hashed to it lose NAT64 entirely. NAT66 is unaffected, since each shard announces its own IPv6 /128. EgressShard now carries spec.nat64Standby (go.datum.net/network, pinned here to a pseudo-version of the unreleased branch). egressShardCandidates marks such a shard a standby for NAT64 classes only. ShardGroupTable.Apply, which is where each candidate's SID is resolved, holds every standby out of a NAT64 group while any candidate without the flag resolves, draining or not, because a draining primary still attracts the standby's replies. A held standby keeps its slot and generation but is published without the alive flag, so it gets no Maglev slot and usid_egress rejects every pin to it as stale; a tenant pinned to it during a failover is re-placed on its next packet. Apply decides afresh on every sweep, and the sweep already runs on every route or neighbour change near a member's locator, so the standby takes the class as soon as no primary resolves and is held again when one returns. A local primary does not count, since this node cannot send it tenants. The pool metric reports a held standby as a new standby state. GalacticEgressPoolBelowMinimum counts it toward GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE, since it takes over without anyone acting; otherwise every NAT64 class would sit below a minimum of 2 at a two-shard site. Related to datum-cloud/infra#7033 Co-Authored-By: Claude Opus 5.5 --- config/monitoring/prometheusrule.yaml | 21 +- config/monitoring/tests/rules-test.yaml | 47 +++- docs/nat/active-active-rollout.md | 3 +- docs/nat/configuration.md | 37 ++- go.mod | 2 +- go.sum | 4 +- internal/config/cni.go | 3 +- internal/installer/egressgroup.go | 9 +- internal/installer/egressgroup_test.go | 145 +++++++++++ .../ebpf/egressroutemap/shardgroup.go | 65 ++++- .../shardgroup_datapath_test.go | 52 ++++ .../egressroutemap/shardgroup_standby_test.go | 229 ++++++++++++++++++ internal/plumbing/ebpf/metrics/egressshard.go | 10 +- .../plumbing/ebpf/metrics/egressshard_test.go | 12 +- internal/plumbing/ebpf/prog/usid.c | 7 + 15 files changed, 609 insertions(+), 37 deletions(-) create mode 100644 internal/plumbing/ebpf/egressroutemap/shardgroup_standby_test.go diff --git a/config/monitoring/prometheusrule.yaml b/config/monitoring/prometheusrule.yaml index 52b1b9f2..8b1726b8 100644 --- a/config/monitoring/prometheusrule.yaml +++ b/config/monitoring/prometheusrule.yaml @@ -546,17 +546,20 @@ spec: is dropped. galactic_cni_egress_pool_members shows whether its shards are unreachable (no route to their SID over an SRv6 uplink) or draining. # Fewer shards take tenants than the operator said the class should - # have, on a node with routes on the group. A class with no eligible - # shard at all is GalacticEgressClassUnserved instead. A node with no - # route on the group is left out for the same reason as in - # GalacticEgressPoolEmpty: an edge node running a shard counts one - # shard fewer than the rest of the cluster, its own. + # have, on a node with routes on the group. A NAT64 standby held out + # of the class while a primary serves it counts toward the minimum: + # it takes the class over as soon as no primary resolves, so a site + # with one primary and one standby still has its redundancy. A class + # with no eligible shard at all is GalacticEgressClassUnserved + # instead. A node with no route on the group is left out for the same + # reason as in GalacticEgressPoolEmpty: an edge node running a shard + # counts one shard fewer than the rest of the cluster, its own. - alert: GalacticEgressPoolBelowMinimum expr: | - sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state="active"}) + sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state=~"active|standby"}) < on (node) group_left max by (node) (galactic_cni_egress_pool_min_active{job="galactic-cni"}) and on (node, class) - sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state=~"active|draining|unreachable"}) > 0 + sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state=~"active|standby|draining|unreachable"}) > 0 and on (node, class) max by (node, class) (galactic_cni_egress_pool_enabled{job="galactic-cni"}) == 1 and on (node, class) @@ -570,7 +573,7 @@ spec: summary: "{{ $labels.node }} has fewer active {{ $labels.class }} egress shards than the pool minimum" runbook_url: "https://github.com/datum-cloud/infra/blob/main/docs/runbooks/galactic/README.md#galacticegresspoolbelowminimum" description: >- - {{ $labels.node }} has had {{ $value }} active {{ $labels.class }} egress shards for 10 + {{ $labels.node }} has had {{ $value }} active or standby {{ $labels.class }} egress shards for 10 minutes, below the GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE the cluster is configured for. The remaining shards carry the metro's whole egress load for that class. galactic_cni_egress_pool_members shows which shards are draining or unreachable. @@ -580,7 +583,7 @@ spec: # for the class, so its traffic has no egress. - alert: GalacticEgressClassUnserved expr: | - sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state=~"active|draining|unreachable|local"}) == 0 + sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state=~"active|standby|draining|unreachable|local"}) == 0 and on (node, class) sum by (node, class) (galactic_cni_egress_pool_members{job="galactic-cni", state="ineligible"}) > 0 for: 15m diff --git a/config/monitoring/tests/rules-test.yaml b/config/monitoring/tests/rules-test.yaml index c7140d6b..523f1397 100644 --- a/config/monitoring/tests/rules-test.yaml +++ b/config/monitoring/tests/rules-test.yaml @@ -738,7 +738,52 @@ tests: summary: "dfw-worker has fewer active nat66 egress shards than the pool minimum" runbook_url: "https://github.com/datum-cloud/infra/blob/main/docs/runbooks/galactic/README.md#galacticegresspoolbelowminimum" description: >- - dfw-worker has had 0 active nat66 egress shards for 10 + dfw-worker has had 0 active or standby nat66 egress shards for 10 + minutes, below the GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE the cluster is configured for. The remaining + shards carry the metro's whole egress load for that class. galactic_cni_egress_pool_members shows + which shards are draining or unreachable. + + # A NAT64 class with one primary and one held standby meets a pool + # minimum of 2 and fires nothing. When the standby stops resolving at 5m + # the class has no failover left, and BelowMinimum fires after 10m. + - interval: 30s + input_series: + - series: 'galactic_cni_egress_pool_enabled{job="galactic-cni",node="dfw-worker",class="nat64:64:ff9b::/96"}' + values: "1x40" + - series: 'galactic_cni_egress_group_routes{job="galactic-cni",node="dfw-worker",class="nat64:64:ff9b::/96"}' + values: "3x40" + - series: 'galactic_cni_egress_pool_min_active{job="galactic-cni",node="dfw-worker"}' + values: "2x40" + - series: 'galactic_cni_egress_pool_members{job="galactic-cni",node="dfw-worker",class="nat64:64:ff9b::/96",state="active"}' + values: "1x40" + - series: 'galactic_cni_egress_pool_members{job="galactic-cni",node="dfw-worker",class="nat64:64:ff9b::/96",state="standby"}' + values: "1x10 0x30" + - series: 'galactic_cni_egress_pool_members{job="galactic-cni",node="dfw-worker",class="nat64:64:ff9b::/96",state="unreachable"}' + values: "0x10 1x30" + alert_rule_test: + - eval_time: 4m + alertname: GalacticEgressPoolBelowMinimum + exp_alerts: [] + - eval_time: 4m + alertname: GalacticEgressPoolEmpty + exp_alerts: [] + - eval_time: 14m + alertname: GalacticEgressPoolBelowMinimum + exp_alerts: [] + - eval_time: 16m + alertname: GalacticEgressPoolBelowMinimum + exp_alerts: + - exp_labels: + severity: warning + service: galactic + team: connect + node: dfw-worker + class: nat64:64:ff9b::/96 + exp_annotations: + summary: "dfw-worker has fewer active nat64:64:ff9b::/96 egress shards than the pool minimum" + runbook_url: "https://github.com/datum-cloud/infra/blob/main/docs/runbooks/galactic/README.md#galacticegresspoolbelowminimum" + description: >- + dfw-worker has had 1 active or standby nat64:64:ff9b::/96 egress shards for 10 minutes, below the GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE the cluster is configured for. The remaining shards carry the metro's whole egress load for that class. galactic_cni_egress_pool_members shows which shards are draining or unreachable. diff --git a/docs/nat/active-active-rollout.md b/docs/nat/active-active-rollout.md index 11dae7d2..4eb64a08 100644 --- a/docs/nat/active-active-rollout.md +++ b/docs/nat/active-active-rollout.md @@ -57,7 +57,8 @@ are deployed by infra. - [ ] On every compute node: `galactic_cni_egress_mode{mode="hashed"}` is 1, `galactic_cni_egress_shard_watch_synced` is 1, `galactic_cni_egress_pool_members{state="active"}` equals the shard - count for every class, `ineligible` is what you expect, and + count for every class (for a NAT64 class, less the shards marked + `spec.nat64Standby`, which count as `standby` instead), `ineligible` is what you expect, and `galactic_cni_egress_sweep_errors_total` does not move. Missing pool series on a node are a failure, not an idle pool. - [ ] On every edge node that runs a shard: its own shard reads `local`, diff --git a/docs/nat/configuration.md b/docs/nat/configuration.md index 5db8fc34..54dde224 100644 --- a/docs/nat/configuration.md +++ b/docs/nat/configuration.md @@ -521,6 +521,21 @@ hashed mode with more than three NAT64 prefixes in candidate once its status carries a `shardSID` and both `Programmed` and `Ready` are `True`, and it is not being deleted. `GALACTIC_CNI_EGRESS_SHARD_SIDS` is not used for placement in hashed mode. +- **A NAT64 standby serves NAT64 only when no primary can.** A site whose + shards share one IPv4 unicast prefix announces it from every edge, with + the standby's announcement prepended, so replies to the standby's + `shardAddressIPv4` arrive at the primary's node, which holds no session + for them and drops them. Mark that shard `spec.nat64Standby: true`. In + every NAT64 class, a standby is held out of the group while any shard + without the flag resolves, draining or not, and its pins stop being + honoured, so tenants pinned to it during a failover move back to the + primary on their next packet. Once no such shard resolves, the standby + takes the class. A standby's NAT66 placement is unchanged, since each + shard's IPv6 address is advertised from its own node. A held standby is + counted as `standby`, and `GalacticEgressPoolBelowMinimum` counts it + toward the pool minimum, since it takes over without anyone acting. A + primary running on this node is local and never resolves, so on the + primary's own node the standby still takes NAT64 tenants. - **A shard counts only while it is reachable.** A dead node's `EgressShard` status goes stale, but its BGP session drops and the route to its SID is withdrawn. A candidate takes tenants only while its SID resolves to a @@ -672,17 +687,17 @@ route as a redirect to a nonexistent interface and drops the traffic. ### Observability -| Metric | Meaning | -| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | -| `galactic_cni_egress_mode{mode}` | 1 for the configured mode. Always present, so a node configured for hashed mode that never published a group still says so. | -| `galactic_cni_egress_shard_watch_synced` | 1 once the `EgressShard` watch has listed the cluster. | -| `galactic_cni_egress_group_routes{class}` | Routes naming each class's group after the last successful sweep. Must be 0 before rolling back to an older release. | -| `galactic_cni_egress_sweep_errors_total{stage}` | Sweep failures: `apply`, `refresh`, `watch`, `close`, `disable`. | -| `galactic_cni_egress_pool_members{class,state}` | Each class's shards: `active`, `draining`, `unreachable`, `local` (runs on this node), and `ineligible` (cannot translate it). | -| `galactic_cni_egress_pool_enabled{class}` | 1 while the class's group takes new routes; 0 while it is being retired. | -| `galactic_cni_egress_pool_min_active` | `GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE`. | -| `galactic_cni_egress_shard_packets_total{shard_sid,class}` | Tenant packets this node sent to each shard for each class, since the shard took its slot. `..._bytes_total` is the same in bytes. | -| `galactic_cni_egress_shard_selections_total{result}` | How each packet was placed: `maglev`, `pin_hit`, `pin_stale` (re-placed), or dropped as `group_empty`, `shard_dead`, `parse_failed`. | +| Metric | Meaning | +| ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `galactic_cni_egress_mode{mode}` | 1 for the configured mode. Always present, so a node configured for hashed mode that never published a group still says so. | +| `galactic_cni_egress_shard_watch_synced` | 1 once the `EgressShard` watch has listed the cluster. | +| `galactic_cni_egress_group_routes{class}` | Routes naming each class's group after the last successful sweep. Must be 0 before rolling back to an older release. | +| `galactic_cni_egress_sweep_errors_total{stage}` | Sweep failures: `apply`, `refresh`, `watch`, `close`, `disable`. | +| `galactic_cni_egress_pool_members{class,state}` | Each class's shards: `active`, `draining`, `unreachable`, `local` (runs on this node), `standby` (a NAT64 standby held out while a primary serves the class), and `ineligible` (cannot translate it). | +| `galactic_cni_egress_pool_enabled{class}` | 1 while the class's group takes new routes; 0 while it is being retired. | +| `galactic_cni_egress_pool_min_active` | `GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE`. | +| `galactic_cni_egress_shard_packets_total{shard_sid,class}` | Tenant packets this node sent to each shard for each class, since the shard took its slot. `..._bytes_total` is the same in bytes. | +| `galactic_cni_egress_shard_selections_total{result}` | How each packet was placed: `maglev`, `pin_hit`, `pin_stale` (re-placed), or dropped as `group_empty`, `shard_dead`, `parse_failed`. | A node that never published a group exports no `pool_*` series. That is not a healthy empty pool: read it together with `galactic_cni_egress_mode` and diff --git a/go.mod b/go.mod index c7bf7deb..fcb07e4c 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/spf13/viper v1.21.0 github.com/vishvananda/netlink v1.3.2-0.20261002201547-1d1c62507e3c go.datum.net/cloud v0.1.2 - go.datum.net/network v0.4.1 + go.datum.net/network v0.4.2-0.20261014202400-2c27b50b5385 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 go.opentelemetry.io/otel v1.47.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 diff --git a/go.sum b/go.sum index fda06dcf..f440fca0 100644 --- a/go.sum +++ b/go.sum @@ -204,8 +204,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= go.datum.net/cloud v0.1.2 h1:Bs8qR+ZCxf70bZGvjscyXeqoIIy8uNo2zjfdrWzIlGE= go.datum.net/cloud v0.1.2/go.mod h1:HTeaepXL0EqiqNsQ13K4/kGEwv+VLJ//BGrUfVIFfEA= -go.datum.net/network v0.4.1 h1:SreFO3bMZBLSwsum+mYljYyB6g0BUXrZA6GXzHNxO3w= -go.datum.net/network v0.4.1/go.mod h1:LawI34k+ddtv3kRbmjiss0Lb6QKpSr60YP3fX1IG5RU= +go.datum.net/network v0.4.2-0.20261014202400-2c27b50b5385 h1:jenPo4QQ9eedLlwImONtHb8MXRpUFoMkbU314RGjFYY= +go.datum.net/network v0.4.2-0.20261014202400-2c27b50b5385/go.mod h1:LawI34k+ddtv3kRbmjiss0Lb6QKpSr60YP3fX1IG5RU= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 h1:Tq+E65HaNbZTeqcLw6QUhsV+/EIWdkfUH2CQzcrlE5M= diff --git a/internal/config/cni.go b/internal/config/cni.go index 10dd07b1..6d49132f 100644 --- a/internal/config/cni.go +++ b/internal/config/cni.go @@ -124,7 +124,8 @@ const ( // EnvCNIEgressPoolMinActive is the fewest active egress shards the cluster // is expected to have. It is exported as a metric for the pool alert to - // compare against, and does not change forwarding. Default 1. + // compare against, and does not change forwarding. A NAT64 standby held + // out of a class counts toward it there. Default 1. EnvCNIEgressPoolMinActive = "GALACTIC_CNI_EGRESS_POOL_MIN_ACTIVE" ) diff --git a/internal/installer/egressgroup.go b/internal/installer/egressgroup.go index 4fbbde40..e3a5a2f0 100644 --- a/internal/installer/egressgroup.go +++ b/internal/installer/egressgroup.go @@ -631,6 +631,9 @@ func (g *egressGroup) logApplied(class egressroutemap.TranslationClass, result e if s.Local { slog.Info("Egress shard runs on this node; it takes no tenant from here", "class", class.String(), "sid", s.SID, "slot", s.Slot) + } else if s.Held { + slog.Info("NAT64 standby egress shard held out while a sibling serves this class", + "class", class.String(), "sid", s.SID, "slot", s.Slot) } else if s.Err != nil { slog.Warn("Egress shard unreachable; it takes no tenant until its SID resolves", "class", class.String(), "sid", s.SID, "slot", s.Slot, "err", s.Err) @@ -712,7 +715,10 @@ func (g *egressGroup) refresh( // One with spec.drain set is draining. One being deleted is no candidate at // all: deletion withdraws its advertisement and clears its translation at // once, so it has nothing to drain with. One whose target is nodeName is -// local. +// local. For a NAT64 class, one with spec.nat64Standby set is a standby, which +// the group holds out of the class while a sibling without it is alive; see +// egressroutemap.ShardGroupTable.Apply. NAT66 ignores the flag, since every +// shard's IPv6 address is attracted to its own node. func egressShardCandidates( shards []bgpv1alpha1.EgressShard, class egressroutemap.TranslationClass, nodeName string, ) (eligible []egressroutemap.ShardCandidate, ineligible []string) { @@ -735,6 +741,7 @@ func egressShardCandidates( } eligible = append(eligible, egressroutemap.ShardCandidate{ SID: sid, Draining: shard.Spec.Drain, Local: targetsNode(shard.Spec.TargetRef, nodeName), + Standby: class.NAT64 && shard.Spec.NAT64Standby, }) } return eligible, ineligible diff --git a/internal/installer/egressgroup_test.go b/internal/installer/egressgroup_test.go index 07d2a628..29ac2ea9 100644 --- a/internal/installer/egressgroup_test.go +++ b/internal/installer/egressgroup_test.go @@ -945,3 +945,148 @@ func groupRoutes(t *testing.T, reg *prometheus.Registry) map[string]float64 { } return got } + +// TestEgressShardCandidates_NAT64Standby checks that spec.nat64Standby marks a +// shard a standby for every NAT64 class it translates and for no NAT66 one. +func TestEgressShardCandidates_NAT64Standby(t *testing.T) { + primary, standby := dual("primary", 0x10), dual("standby", 0x11) + primary.Status.TranslatesWellKnownPrefix = true + standby.Status.TranslatesWellKnownPrefix = true + standby.Spec.NAT64Standby = true + shards := []bgpv1alpha1.EgressShard{primary, standby} + + for _, tt := range []struct { + class egressroutemap.TranslationClass + want map[string]bool + }{ + {class: egressroutemap.NAT66Class, want: map[string]bool{shardSIDFor(0x10): false, shardSIDFor(0x11): false}}, + {class: nat64NSP, want: map[string]bool{shardSIDFor(0x10): false, shardSIDFor(0x11): true}}, + {class: nat64WKP, want: map[string]bool{shardSIDFor(0x10): false, shardSIDFor(0x11): true}}, + } { + t.Run(tt.class.String(), func(t *testing.T) { + eligible, _ := egressShardCandidates(shards, tt.class, "") + got := map[string]bool{} + for _, c := range eligible { + got[c.SID.String()] = c.Standby + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("standby = %v, want %v", got, tt.want) + } + }) + } +} + +// The pool states poolStates reports, as galactic_cni_egress_pool_members +// names them. +const ( + poolActive = "active" + poolStandby = "standby" + poolUnreachable = "unreachable" +) + +// poolStates is each class group's members by base SID, as the pool metric +// would classify them. +func poolStates(t *testing.T, f *fixture) map[egressroutemap.TranslationClass]map[string]string { + t.Helper() + out := map[egressroutemap.TranslationClass]map[string]string{} + for id := range uint32(egressroutemap.MaxShardGroups) { + s := status(t, f.groups, id) + if !s.Present { + continue + } + members, err := f.groups.Members(id) + if err != nil { + t.Fatal(err) + } + states := map[string]string{} + for _, m := range members { + switch { + case m.Held: + states[m.SID.String()] = poolStandby + case !m.Alive: + states[m.SID.String()] = poolUnreachable + default: + states[m.SID.String()] = poolActive + } + } + out[s.Class] = states + } + return out +} + +// TestSweepHashed_NAT64Standby checks the sweep end to end: a NAT64 standby is +// held out of every NAT64 class while a primary resolves, serves it when none +// does, and is unaffected in NAT66; with no standby flags nothing changes. +func TestSweepHashed_NAT64Standby(t *testing.T) { + primary, standby, other := staticSIDFor(0x10), staticSIDFor(0x11), staticSIDFor(0x12) + flagged := func(name string, nodeID uint16, isStandby bool) bgpv1alpha1.EgressShard { + s := dual(name, nodeID) + s.Spec.NAT64Standby = isStandby + return s + } + tests := []struct { + name string + shards []bgpv1alpha1.EgressShard + reachable []string + nat66 map[string]string + nat64 map[string]string + }{ + { + name: "both up", + shards: []bgpv1alpha1.EgressShard{flagged("primary", 0x10, false), flagged(poolStandby, 0x11, true)}, + reachable: []string{primary, standby}, + nat66: map[string]string{primary: poolActive, standby: poolActive}, + nat64: map[string]string{primary: poolActive, standby: poolStandby}, + }, + { + name: "primary unreachable", + shards: []bgpv1alpha1.EgressShard{flagged("primary", 0x10, false), flagged(poolStandby, 0x11, true)}, + reachable: []string{standby}, + nat66: map[string]string{primary: poolUnreachable, standby: poolActive}, + nat64: map[string]string{primary: poolUnreachable, standby: poolActive}, + }, + { + name: "only the standby exists", + shards: []bgpv1alpha1.EgressShard{flagged(poolStandby, 0x11, true)}, + reachable: []string{standby}, + nat66: map[string]string{standby: poolActive}, + nat64: map[string]string{standby: poolActive}, + }, + { + name: "no standby flags", + shards: []bgpv1alpha1.EgressShard{flagged("primary", 0x10, false), flagged("other", 0x12, false)}, + reachable: []string{primary, other}, + nat66: map[string]string{primary: poolActive, other: poolActive}, + nat64: map[string]string{primary: poolActive, other: poolActive}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := newFixture(t, testHashed, nil, []netip.Prefix{testNSP}) + f.g.shards = &fakeLister{shards: tt.shards, synced: true} + up := map[string]bool{} + for _, s := range tt.reachable { + up[s] = true + } + t.Cleanup(egressroutemap.OverrideResolverForTest( + func(sid net.IP) (int, net.HardwareAddr, net.HardwareAddr, error) { + base, _ := egressroutemap.BaseShardSID(sid) + if !up[base.String()] { + return 0, nil, nil, errInjected + } + return 7, net.HardwareAddr{2, 0, 0, 0, 0, 1}, net.HardwareAddr{2, 0, 0, 0, 0, 2}, nil + })) + + if err := f.sweep(t); err != nil { + t.Fatal(err) + } + got := poolStates(t, f) + if !reflect.DeepEqual(got[egressroutemap.NAT66Class], tt.nat66) { + t.Errorf("NAT66 members = %v, want %v", got[egressroutemap.NAT66Class], tt.nat66) + } + if !reflect.DeepEqual(got[nat64NSP], tt.nat64) { + t.Errorf("NAT64 members = %v, want %v", got[nat64NSP], tt.nat64) + } + }) + } +} diff --git a/internal/plumbing/ebpf/egressroutemap/shardgroup.go b/internal/plumbing/ebpf/egressroutemap/shardgroup.go index a9848687..8cd1cda4 100644 --- a/internal/plumbing/ebpf/egressroutemap/shardgroup.go +++ b/internal/plumbing/ebpf/egressroutemap/shardgroup.go @@ -53,6 +53,8 @@ const ( shardFlagAlive uint32 = 1 << 0 shardFlagDraining uint32 = 1 << 1 shardFlagLocal uint32 = 1 << 2 + shardFlagStandby uint32 = 1 << 3 + shardFlagHeld uint32 = 1 << 4 classKindNAT66 uint8 = 1 classKindNAT64 uint8 = 2 @@ -181,6 +183,12 @@ type ShardCandidate struct { // takes no tenant from this node, the same as in ordered mode, but it is // reported as local rather than unreachable, since nothing is wrong. Local bool + // Standby marks a NAT64 standby shard: its IPv4 address is attracted to + // a sibling shard's node while that sibling is up, so replies to it never + // reach it. In a NAT64 group, Apply holds a standby out of the class while + // any shard without the flag is alive, and lets it serve only when none + // is. A NAT66 group ignores it. + Standby bool } // ShardState reports one group member. @@ -197,6 +205,12 @@ type ShardState struct { Draining bool // Local echoes the candidate's Local. Local bool + // Standby echoes the candidate's Standby. + Standby bool + // Held reports that a standby shard is held out of the class because a + // shard without Standby is alive to serve it. A held shard resolves but + // is not alive: it takes no new tenant and no pin to it is honoured. + Held bool // Err is why the SID did not resolve, when it did not. Err error // Assigned reports that this Apply gave the shard its slot, so nothing @@ -204,7 +218,8 @@ type ShardState struct { Assigned bool } -// Active reports whether the shard takes new tenant addresses. +// Active reports whether the shard takes new tenant addresses. A held shard +// is never alive, so it is never active. func (s ShardState) Active() bool { return s.Alive && !s.Draining } // GroupResult reports what Apply found and published. @@ -405,6 +420,8 @@ func (g *ShardGroupTable) Members(groupID uint32) ([]ShardState, error) { Alive: s.Flags&shardFlagAlive != 0, Draining: s.Flags&shardFlagDraining != 0, Local: s.Flags&shardFlagLocal != 0, + Standby: s.Flags&shardFlagStandby != 0, + Held: s.Flags&shardFlagHeld != 0, }) } return members, nil @@ -445,6 +462,14 @@ func (a *allocator) take() uint32 { // Every node computes the same table from the same set, whatever slots the // shards occupy locally. // +// In a NAT64 group, a standby candidate that resolves is held rather than +// alive while any candidate without Standby is alive, draining or not: a +// draining sibling still attracts the standby's replies. A held shard keeps +// its slot and generation but is not alive, so it gets no share of the Maglev +// table and the datapath stops honouring every pin to it. Each Apply decides +// afresh, so a standby takes tenants as soon as its last alive sibling stops +// resolving and is held again once one resolves. +// // A shard keeps its slot, and the slot keeps its generation, for as long as // the shard stays a candidate of the same class, so a rebuild leaves its pins // intact. A new shard takes the lowest free slot under a fresh generation. @@ -476,6 +501,7 @@ func (g *ShardGroupTable) Apply(groupID uint32, policy GroupPolicy, candidates [ assignment := assignSlots(previous, wanted) resolved := resolveShards(wanted) + hold := policy.Class.NAT64 && anyPrimaryAlive(wanted, resolved) var ( value prog.UsidEgressShardGroupValue @@ -491,7 +517,9 @@ func (g *ShardGroupTable) Apply(groupID uint32, policy GroupPolicy, candidates [ if assigned { member.Generation = alloc.take() } - if !c.local && hop.err == nil { + held := hold && c.standby && !c.local && hop.err == nil + alive := !c.local && hop.err == nil && !held + if alive { member.Flags |= shardFlagAlive member.LinkIfindex = uint32(hop.hop.link) //nolint:gosec // a kernel ifindex copy(member.Dmac[:], hop.hop.dmac) @@ -503,11 +531,18 @@ func (g *ShardGroupTable) Apply(groupID uint32, policy GroupPolicy, candidates [ if c.local { member.Flags |= shardFlagLocal } + if c.standby && policy.Class.NAT64 { + member.Flags |= shardFlagStandby + } + if held { + member.Flags |= shardFlagHeld + } value.Shards[slot] = member state := ShardState{ - SID: c.sid, Slot: slot, Generation: member.Generation, Alive: !c.local && hop.err == nil, - Draining: c.draining, Local: c.local, Err: hop.err, Assigned: assigned, + SID: c.sid, Slot: slot, Generation: member.Generation, Alive: alive, + Draining: c.draining, Local: c.local, Standby: member.Flags&shardFlagStandby != 0, Held: held, + Err: hop.err, Assigned: assigned, } result.Shards = append(result.Shards, state) if state.Active() { @@ -562,6 +597,7 @@ func (g *ShardGroupTable) Close(groupID uint32) (bool, error) { SID: net.IP(netip.AddrFrom16(s.Sid).AsSlice()), Draining: s.Flags&shardFlagDraining != 0, Local: s.Flags&shardFlagLocal != 0, + Standby: s.Flags&shardFlagStandby != 0, }) } } @@ -608,10 +644,12 @@ type candidate struct { sid netip.Addr draining bool local bool + standby bool } // canonicalCandidates reduces candidates to base SIDs, ordered by SID. A -// shard listed twice is draining, or local, if either listing says so. +// shard listed twice is draining, local, or a standby, if either listing says +// so. func canonicalCandidates(candidates []ShardCandidate) ([]candidate, error) { bySID := map[netip.Addr]candidate{} for _, c := range candidates { @@ -620,7 +658,10 @@ func canonicalCandidates(candidates []ShardCandidate) ([]candidate, error) { return nil, fmt.Errorf("egressroutemap: %w", err) } prev := bySID[base] - bySID[base] = candidate{sid: base, draining: prev.draining || c.Draining, local: prev.local || c.Local} + bySID[base] = candidate{ + sid: base, draining: prev.draining || c.Draining, local: prev.local || c.Local, + standby: prev.standby || c.Standby, + } } out := make([]candidate, 0, len(bySID)) for _, c := range bySID { @@ -662,6 +703,18 @@ func assignSlots(slots [MaxShardsPerGroup]prog.UsidEgressShardMember, wanted []c return assignment } +// anyPrimaryAlive reports whether a wanted shard without Standby resolved, so +// it can serve the class and the standbys are held. A local shard does not +// count: this node cannot send it tenants. +func anyPrimaryAlive(wanted []candidate, resolved map[netip.Addr]resolution) bool { + for _, c := range wanted { + if !c.standby && !c.local && resolved[c.sid].err == nil { + return true + } + } + return false +} + // resolveShards resolves every wanted shard's next hop, except a local one's. func resolveShards(wanted []candidate) map[netip.Addr]resolution { out := make(map[netip.Addr]resolution, len(wanted)) diff --git a/internal/plumbing/ebpf/egressroutemap/shardgroup_datapath_test.go b/internal/plumbing/ebpf/egressroutemap/shardgroup_datapath_test.go index 36350832..6ed6ce28 100644 --- a/internal/plumbing/ebpf/egressroutemap/shardgroup_datapath_test.go +++ b/internal/plumbing/ebpf/egressroutemap/shardgroup_datapath_test.go @@ -599,3 +599,55 @@ func TestGroupDatapath_CountsVRFEgress(t *testing.T) { t.Errorf("vrf_traffic_counters = %+v, want %+v", got, want) } } + +// TestGroupDatapath_HeldStandbyLosesItsPins checks that a tenant pinned to a +// NAT64 standby while the primary was down is re-placed on the primary once it +// returns and the standby is held, rather than kept on a shard whose replies +// the primary's node attracts, and that it goes back to the standby when the +// primary fails again. +func TestGroupDatapath_HeldStandbyLosesItsPins(t *testing.T) { + dp := loadDatapath(t) + primary, standby := shardSID(0x10), shardSID(0x11) + nat64 := netip.MustParsePrefix("64:ff9b::/96") + const nat64Group = NAT66ShardGroup + 1 + route := &net.IPNet{IP: nat64.Addr().AsSlice(), Mask: net.CIDRMask(nat64.Bits(), 128)} + if err := dp.routes.RegisterGroup(dpTableID, route, nat64Group, dpArgument); err != nil { + t.Fatalf("RegisterGroup(%s): %v", nat64, err) + } + policy := GroupPolicy{Class: NAT64Class(nat64), PinIdle: time.Hour} + in := []ShardCandidate{{SID: net.IP(primary.AsSlice())}, {SID: net.IP(standby.AsSlice()), Standby: true}} + apply := func() { + t.Helper() + if _, err := dp.groups.Apply(nat64Group, policy, in); err != nil { + t.Fatalf("Apply: %v", err) + } + } + pkt := func(port uint16) []byte { + return udp6(tenant(1), netip.MustParseAddr("64:ff9b::c000:201"), port, 443) + } + + reachable(t, standby) // the primary is down + apply() + if verdict, got, _ := dp.send(t, pkt(1)); verdict != tcActRedirect || got != standby { + t.Fatalf("with the primary down: verdict %d toward %s, want redirect to the standby %s", verdict, got, standby) + } + + reachable(t, primary, standby) + apply() + if verdict, got, _ := dp.send(t, pkt(2)); verdict != tcActRedirect || got != primary { + t.Fatalf("after the primary returned: verdict %d toward %s, want the pinned tenant re-placed on %s", + verdict, got, primary) + } + if got := stat(t, dp.objs.EgressShardStats, prog.EgressShardStatPinStale); got != 1 { + t.Errorf("egress_shard_stats[pin_stale] = %d, want 1: the pin to the held standby must be rejected", got) + } + if _, got, _ := dp.send(t, pkt(3)); got != primary { + t.Errorf("next packet went to %s, want it to follow its new pin to %s", got, primary) + } + + reachable(t, standby) + apply() + if verdict, got, _ := dp.send(t, pkt(4)); verdict != tcActRedirect || got != standby { + t.Errorf("after the primary failed again: verdict %d toward %s, want the standby %s", verdict, got, standby) + } +} diff --git a/internal/plumbing/ebpf/egressroutemap/shardgroup_standby_test.go b/internal/plumbing/ebpf/egressroutemap/shardgroup_standby_test.go new file mode 100644 index 00000000..4ccacf51 --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/shardgroup_standby_test.go @@ -0,0 +1,229 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "net" + "net/netip" + "testing" + "time" +) + +var standbyTestNAT64 = NAT64Class(netip.MustParsePrefix("64:ff9b::/96")) + +// standbyShard describes one candidate in a standby test case. +type standbyShard struct { + sid netip.Addr + standby bool + draining bool + local bool + reachable bool +} + +// wantShard is the state Apply must report, and publish, for one shard. +type wantShard struct { + active bool + held bool + // unreachable is a shard that is neither alive nor held because its SID + // did not resolve. + unreachable bool +} + +func standbyCandidates(shards []standbyShard) (in []ShardCandidate, resolves []netip.Addr) { + for _, s := range shards { + in = append(in, ShardCandidate{ + SID: net.IP(s.sid.AsSlice()), Standby: s.standby, Draining: s.draining, Local: s.local, + }) + if s.reachable { + resolves = append(resolves, s.sid) + } + } + return in, resolves +} + +// TestApplyNAT64Standby checks which shards a group spreads tenants over when +// some are NAT64 standbys: a NAT64 group holds every standby out while a shard +// without the flag is alive, and a NAT66 group ignores the flag. +func TestApplyNAT64Standby(t *testing.T) { + primary, standby, other := shardSID(0x10), shardSID(0x11), shardSID(0x12) + tests := []struct { + name string + class TranslationClass + shards []standbyShard + want map[netip.Addr]wantShard + }{ + { + name: "both up: the standby is held out of NAT64", + class: standbyTestNAT64, + shards: []standbyShard{ + {sid: primary, reachable: true}, {sid: standby, standby: true, reachable: true}, + }, + want: map[netip.Addr]wantShard{primary: {active: true}, standby: {held: true}}, + }, + { + name: "both up: NAT66 ignores the flag", + class: NAT66Class, + shards: []standbyShard{ + {sid: primary, reachable: true}, {sid: standby, standby: true, reachable: true}, + }, + want: map[netip.Addr]wantShard{primary: {active: true}, standby: {active: true}}, + }, + { + name: "primary unreachable: the standby serves NAT64", + class: standbyTestNAT64, + shards: []standbyShard{ + {sid: primary}, {sid: standby, standby: true, reachable: true}, + }, + want: map[netip.Addr]wantShard{primary: {unreachable: true}, standby: {active: true}}, + }, + { + name: "only a standby exists: it serves NAT64", + class: standbyTestNAT64, + shards: []standbyShard{{sid: standby, standby: true, reachable: true}}, + want: map[netip.Addr]wantShard{standby: {active: true}}, + }, + { + name: "no standby flags: unchanged", + class: standbyTestNAT64, + shards: []standbyShard{ + {sid: primary, reachable: true}, {sid: other, reachable: true}, + }, + want: map[netip.Addr]wantShard{primary: {active: true}, other: {active: true}}, + }, + { + name: "draining primary still attracts the standby's replies", + class: standbyTestNAT64, + shards: []standbyShard{ + {sid: primary, draining: true, reachable: true}, {sid: standby, standby: true, reachable: true}, + }, + want: map[netip.Addr]wantShard{primary: {}, standby: {held: true}}, + }, + { + name: "a local primary takes no tenant from here, so the standby serves", + class: standbyTestNAT64, + shards: []standbyShard{ + {sid: primary, local: true}, {sid: standby, standby: true, reachable: true}, + }, + want: map[netip.Addr]wantShard{primary: {}, standby: {active: true}}, + }, + { + name: "an unreachable standby reports unreachable, not held", + class: standbyTestNAT64, + shards: []standbyShard{ + {sid: primary, reachable: true}, {sid: standby, standby: true}, + }, + want: map[netip.Addr]wantShard{primary: {active: true}, standby: {unreachable: true}}, + }, + { + name: "one live primary of two holds the standby", + class: standbyTestNAT64, + shards: []standbyShard{ + {sid: primary}, {sid: other, reachable: true}, {sid: standby, standby: true, reachable: true}, + }, + want: map[netip.Addr]wantShard{primary: {unreachable: true}, other: {active: true}, standby: {held: true}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + in, resolves := standbyCandidates(tt.shards) + reachable(t, resolves...) + + result := applyOK(t, g, GroupPolicy{Class: tt.class}, in) + v := published(t, store) + activeSlots := map[uint8]bool{} + for sid, want := range tt.want { + s := slotOf(t, result, sid) + if s.Active() != want.active || s.Held != want.held || (s.Err != nil) != want.unreachable { + t.Errorf("%s: state %+v, want active %v held %v unreachable %v", + sid, s, want.active, want.held, want.unreachable) + } + if s.Held && s.Alive { + t.Errorf("%s: held and alive at once", sid) + } + flags := v.Shards[s.Slot].Flags + if (flags&shardFlagHeld != 0) != want.held || (flags&shardFlagAlive != 0 && want.held) { + t.Errorf("%s: published flags %#x, want held %v and never alive while held", sid, flags, want.held) + } + wantStandbyFlag := tt.class.NAT64 && sid == standby + if (flags&shardFlagStandby != 0) != wantStandbyFlag { + t.Errorf("%s: published flags %#x, want standby flag %v", sid, flags, wantStandbyFlag) + } + if want.active { + activeSlots[uint8(s.Slot)] = true //nolint:gosec // < 32 + } + } + if int(v.Active) != len(activeSlots) || int(v.Candidates) != len(tt.shards) { + t.Errorf("snapshot active %d candidates %d, want %d and %d", + v.Active, v.Candidates, len(activeSlots), len(tt.shards)) + } + for i, slot := range v.Maglev { + if len(activeSlots) == 0 { + if slot != shardSlotNone { + t.Fatalf("Maglev slot %d = %d, want empty", i, slot) + } + continue + } + if !activeSlots[slot] { + t.Fatalf("Maglev slot %d = %d, want one of the active slots %v", i, slot, activeSlots) + } + } + }) + } +} + +// TestApplyNAT64StandbyFollowsPrimaryLiveness checks that each Apply decides +// again: the standby serves once the primary stops resolving and is held again +// when it returns, keeping its slot and generation throughout, and that Close +// keeps the standby flag so a retired group still holds it. +func TestApplyNAT64StandbyFollowsPrimaryLiveness(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + primary, standby := shardSID(0x10), shardSID(0x11) + policy := GroupPolicy{Class: standbyTestNAT64, PinIdle: time.Hour} + in := []ShardCandidate{{SID: net.IP(primary.AsSlice())}, {SID: net.IP(standby.AsSlice()), Standby: true}} + + reachable(t, primary, standby) + first := slotOf(t, applyOK(t, g, policy, in), standby) + if !first.Held { + t.Fatalf("standby %+v, want held while the primary resolves", first) + } + + reachable(t, standby) + failover := applyOK(t, g, policy, in) + if s := slotOf(t, failover, standby); !s.Active() || s.Held || s.Slot != first.Slot || + s.Generation != first.Generation { + t.Errorf("standby after the primary failed %+v, want active in slot %d generation %d", + s, first.Slot, first.Generation) + } + if !failover.Changed { + t.Error("losing the primary published nothing") + } + + reachable(t, primary, standby) + back := applyOK(t, g, policy, in) + if s := slotOf(t, back, standby); !s.Held || s.Active() || s.Slot != first.Slot || + s.Generation != first.Generation { + t.Errorf("standby after the primary returned %+v, want held in slot %d generation %d", + s, first.Slot, first.Generation) + } + if back.Active() != 1 || !slotOf(t, back, primary).Active() { + t.Errorf("after the primary returned, active = %d, want the primary alone", back.Active()) + } + + if _, err := g.Close(NAT66ShardGroup); err != nil { + t.Fatal(err) + } + members, err := g.Members(NAT66ShardGroup) + if err != nil { + t.Fatal(err) + } + for _, m := range members { + if (m.SID == standby) != m.Standby || (m.SID == standby) != m.Held { + t.Errorf("member after Close %+v: standby and held = %v, want %v", m, m.Standby, m.SID == standby) + } + } +} diff --git a/internal/plumbing/ebpf/metrics/egressshard.go b/internal/plumbing/ebpf/metrics/egressshard.go index f79d8abf..7339dc6e 100644 --- a/internal/plumbing/ebpf/metrics/egressshard.go +++ b/internal/plumbing/ebpf/metrics/egressshard.go @@ -27,6 +27,7 @@ const ( poolStateDraining = "draining" poolStateUnreachable = "unreachable" poolStateLocal = "local" + poolStateStandby = "standby" poolStateIneligible = "ineligible" ) @@ -54,8 +55,9 @@ var ( prometheus.BuildFQName(egressNamespace, "egress_pool", "members"), "The cluster's egress shards as this node's group for each translation class sees them: active shards "+ "take new tenant addresses, draining ones serve only the tenants pinned to them, unreachable ones "+ - "have no route from this node, local ones run on this node and take no tenant from it, and "+ - "ineligible ones cannot translate the class (no address in its family, or another NAT64 prefix).", + "have no route from this node, local ones run on this node and take no tenant from it, standby ones "+ + "are NAT64 standbys held out of the class while a sibling serves it, and ineligible ones cannot "+ + "translate the class (no address in its family, or another NAT64 prefix).", []string{labelClass, "state"}, nil, ) egressPoolMinActiveDesc = prometheus.NewDesc( @@ -138,12 +140,14 @@ func (c *EgressShardCollector) Collect(ch chan<- prometheus.Metric) { } states := map[string]int{ poolStateActive: 0, poolStateDraining: 0, poolStateUnreachable: 0, poolStateLocal: 0, - poolStateIneligible: status.Ineligible, + poolStateStandby: 0, poolStateIneligible: status.Ineligible, } for _, member := range members { switch { case member.Local: states[poolStateLocal]++ + case member.Held: + states[poolStateStandby]++ case !member.Alive: states[poolStateUnreachable]++ case member.Draining: diff --git a/internal/plumbing/ebpf/metrics/egressshard_test.go b/internal/plumbing/ebpf/metrics/egressshard_test.go index fbe37a4a..d154ff1a 100644 --- a/internal/plumbing/ebpf/metrics/egressshard_test.go +++ b/internal/plumbing/ebpf/metrics/egressshard_test.go @@ -54,7 +54,12 @@ func TestEgressShardCollector(t *testing.T) { // not open. nat64 := prog.UsidEgressShardGroupValue{Flags: 1 | 4, Ineligible: 3, ClassKind: 2, ClassPrefix: sidBytes("2001:db8:64::")} - store := snapshotStore{0: nat66, 1: nat64, 2: {ClassKind: 2}} + // A NAT64 class its primary serves, with the standby held out. + wkp := prog.UsidEgressShardGroupValue{Flags: 1, Candidates: 2, Active: 1, ClassKind: 2, + ClassPrefix: sidBytes("64:ff9b::")} + wkp.Shards[0] = prog.UsidEgressShardMember{Sid: sidBytes("2001:db8:ff09:20:e000::"), Flags: 1, Generation: 5} + wkp.Shards[1] = prog.UsidEgressShardMember{Sid: sidBytes("2001:db8:ff09:21:e000::"), Flags: 8 | 16, Generation: 6} + store := snapshotStore{0: nat66, 1: nat64, 2: {ClassKind: 2}, 3: wkp} counters := perCPUTable{ uint32(0): []prog.UsidEgressShardCounter{{Packets: 3, Bytes: 300}, {Packets: 4, Bytes: 400}}, } @@ -69,6 +74,7 @@ func TestEgressShardCollector(t *testing.T) { got[m.name] = m.value } const nat64Class = `class="nat64:2001:db8:64::/96"` + const wkpClass = `class="nat64:64:ff9b::/96"` want := map[string]float64{ `galactic_cni_egress_pool_min_active`: 2, `galactic_cni_egress_pool_enabled{class="nat66"}`: 1, @@ -78,6 +84,10 @@ func TestEgressShardCollector(t *testing.T) { `galactic_cni_egress_pool_members{class="nat66",state="unreachable"}`: 1, `galactic_cni_egress_pool_members{class="nat66",state="local"}`: 1, `galactic_cni_egress_pool_members{class="nat66",state="ineligible"}`: 0, + `galactic_cni_egress_pool_members{class="nat66",state="standby"}`: 0, + `galactic_cni_egress_pool_members{` + wkpClass + `,state="active"}`: 1, + `galactic_cni_egress_pool_members{` + wkpClass + `,state="standby"}`: 1, + `galactic_cni_egress_pool_members{` + wkpClass + `,state="unreachable"}`: 0, `galactic_cni_egress_pool_members{` + nat64Class + `,state="active"}`: 0, `galactic_cni_egress_pool_members{` + nat64Class + `,state="ineligible"}`: 3, `galactic_cni_egress_shard_packets_total{class="nat66",shard_sid="2001:db8:ff09:10:e000::"}`: 7, diff --git a/internal/plumbing/ebpf/prog/usid.c b/internal/plumbing/ebpf/prog/usid.c index 8c3893ca..adcc2b22 100644 --- a/internal/plumbing/ebpf/prog/usid.c +++ b/internal/plumbing/ebpf/prog/usid.c @@ -723,9 +723,16 @@ struct egress_route_value { // // LOCAL means the shard runs on this node. The control plane sets it for the // metrics alone and never sets ALIVE with it, so the datapath ignores it. +// +// STANDBY marks a NAT64 standby shard, and HELD one the control plane is +// keeping out of the class because a sibling without STANDBY is alive. Both +// are for the metrics alone: the control plane never sets ALIVE with HELD, so +// a held shard is absent from the Maglev table and its pins are stale. #define USID_EGRESS_SHARD_ALIVE (1u << 0) #define USID_EGRESS_SHARD_DRAINING (1u << 1) #define USID_EGRESS_SHARD_LOCAL (1u << 2) +#define USID_EGRESS_SHARD_STANDBY (1u << 3) +#define USID_EGRESS_SHARD_HELD (1u << 4) // struct egress_shard_member is one group member, with the link and L2 // addresses resolved ahead of time for the same reason egress_route_value