From 2c27b50b5385d33d2bdae4bf4ecc7ca205e49b81 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Wed, 14 Oct 2026 16:24:00 -0400 Subject: [PATCH] feat(api): Add nat64Standby to EgressShard An edge site runs two egress shards, and both announce the site's IPv4 unicast prefix upstream. The standby prepends its announcement, so every IPv4 reply, including replies to the standby's own shardAddressIPv4, arrives at the primary, which holds no session for it and drops it. NAT66 is unaffected because each shard announces its own IPv6 /128. Compute nodes that spread egress across shards cannot see this from the shard spec, so they hash NAT64 tenants onto the standby and those tenants lose NAT64 entirely. Add spec.nat64Standby. Compute nodes that spread egress keep NAT64 tenants off a standby shard while any shard without the flag can serve that NAT64 class, and use it only when none can. NAT66 placement is unaffected, and compute nodes in ordered mode ignore it. The field is optional and mutable, and a CEL rule requires shardAddressIPv4 alongside it. Related to datum-cloud/infra#7033 Co-Authored-By: Claude Opus 5.5 --- api/v1alpha1/egressshard_types.go | 19 +++++++ api/v1alpha1/egressshard_types_test.go | 6 ++- .../network.datumapis.com_egressshards.yaml | 21 ++++++++ docs/api/bgp.md | 1 + docs/api/gateway.md | 1 + .../chainsaw-test.yaml | 51 ++++++++++++++++++- 6 files changed, 97 insertions(+), 2 deletions(-) diff --git a/api/v1alpha1/egressshard_types.go b/api/v1alpha1/egressshard_types.go index 4c8f6f3..ebb027d 100644 --- a/api/v1alpha1/egressshard_types.go +++ b/api/v1alpha1/egressshard_types.go @@ -214,6 +214,7 @@ type AddressClaimRef struct { // +kubebuilder:validation:XValidation:rule="!has(oldSelf.shardAddressIPv6ClaimRef) || has(self.shardAddressIPv6ClaimRef)",message="shardAddressIPv6ClaimRef cannot be unassigned once assigned" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.shardAddressIPv4ClaimRef) || has(self.shardAddressIPv4ClaimRef)",message="shardAddressIPv4ClaimRef cannot be unassigned once assigned" // +kubebuilder:validation:XValidation:rule="!(has(self.translateWellKnownPrefix) && self.translateWellKnownPrefix) || (has(self.nat64Prefix) && size(self.nat64Prefix) > 0)",message="translateWellKnownPrefix requires nat64Prefix" +// +kubebuilder:validation:XValidation:rule="!(has(self.nat64Standby) && self.nat64Standby) || (has(self.shardAddressIPv4) && size(self.shardAddressIPv4) > 0)",message="nat64Standby requires shardAddressIPv4" type EgressShardSpec struct { // TargetRef identifies the Node this shard executes on. // +kubebuilder:validation:Required @@ -330,6 +331,24 @@ type EgressShardSpec struct { // it. // +optional Drain bool `json:"drain,omitempty"` + + // NAT64Standby marks this shard's ShardAddressIPv4 as not attracted to its + // node while a sibling NAT64 shard is up: replies to it arrive at that + // sibling, which holds no session for them and drops them. Compute nodes + // that spread egress across a cluster's shards therefore keep NAT64 + // tenants off a standby shard while any shard without the flag can serve + // that NAT64 class, and place them on it only when none can. Requires + // ShardAddressIPv4. + // + // It changes nothing on the shard itself, and NAT66 placement is + // unaffected: the shard's IPv6 address is advertised from its own node and + // keeps taking NAT66 tenants. Mutable, so the flag can follow the + // upstream announcement it describes. + // + // Compute nodes that send each tenant to a single configured shard ignore + // it. + // +optional + NAT64Standby bool `json:"nat64Standby,omitempty"` } // EgressShardStatus defines the observed state of an EgressShard. diff --git a/api/v1alpha1/egressshard_types_test.go b/api/v1alpha1/egressshard_types_test.go index fb6d2c2..21a556f 100644 --- a/api/v1alpha1/egressshard_types_test.go +++ b/api/v1alpha1/egressshard_types_test.go @@ -144,6 +144,7 @@ func TestEgressShardSpecFieldNames(t *testing.T) { orig.Spec.NAT64Prefix = "64:ff9b::/96" orig.Spec.TranslateWellKnownPrefix = true orig.Spec.Drain = true + orig.Spec.NAT64Standby = true data, err := json.Marshal(orig.Spec) if err != nil { @@ -172,6 +173,9 @@ func TestEgressShardSpecFieldNames(t *testing.T) { if raw, ok := m["drain"]; !ok || raw != true { t.Errorf("drain: got %v, want true", raw) } + if raw, ok := m["nat64Standby"]; !ok || raw != true { + t.Errorf("nat64Standby: got %v, want true", raw) + } if _, ok := m["targetRef"]; !ok { t.Error("expected \"targetRef\" key to be present") } @@ -213,7 +217,7 @@ func TestEgressShardSpecOmitEmpty(t *testing.T) { for _, key := range []string{ "shardSID", "shardAddressIPv6", "shardAddressIPv4", "nat64Prefix", "shardAddressIPv6ClaimRef", "shardAddressIPv4ClaimRef", - "translateWellKnownPrefix", "drain", + "translateWellKnownPrefix", "drain", "nat64Standby", } { if _, ok := m[key]; ok { t.Errorf("expected %q key to be absent when empty", key) diff --git a/config/crd/network.datumapis.com_egressshards.yaml b/config/crd/network.datumapis.com_egressshards.yaml index 8a6e45d..a08d17a 100644 --- a/config/crd/network.datumapis.com_egressshards.yaml +++ b/config/crd/network.datumapis.com_egressshards.yaml @@ -159,6 +159,24 @@ spec: rule: self == '' || isCIDR(self) - message: nat64Prefix is immutable once assigned rule: self == oldSelf || oldSelf == '' + nat64Standby: + description: |- + NAT64Standby marks this shard's ShardAddressIPv4 as not attracted to its + node while a sibling NAT64 shard is up: replies to it arrive at that + sibling, which holds no session for them and drops them. Compute nodes + that spread egress across a cluster's shards therefore keep NAT64 + tenants off a standby shard while any shard without the flag can serve + that NAT64 class, and place them on it only when none can. Requires + ShardAddressIPv4. + + It changes nothing on the shard itself, and NAT66 placement is + unaffected: the shard's IPv6 address is advertised from its own node and + keeps taking NAT66 tenants. Mutable, so the flag can follow the + upstream announcement it describes. + + Compute nodes that send each tenant to a single configured shard ignore + it. + type: boolean shardAddressIPv4: description: |- ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this @@ -362,6 +380,9 @@ spec: - message: translateWellKnownPrefix requires nat64Prefix rule: '!(has(self.translateWellKnownPrefix) && self.translateWellKnownPrefix) || (has(self.nat64Prefix) && size(self.nat64Prefix) > 0)' + - message: nat64Standby requires shardAddressIPv4 + rule: '!(has(self.nat64Standby) && self.nat64Standby) || (has(self.shardAddressIPv4) + && size(self.shardAddressIPv4) > 0)' status: description: |- EgressShardStatus defines the observed state of an EgressShard. diff --git a/docs/api/bgp.md b/docs/api/bgp.md index 39b681e..98eda45 100644 --- a/docs/api/bgp.md +++ b/docs/api/bgp.md @@ -1068,6 +1068,7 @@ _Appears in:_ | `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — a Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. TranslateWellKnownPrefix adds the RFC 6052
Well-Known Prefix alongside it. It must be the prefix the resolver
synthesizes into; a shard translating for a different one is a
blackhole with no symptom on either side.
Set together with ShardAddressIPv4 or not at all: an address with no
prefix has nothing to translate for, and a prefix with no address has
nothing to translate into. Write-once for the same reason the addresses
are: a shard that starts translating a different prefix blackholes every
destination the resolver already synthesized into the old one. | | | | `translateWellKnownPrefix` _boolean_ | TranslateWellKnownPrefix additionally translates destinations inside the
RFC 6052 Well-Known Prefix 64:ff9b::/96 to IPv4 from ShardAddressIPv4, so
clients using a public DNS64 resolver reach IPv4 destinations. Requires
NAT64Prefix.
RFC 6052 §3.1 forbids the Well-Known Prefix from representing non-global
IPv4 addresses; the implementation must refuse such destinations.
Mutable, unlike NAT64Prefix. Disabling it blackholes every destination
clients already resolved into the Well-Known Prefix until their DNS TTLs
expire. | | | | `drain` _boolean_ | Drain stops this shard from taking new tenants while it keeps serving
the ones it has. Compute nodes that spread egress across a cluster's
shards give a draining shard no share of new tenant addresses, but keep
sending it every flow already pinned to it until that flow goes idle, so
established sessions finish where their translation state lives.
It changes nothing on the shard itself: the shard keeps translating,
advertising its SID and receiving replies. A shard is safe to remove
once its session count has fallen to zero. Clearing it puts the shard
back into rotation.
Compute nodes that send each tenant to a single configured shard ignore
it. | | | +| `nat64Standby` _boolean_ | NAT64Standby marks this shard's ShardAddressIPv4 as not attracted to its
node while a sibling NAT64 shard is up: replies to it arrive at that
sibling, which holds no session for them and drops them. Compute nodes
that spread egress across a cluster's shards therefore keep NAT64
tenants off a standby shard while any shard without the flag can serve
that NAT64 class, and place them on it only when none can. Requires
ShardAddressIPv4.
It changes nothing on the shard itself, and NAT66 placement is
unaffected: the shard's IPv6 address is advertised from its own node and
keeps taking NAT66 tenants. Mutable, so the flag can follow the
upstream announcement it describes.
Compute nodes that send each tenant to a single configured shard ignore
it. | | | #### EgressShardStatus diff --git a/docs/api/gateway.md b/docs/api/gateway.md index 7e2818b..631dd38 100644 --- a/docs/api/gateway.md +++ b/docs/api/gateway.md @@ -169,6 +169,7 @@ _Appears in:_ | `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — a Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. TranslateWellKnownPrefix adds the RFC 6052
Well-Known Prefix alongside it. It must be the prefix the resolver
synthesizes into; a shard translating for a different one is a
blackhole with no symptom on either side.
Set together with ShardAddressIPv4 or not at all: an address with no
prefix has nothing to translate for, and a prefix with no address has
nothing to translate into. Write-once for the same reason the addresses
are: a shard that starts translating a different prefix blackholes every
destination the resolver already synthesized into the old one. | | | | `translateWellKnownPrefix` _boolean_ | TranslateWellKnownPrefix additionally translates destinations inside the
RFC 6052 Well-Known Prefix 64:ff9b::/96 to IPv4 from ShardAddressIPv4, so
clients using a public DNS64 resolver reach IPv4 destinations. Requires
NAT64Prefix.
RFC 6052 §3.1 forbids the Well-Known Prefix from representing non-global
IPv4 addresses; the implementation must refuse such destinations.
Mutable, unlike NAT64Prefix. Disabling it blackholes every destination
clients already resolved into the Well-Known Prefix until their DNS TTLs
expire. | | | | `drain` _boolean_ | Drain stops this shard from taking new tenants while it keeps serving
the ones it has. Compute nodes that spread egress across a cluster's
shards give a draining shard no share of new tenant addresses, but keep
sending it every flow already pinned to it until that flow goes idle, so
established sessions finish where their translation state lives.
It changes nothing on the shard itself: the shard keeps translating,
advertising its SID and receiving replies. A shard is safe to remove
once its session count has fallen to zero. Clearing it puts the shard
back into rotation.
Compute nodes that send each tenant to a single configured shard ignore
it. | | | +| `nat64Standby` _boolean_ | NAT64Standby marks this shard's ShardAddressIPv4 as not attracted to its
node while a sibling NAT64 shard is up: replies to it arrive at that
sibling, which holds no session for them and drops them. Compute nodes
that spread egress across a cluster's shards therefore keep NAT64
tenants off a standby shard while any shard without the flag can serve
that NAT64 class, and place them on it only when none can. Requires
ShardAddressIPv4.
It changes nothing on the shard itself, and NAT66 placement is
unaffected: the shard's IPv6 address is advertised from its own node and
keeps taking NAT66 tenants. Mutable, so the flag can follow the
upstream announcement it describes.
Compute nodes that send each tenant to a single configured shard ignore
it. | | | #### EgressShardStatus diff --git a/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml b/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml index 6380e20..773d2d5 100644 --- a/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml +++ b/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml @@ -12,7 +12,8 @@ spec: kind, and that an address and a claim reference are independently optional. Also verifies that translateWellKnownPrefix requires nat64Prefix and, unlike it, can be turned off again. Also verifies that drain can be set - and cleared on a shard whose addresses are fixed. Also verifies that status remains writable, + and cleared on a shard whose addresses are fixed, and that nat64Standby + requires shardAddressIPv4 and can be set and cleared. Also verifies that status remains writable, since status reports what the datapath actually programmed and must still converge. Does not require any operator to be running — every rule under test is enforced by the API server alone. @@ -423,6 +424,54 @@ spec: fi echo "OK: drain set on an addressed shard, then cleared" + - name: reject-nat64-standby-without-ipv4 + try: + - script: + content: | + set +e + OUTPUT=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 <<'EOF' + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard-standby-v6-only + spec: + targetRef: + kind: Node + name: bgp-e2e-worker + shardAddressIPv6: 2001:db8:f00d::102 + nat64Standby: true + EOF + ) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of nat64Standby with no shardAddressIPv4 but the resource was accepted" + kubectl delete egressshard e2e-egress-shard-standby-v6-only -n "$NAMESPACE" 2>/dev/null || true + exit 1 + fi + case "$OUTPUT" in + *"nat64Standby requires shardAddressIPv4"*) ;; + *) echo "ERROR: rejected for the wrong reason: $OUTPUT"; exit 1 ;; + esac + echo "OK: nat64Standby with no shardAddressIPv4 correctly rejected" + + - name: toggle-nat64-standby + try: + - script: + content: | + set -e + kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"nat64Standby":true}}' + ON=$(kubectl get egressshard e2e-egress-shard -n "$NAMESPACE" -o jsonpath='{.spec.nat64Standby}') + kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"nat64Standby":false}}' + OFF=$(kubectl get egressshard e2e-egress-shard -n "$NAMESPACE" -o jsonpath='{.spec.nat64Standby}') + if [ "$ON" != "true" ] || [ "$OFF" != "false" ]; then + echo "ERROR: nat64Standby toggled to $ON then $OFF, want true then false" + exit 1 + fi + echo "OK: nat64Standby set on a shard with an IPv4 address, then cleared" + - name: accept-status-changes try: - script: