diff --git a/api/v1alpha1/egressshard_types.go b/api/v1alpha1/egressshard_types.go index c0e5dec..4c8f6f3 100644 --- a/api/v1alpha1/egressshard_types.go +++ b/api/v1alpha1/egressshard_types.go @@ -49,6 +49,7 @@ import ( // +kubebuilder:printcolumn:name="SHARD-SID",type="string",JSONPath=".status.shardSID" // +kubebuilder:printcolumn:name="IPV6",type="string",JSONPath=".status.shardAddressIPv6" // +kubebuilder:printcolumn:name="IPV4",type="string",JSONPath=".status.shardAddressIPv4" +// +kubebuilder:printcolumn:name="DRAIN",type="boolean",JSONPath=".spec.drain" // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" type EgressShard struct { metav1.TypeMeta `json:",inline"` @@ -313,6 +314,22 @@ type EgressShardSpec struct { // expire. // +optional TranslateWellKnownPrefix bool `json:"translateWellKnownPrefix,omitempty"` + + // Drain stops this shard from taking new tenants while it keeps serving + // the ones it has. Compute nodes that spread egress across a cluster's + // shards give a draining shard no share of new tenant addresses, but keep + // sending it every flow already pinned to it until that flow goes idle, so + // established sessions finish where their translation state lives. + // + // It changes nothing on the shard itself: the shard keeps translating, + // advertising its SID and receiving replies. A shard is safe to remove + // once its session count has fallen to zero. Clearing it puts the shard + // back into rotation. + // + // Compute nodes that send each tenant to a single configured shard ignore + // it. + // +optional + Drain bool `json:"drain,omitempty"` } // EgressShardStatus defines the observed state of an EgressShard. diff --git a/api/v1alpha1/egressshard_types_test.go b/api/v1alpha1/egressshard_types_test.go index 36da54d..fb6d2c2 100644 --- a/api/v1alpha1/egressshard_types_test.go +++ b/api/v1alpha1/egressshard_types_test.go @@ -143,6 +143,7 @@ func TestEgressShardSpecFieldNames(t *testing.T) { orig.Spec.ShardAddressIPv4 = "198.51.100.7" orig.Spec.NAT64Prefix = "64:ff9b::/96" orig.Spec.TranslateWellKnownPrefix = true + orig.Spec.Drain = true data, err := json.Marshal(orig.Spec) if err != nil { @@ -168,6 +169,9 @@ func TestEgressShardSpecFieldNames(t *testing.T) { if raw, ok := m["translateWellKnownPrefix"]; !ok || raw != true { t.Errorf("translateWellKnownPrefix: got %v, want true", raw) } + if raw, ok := m["drain"]; !ok || raw != true { + t.Errorf("drain: got %v, want true", raw) + } if _, ok := m["targetRef"]; !ok { t.Error("expected \"targetRef\" key to be present") } @@ -209,7 +213,7 @@ func TestEgressShardSpecOmitEmpty(t *testing.T) { for _, key := range []string{ "shardSID", "shardAddressIPv6", "shardAddressIPv4", "nat64Prefix", "shardAddressIPv6ClaimRef", "shardAddressIPv4ClaimRef", - "translateWellKnownPrefix", + "translateWellKnownPrefix", "drain", } { if _, ok := m[key]; ok { t.Errorf("expected %q key to be absent when empty", key) diff --git a/config/crd/network.datumapis.com_egressshards.yaml b/config/crd/network.datumapis.com_egressshards.yaml index 60adfcd..8a6e45d 100644 --- a/config/crd/network.datumapis.com_egressshards.yaml +++ b/config/crd/network.datumapis.com_egressshards.yaml @@ -29,6 +29,9 @@ spec: - jsonPath: .status.shardAddressIPv4 name: IPV4 type: string + - jsonPath: .spec.drain + name: DRAIN + type: boolean - jsonPath: .metadata.creationTimestamp name: AGE type: date @@ -120,6 +123,22 @@ spec: is therefore accepted and is unattributable, which is a fact about the allocation rather than something a schema can repair. properties: + drain: + description: |- + Drain stops this shard from taking new tenants while it keeps serving + the ones it has. Compute nodes that spread egress across a cluster's + shards give a draining shard no share of new tenant addresses, but keep + sending it every flow already pinned to it until that flow goes idle, so + established sessions finish where their translation state lives. + + It changes nothing on the shard itself: the shard keeps translating, + advertising its SID and receiving replies. A shard is safe to remove + once its session count has fallen to zero. Clearing it puts the shard + back into rotation. + + Compute nodes that send each tenant to a single configured shard ignore + it. + type: boolean nat64Prefix: description: |- NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard diff --git a/docs/api/bgp.md b/docs/api/bgp.md index 71bca13..39b681e 100644 --- a/docs/api/bgp.md +++ b/docs/api/bgp.md @@ -1067,6 +1067,7 @@ _Appears in:_ | `shardAddressIPv4ClaimRef` _[AddressClaimRef](#addressclaimref)_ | ShardAddressIPv4ClaimRef records the addressing-service claim
ShardAddressIPv4 came from — see AddressClaimRef. Set it in the same
write as the address whenever that address came from a claim.
Nothing here resolves it and no component reads it to program anything;
it is the only trail from a translating address back to the allocation
accountable for it. | | | | `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — a Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. TranslateWellKnownPrefix adds the RFC 6052
Well-Known Prefix alongside it. It must be the prefix the resolver
synthesizes into; a shard translating for a different one is a
blackhole with no symptom on either side.
Set together with ShardAddressIPv4 or not at all: an address with no
prefix has nothing to translate for, and a prefix with no address has
nothing to translate into. Write-once for the same reason the addresses
are: a shard that starts translating a different prefix blackholes every
destination the resolver already synthesized into the old one. | | | | `translateWellKnownPrefix` _boolean_ | TranslateWellKnownPrefix additionally translates destinations inside the
RFC 6052 Well-Known Prefix 64:ff9b::/96 to IPv4 from ShardAddressIPv4, so
clients using a public DNS64 resolver reach IPv4 destinations. Requires
NAT64Prefix.
RFC 6052 §3.1 forbids the Well-Known Prefix from representing non-global
IPv4 addresses; the implementation must refuse such destinations.
Mutable, unlike NAT64Prefix. Disabling it blackholes every destination
clients already resolved into the Well-Known Prefix until their DNS TTLs
expire. | | | +| `drain` _boolean_ | Drain stops this shard from taking new tenants while it keeps serving
the ones it has. Compute nodes that spread egress across a cluster's
shards give a draining shard no share of new tenant addresses, but keep
sending it every flow already pinned to it until that flow goes idle, so
established sessions finish where their translation state lives.
It changes nothing on the shard itself: the shard keeps translating,
advertising its SID and receiving replies. A shard is safe to remove
once its session count has fallen to zero. Clearing it puts the shard
back into rotation.
Compute nodes that send each tenant to a single configured shard ignore
it. | | | #### EgressShardStatus diff --git a/docs/api/gateway.md b/docs/api/gateway.md index 9a4c898..7e2818b 100644 --- a/docs/api/gateway.md +++ b/docs/api/gateway.md @@ -168,6 +168,7 @@ _Appears in:_ | `shardAddressIPv4ClaimRef` _[AddressClaimRef](#addressclaimref)_ | ShardAddressIPv4ClaimRef records the addressing-service claim
ShardAddressIPv4 came from — see AddressClaimRef. Set it in the same
write as the address whenever that address came from a claim.
Nothing here resolves it and no component reads it to program anything;
it is the only trail from a translating address back to the allocation
accountable for it. | | | | `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — a Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. TranslateWellKnownPrefix adds the RFC 6052
Well-Known Prefix alongside it. It must be the prefix the resolver
synthesizes into; a shard translating for a different one is a
blackhole with no symptom on either side.
Set together with ShardAddressIPv4 or not at all: an address with no
prefix has nothing to translate for, and a prefix with no address has
nothing to translate into. Write-once for the same reason the addresses
are: a shard that starts translating a different prefix blackholes every
destination the resolver already synthesized into the old one. | | | | `translateWellKnownPrefix` _boolean_ | TranslateWellKnownPrefix additionally translates destinations inside the
RFC 6052 Well-Known Prefix 64:ff9b::/96 to IPv4 from ShardAddressIPv4, so
clients using a public DNS64 resolver reach IPv4 destinations. Requires
NAT64Prefix.
RFC 6052 §3.1 forbids the Well-Known Prefix from representing non-global
IPv4 addresses; the implementation must refuse such destinations.
Mutable, unlike NAT64Prefix. Disabling it blackholes every destination
clients already resolved into the Well-Known Prefix until their DNS TTLs
expire. | | | +| `drain` _boolean_ | Drain stops this shard from taking new tenants while it keeps serving
the ones it has. Compute nodes that spread egress across a cluster's
shards give a draining shard no share of new tenant addresses, but keep
sending it every flow already pinned to it until that flow goes idle, so
established sessions finish where their translation state lives.
It changes nothing on the shard itself: the shard keeps translating,
advertising its SID and receiving replies. A shard is safe to remove
once its session count has fallen to zero. Clearing it puts the shard
back into rotation.
Compute nodes that send each tenant to a single configured shard ignore
it. | | | #### EgressShardStatus diff --git a/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml b/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml index 6452818..6380e20 100644 --- a/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml +++ b/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml @@ -11,7 +11,8 @@ spec: rejected. Also verifies that a claim reference defaults its apiGroup and kind, and that an address and a claim reference are independently optional. Also verifies that translateWellKnownPrefix requires nat64Prefix - and, unlike it, can be turned off again. Also verifies that status remains writable, + and, unlike it, can be turned off again. Also verifies that drain can be set + and cleared on a shard whose addresses are fixed. Also verifies that status remains writable, since status reports what the datapath actually programmed and must still converge. Does not require any operator to be running — every rule under test is enforced by the API server alone. @@ -405,6 +406,23 @@ spec: fi echo "OK: translateWellKnownPrefix enabled with nat64Prefix set, then disabled" + - name: toggle-drain + try: + - script: + content: | + set -e + kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"drain":true}}' + ON=$(kubectl get egressshard e2e-egress-shard -n "$NAMESPACE" -o jsonpath='{.spec.drain}') + kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"drain":false}}' + OFF=$(kubectl get egressshard e2e-egress-shard -n "$NAMESPACE" -o jsonpath='{.spec.drain}') + if [ "$ON" != "true" ] || [ "$OFF" != "false" ]; then + echo "ERROR: drain toggled to $ON then $OFF, want true then false" + exit 1 + fi + echo "OK: drain set on an addressed shard, then cleared" + - name: accept-status-changes try: - script: