feat: Add drain to EgressShard

api/v1alpha1/egressshard_types.go

@@ -49,6 +49,7 @@ import (
 // +kubebuilder:printcolumn:name="SHARD-SID",type="string",JSONPath=".status.shardSID"
 // +kubebuilder:printcolumn:name="IPV6",type="string",JSONPath=".status.shardAddressIPv6"
 // +kubebuilder:printcolumn:name="IPV4",type="string",JSONPath=".status.shardAddressIPv4"
+// +kubebuilder:printcolumn:name="DRAIN",type="boolean",JSONPath=".spec.drain"
 // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp"
 type EgressShard struct {
 	metav1.TypeMeta   `json:",inline"`
@@ -313,6 +314,22 @@ type EgressShardSpec struct {
 	// expire.
 	// +optional
 	TranslateWellKnownPrefix bool `json:"translateWellKnownPrefix,omitempty"`
+
+	// Drain stops this shard from taking new tenants while it keeps serving
+	// the ones it has. Compute nodes that spread egress across a cluster's
+	// shards give a draining shard no share of new tenant addresses, but keep
+	// sending it every flow already pinned to it until that flow goes idle, so
+	// established sessions finish where their translation state lives.
+	//
+	// It changes nothing on the shard itself: the shard keeps translating,
+	// advertising its SID and receiving replies. A shard is safe to remove
+	// once its session count has fallen to zero. Clearing it puts the shard
+	// back into rotation.
+	//
+	// Compute nodes that send each tenant to a single configured shard ignore
+	// it.
+	// +optional
+	Drain bool `json:"drain,omitempty"`
 }
 
 // EgressShardStatus defines the observed state of an EgressShard.

api/v1alpha1/egressshard_types_test.go

@@ -143,6 +143,7 @@ func TestEgressShardSpecFieldNames(t *testing.T) {
 	orig.Spec.ShardAddressIPv4 = "198.51.100.7"
 	orig.Spec.NAT64Prefix = "64:ff9b::/96"
 	orig.Spec.TranslateWellKnownPrefix = true
+	orig.Spec.Drain = true
 
 	data, err := json.Marshal(orig.Spec)
 	if err != nil {
@@ -168,6 +169,9 @@ func TestEgressShardSpecFieldNames(t *testing.T) {
 	if raw, ok := m["translateWellKnownPrefix"]; !ok || raw != true {
 		t.Errorf("translateWellKnownPrefix: got %v, want true", raw)
 	}
+	if raw, ok := m["drain"]; !ok || raw != true {
+		t.Errorf("drain: got %v, want true", raw)
+	}
 	if _, ok := m["targetRef"]; !ok {
 		t.Error("expected \"targetRef\" key to be present")
 	}
@@ -209,7 +213,7 @@ func TestEgressShardSpecOmitEmpty(t *testing.T) {
 	for _, key := range []string{
 		"shardSID", "shardAddressIPv6", "shardAddressIPv4", "nat64Prefix",
 		"shardAddressIPv6ClaimRef", "shardAddressIPv4ClaimRef",
-		"translateWellKnownPrefix",
+		"translateWellKnownPrefix", "drain",
 	} {
 		if _, ok := m[key]; ok {
 			t.Errorf("expected %q key to be absent when empty", key)

config/crd/network.datumapis.com_egressshards.yaml

@@ -29,6 +29,9 @@ spec:
     - jsonPath: .status.shardAddressIPv4
       name: IPV4
       type: string
+    - jsonPath: .spec.drain
+      name: DRAIN
+      type: boolean
     - jsonPath: .metadata.creationTimestamp
       name: AGE
       type: date
@@ -120,6 +123,22 @@ spec:
               is therefore accepted and is unattributable, which is a fact about the
               allocation rather than something a schema can repair.
             properties:
+              drain:
+                description: |-
+                  Drain stops this shard from taking new tenants while it keeps serving
+                  the ones it has. Compute nodes that spread egress across a cluster's
+                  shards give a draining shard no share of new tenant addresses, but keep
+                  sending it every flow already pinned to it until that flow goes idle, so
+                  established sessions finish where their translation state lives.
+
+                  It changes nothing on the shard itself: the shard keeps translating,
+                  advertising its SID and receiving replies. A shard is safe to remove
+                  once its session count has fallen to zero. Clearing it puts the shard
+                  back into rotation.
+
+                  Compute nodes that send each tenant to a single configured shard ignore
+                  it.
+                type: boolean
               nat64Prefix:
                 description: |-
                   NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard

docs/api/bgp.md

@@ -1067,6 +1067,7 @@ _Appears in:_
 | `shardAddressIPv4ClaimRef` _[AddressClaimRef](#addressclaimref)_ | ShardAddressIPv4ClaimRef records the addressing-service claim<br />ShardAddressIPv4 came from — see AddressClaimRef. Set it in the same<br />write as the address whenever that address came from a claim.<br />Nothing here resolves it and no component reads it to program anything;<br />it is the only trail from a translating address back to the allocation<br />accountable for it. |  |  |
 | `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard<br />translates to IPv4 — a Datum-operated Network-Specific Prefix, shared<br />fabric-wide, never per-tenant. TranslateWellKnownPrefix adds the RFC 6052<br />Well-Known Prefix alongside it. It must be the prefix the resolver<br />synthesizes into; a shard translating for a different one is a<br />blackhole with no symptom on either side.<br />Set together with ShardAddressIPv4 or not at all: an address with no<br />prefix has nothing to translate for, and a prefix with no address has<br />nothing to translate into. Write-once for the same reason the addresses<br />are: a shard that starts translating a different prefix blackholes every<br />destination the resolver already synthesized into the old one. |  |  |
 | `translateWellKnownPrefix` _boolean_ | TranslateWellKnownPrefix additionally translates destinations inside the<br />RFC 6052 Well-Known Prefix 64:ff9b::/96 to IPv4 from ShardAddressIPv4, so<br />clients using a public DNS64 resolver reach IPv4 destinations. Requires<br />NAT64Prefix.<br />RFC 6052 §3.1 forbids the Well-Known Prefix from representing non-global<br />IPv4 addresses; the implementation must refuse such destinations.<br />Mutable, unlike NAT64Prefix. Disabling it blackholes every destination<br />clients already resolved into the Well-Known Prefix until their DNS TTLs<br />expire. |  |  |
+| `drain` _boolean_ | Drain stops this shard from taking new tenants while it keeps serving<br />the ones it has. Compute nodes that spread egress across a cluster's<br />shards give a draining shard no share of new tenant addresses, but keep<br />sending it every flow already pinned to it until that flow goes idle, so<br />established sessions finish where their translation state lives.<br />It changes nothing on the shard itself: the shard keeps translating,<br />advertising its SID and receiving replies. A shard is safe to remove<br />once its session count has fallen to zero. Clearing it puts the shard<br />back into rotation.<br />Compute nodes that send each tenant to a single configured shard ignore<br />it. |  |  |
 
 
 #### EgressShardStatus

docs/api/gateway.md

@@ -168,6 +168,7 @@ _Appears in:_
 | `shardAddressIPv4ClaimRef` _[AddressClaimRef](#addressclaimref)_ | ShardAddressIPv4ClaimRef records the addressing-service claim<br />ShardAddressIPv4 came from — see AddressClaimRef. Set it in the same<br />write as the address whenever that address came from a claim.<br />Nothing here resolves it and no component reads it to program anything;<br />it is the only trail from a translating address back to the allocation<br />accountable for it. |  |  |
 | `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard<br />translates to IPv4 — a Datum-operated Network-Specific Prefix, shared<br />fabric-wide, never per-tenant. TranslateWellKnownPrefix adds the RFC 6052<br />Well-Known Prefix alongside it. It must be the prefix the resolver<br />synthesizes into; a shard translating for a different one is a<br />blackhole with no symptom on either side.<br />Set together with ShardAddressIPv4 or not at all: an address with no<br />prefix has nothing to translate for, and a prefix with no address has<br />nothing to translate into. Write-once for the same reason the addresses<br />are: a shard that starts translating a different prefix blackholes every<br />destination the resolver already synthesized into the old one. |  |  |
 | `translateWellKnownPrefix` _boolean_ | TranslateWellKnownPrefix additionally translates destinations inside the<br />RFC 6052 Well-Known Prefix 64:ff9b::/96 to IPv4 from ShardAddressIPv4, so<br />clients using a public DNS64 resolver reach IPv4 destinations. Requires<br />NAT64Prefix.<br />RFC 6052 §3.1 forbids the Well-Known Prefix from representing non-global<br />IPv4 addresses; the implementation must refuse such destinations.<br />Mutable, unlike NAT64Prefix. Disabling it blackholes every destination<br />clients already resolved into the Well-Known Prefix until their DNS TTLs<br />expire. |  |  |
+| `drain` _boolean_ | Drain stops this shard from taking new tenants while it keeps serving<br />the ones it has. Compute nodes that spread egress across a cluster's<br />shards give a draining shard no share of new tenant addresses, but keep<br />sending it every flow already pinned to it until that flow goes idle, so<br />established sessions finish where their translation state lives.<br />It changes nothing on the shard itself: the shard keeps translating,<br />advertising its SID and receiving replies. A shard is safe to remove<br />once its session count has fallen to zero. Clearing it puts the shard<br />back into rotation.<br />Compute nodes that send each tenant to a single configured shard ignore<br />it. |  |  |
 
 
 #### EgressShardStatus

test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml

@@ -11,7 +11,8 @@ spec:
     rejected. Also verifies that a claim reference defaults its apiGroup and
     kind, and that an address and a claim reference are independently
     optional. Also verifies that translateWellKnownPrefix requires nat64Prefix
-    and, unlike it, can be turned off again. Also verifies that status remains writable,
+    and, unlike it, can be turned off again. Also verifies that drain can be set
+    and cleared on a shard whose addresses are fixed. Also verifies that status remains writable,
     since status reports what the datapath actually programmed and must still
     converge. Does not require any operator to be running — every rule under
     test is enforced by the API server alone.
@@ -405,6 +406,23 @@ spec:
               fi
               echo "OK: translateWellKnownPrefix enabled with nat64Prefix set, then disabled"
 
+    - name: toggle-drain
+      try:
+        - script:
+            content: |
+              set -e
+              kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p '
+              {"spec":{"drain":true}}'
+              ON=$(kubectl get egressshard e2e-egress-shard -n "$NAMESPACE" -o jsonpath='{.spec.drain}')
+              kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p '
+              {"spec":{"drain":false}}'
+              OFF=$(kubectl get egressshard e2e-egress-shard -n "$NAMESPACE" -o jsonpath='{.spec.drain}')
+              if [ "$ON" != "true" ] || [ "$OFF" != "false" ]; then
+                echo "ERROR: drain toggled to $ON then $OFF, want true then false"
+                exit 1
+              fi
+              echo "OK: drain set on an addressed shard, then cleared"
+
     - name: accept-status-changes
       try:
         - script: