From e69c4dccaa8dd44d36f987116031f85a4665d67c Mon Sep 17 00:00:00 2001
From: Abhishek Dalbanjan <abhishekdalbanjan1@gmail.com>
Date: Wed, 1 Jul 2026 12:23:58 +0530
Subject: [PATCH] Drain nodegroup: gracefully handle transient control plane
 failures

When draining nodegroups, the kubernetes api server might return transient HTTP 5xx errors like 'etcdserver: leader changed' or 'raft proposal dropped' during leadership transitions. This commit recognizes these string patterns as recoverable errors so eksctl retries evicting the pods instead of surfacing an unrecoverable failure.

Fixes #7550

Signed-off-by: Abhishek Dalbanjan <abhishekdalbanjan1@gmail.com>
---
 pkg/drain/nodegroup.go      | 16 ++++++++++++
 pkg/drain/nodegroup_test.go | 51 +++++++++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+)

diff --git a/pkg/drain/nodegroup.go b/pkg/drain/nodegroup.go
index 5024dfcfc4..f049499c75 100644
--- a/pkg/drain/nodegroup.go
+++ b/pkg/drain/nodegroup.go
@@ -3,6 +3,7 @@ package drain
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	"golang.org/x/sync/errgroup"
@@ -261,6 +262,7 @@ func isEvictionErrorRecoverable(err error) bool {
 		apierrors.IsTimeout,
 		// IsTooManyRequests also captures PDB errors
 		apierrors.IsTooManyRequests,
+		isTransientControlPlaneError,
 	)
 
 	for _, f := range recoverableCheckerFuncs {
@@ -270,3 +272,17 @@ func isEvictionErrorRecoverable(err error) bool {
 	}
 	return false
 }
+
+func isTransientControlPlaneError(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := err.Error()
+	if strings.Contains(msg, "etcdserver: leader changed") {
+		return true
+	}
+	if strings.Contains(msg, "raft proposal dropped") {
+		return true
+	}
+	return false
+}
diff --git a/pkg/drain/nodegroup_test.go b/pkg/drain/nodegroup_test.go
index f70fc41e63..ad5a954011 100644
--- a/pkg/drain/nodegroup_test.go
+++ b/pkg/drain/nodegroup_test.go
@@ -286,6 +286,57 @@ var _ = Describe("Drain", func() {
 		})
 	})
 
+	When("an eviction fails recoverably due to leader changed", func() {
+		var pod corev1.Pod
+
+		BeforeEach(func() {
+			pod = corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "pod-1",
+				},
+			}
+
+			for i := 0; i < 2; i++ {
+				fakeEvictor.GetPodsForEvictionReturnsOnCall(i, &evictor.PodDeleteList{
+					Items: []evictor.PodDelete{
+						{
+							Pod: pod,
+							Status: evictor.PodDeleteStatus{
+								Delete: true,
+							},
+						},
+					},
+				}, nil)
+			}
+			fakeEvictor.GetPodsForEvictionReturnsOnCall(2, nil, nil)
+
+			fakeEvictor.EvictOrDeletePodReturnsOnCall(0, errors.New("etcdserver: leader changed"))
+			fakeEvictor.EvictOrDeletePodReturnsOnCall(1, nil)
+
+			_, err := fakeClientSet.CoreV1().Nodes().Create(context.Background(), &corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: nodeName,
+				},
+				Spec: corev1.NodeSpec{
+					Unschedulable: false,
+				},
+			}, metav1.CreateOptions{})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("does not error", func() {
+			nodeGroupDrainer := drain.NewNodeGroupDrainer(fakeClientSet, &mockNG, time.Second, time.Second, 0, false, false, 1)
+			nodeGroupDrainer.SetDrainer(fakeEvictor)
+
+			Expect(nodeGroupDrainer.Drain(ctx, sem)).To(Succeed())
+
+			Expect(fakeEvictor.GetPodsForEvictionCallCount()).To(Equal(3))
+			Expect(fakeEvictor.EvictOrDeletePodCallCount()).To(Equal(2))
+			Expect(fakeEvictor.EvictOrDeletePodArgsForCall(0)).To(Equal(pod))
+			Expect(fakeEvictor.EvictOrDeletePodArgsForCall(1)).To(Equal(pod))
+		})
+	})
+
 	When("an eviction fails irrecoverably", func() {
 		var pod corev1.Pod
 		var evictionError error