[Feature] Remove stuck Pods (#1148)

Author: ajanikow

CHANGELOG.md

@@ -10,6 +10,7 @@
 - (Feature) PVC Member Status info
 - (Feature) Respect ToBeCleanedServers in Agency
 - (Improvement) Unify K8S Error Handling
+- (Feature) Remove stuck Pods
 
 ## [1.2.19](https://github.com/arangodb/kube-arangodb/tree/1.2.19) (2022-10-05)
 - (Bugfix) Prevent changes when UID is wrong

pkg/deployment/resources/pod_inspector.go

@@ -37,6 +37,7 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/metrics"
 	"github.com/arangodb/kube-arangodb/pkg/util"
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
+	"github.com/arangodb/kube-arangodb/pkg/util/globals"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/info"
 	inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector"
@@ -372,6 +373,7 @@ func (r *Resources) InspectPods(ctx context.Context, cachedStatus inspectorInter
 				updateMemberStatusNeeded = true
 				log.Str("pod-name", pod.GetName()).Debug("Pod marked as terminating")
 			}
+
 			// Process finalizers
 			if x, err := r.runPodFinalizers(ctx, pod, memberStatus, func(m api.MemberStatus) error {
 				updateMemberStatusNeeded = true
@@ -383,6 +385,26 @@ func (r *Resources) InspectPods(ctx context.Context, cachedStatus inspectorInter
 			} else {
 				nextInterval = nextInterval.ReduceTo(x)
 			}
+
+			// Check if any additional deletion request is required
+			if !k8sutil.IsPodAlive(pod) {
+				// Pod is dead, but still not removed. Send additional deletion request
+				nctx, c := globals.GetGlobals().Timeouts().Kubernetes().WithTimeout(ctx)
+				defer c()
+
+				if err := cachedStatus.PodsModInterface().V1().Delete(nctx, pod.GetName(), meta.DeleteOptions{
+					GracePeriodSeconds: util.NewInt64(10),
+					Preconditions:      meta.NewUIDPreconditions(string(pod.GetUID())),
+				}); err != nil {
+					if k8sutil.IsNotFound(err) {
+						// Pod is already gone, we are fine with it
+					} else if k8sutil.IsConflict(err) {
+						log.Warn("UID of Pod Changed")
+					} else {
+						log.Err(err).Error("Unknown error while deleting Pod")
+					}
+				}
+			}
 		}
 
 		if updateMemberStatusNeeded {

pkg/util/k8sutil/errors.go

@@ -75,11 +75,11 @@ func isNotFoundC(err error) bool {
 // IsInvalid returns true if the given error is or is caused by a
 // kubernetes InvalidError,
 func IsInvalid(err error) bool {
-	return apierrors.IsInvalid(errors.Cause(err))
+	return isError(err, isInvalidC)
 }
 
 func isInvalidC(err error) bool {
-	return isError(err, isInvalidC)
+	return apierrors.IsInvalid(errors.Cause(err))
 }
 
 // IsForbiddenOrNotFound returns true if the given error is or is caused by a

pkg/util/k8sutil/pods.go

@@ -360,6 +360,29 @@ func MasterJWTVolumeMount() core.VolumeMount {
 	}
 }
 
+// IsPodAlive returns true if any of the containers within pod is running
+func IsPodAlive(pod *core.Pod) bool {
+	return IsAnyContainerAlive(pod.Status.ContainerStatuses) ||
+		IsAnyContainerAlive(pod.Status.InitContainerStatuses) ||
+		IsAnyContainerAlive(pod.Status.EphemeralContainerStatuses)
+}
+
+// IsAnyContainerAlive returns true if any of the containers is running
+func IsAnyContainerAlive(containers []core.ContainerStatus) bool {
+	for _, c := range containers {
+		if IsContainerAlive(c) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// IsContainerAlive returns true if container is running
+func IsContainerAlive(container core.ContainerStatus) bool {
+	return container.State.Running != nil
+}
+
 // ClusterJWTVolumeMount creates a volume mount structure for a cluster JWT secret (token).
 func ClusterJWTVolumeMount() core.VolumeMount {
 	return core.VolumeMount{