[Feature] ArangoMember Removal Priority (#1409)

Author: ajanikow

CHANGELOG.md

@@ -19,6 +19,7 @@
 - (Bugfix) Use ArangoMember as owner reference for syncmaster secrets instead of Deployment
 - (Improvement) Remove PodSchedulingFailure condition instead of setting to false, restart pod if it could not be scheduled
 - (Feature) Add ArangoMember overrides
+- (Feature) ArangoMember Removal Priority
 
 ## [1.2.32](https://github.com/arangodb/kube-arangodb/tree/1.2.32) (2023-08-07)
 - (Feature) Backup lifetime - remove Backup once its lifetime has been reached

pkg/apis/deployment/v1/arango_member_spec.go

@@ -22,6 +22,8 @@ package v1
 
 import (
 	"k8s.io/apimachinery/pkg/types"
+
+	"github.com/arangodb/kube-arangodb/pkg/util"
 )
 
 type ArangoMemberSpec struct {
@@ -38,4 +40,14 @@ type ArangoMemberSpec struct {
 
 	// Template keeps template which is gonna be applied on the Pod.
 	Template *ArangoMemberPodTemplate `json:"template,omitempty"`
+
+	DeletionPriority *int `json:"deletion_priority,omitempty"`
+}
+
+func (a *ArangoMemberSpec) GetDeletionPriority() int {
+	if a == nil {
+		return 0
+	}
+
+	return util.TypeOrDefault[int](a.DeletionPriority, 0)
 }

pkg/apis/deployment/v1/topology_status.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -140,6 +140,14 @@ func (t *TopologyStatus) IsTopologyEvenlyDistributed(group ServerGroup) bool {
 	return min+1 >= max
 }
 
+func (t *TopologyStatus) GetID() types.UID {
+	if t == nil {
+		return ""
+	}
+
+	return t.ID
+}
+
 func (t *TopologyStatus) Enabled() bool {
 	return t != nil
 }

pkg/apis/deployment/v1/zz_generated.deepcopy.go

@@ -374,7 +374,7 @@ const (
 	// ActionTypeJWTStatusUpdate in scopes Normal. Update status of JWT propagation
 	ActionTypeJWTStatusUpdate ActionType = "JWTStatusUpdate"
 
-	// ActionTypeKillMemberPod in scopes Normal. Execute Delete on Pod (put pod in Terminating state)
+	// ActionTypeKillMemberPod in scopes High and Normal. Execute Delete on Pod (put pod in Terminating state)
 	ActionTypeKillMemberPod ActionType = "KillMemberPod"
 
 	// ActionTypeLicenseSet in scopes Normal. Update Cluster license (3.9+)
@@ -776,7 +776,7 @@ func (a ActionType) Priority() ActionPriority {
 	case ActionTypeJWTStatusUpdate:
 		return ActionPriorityNormal
 	case ActionTypeKillMemberPod:
-		return ActionPriorityNormal
+		return ActionPriorityHigh
 	case ActionTypeLicenseSet:
 		return ActionPriorityNormal
 	case ActionTypeMarkToRemoveMember:

pkg/apis/deployment/v2alpha1/actions.generated.go

@@ -22,6 +22,8 @@ package v2alpha1
 
 import (
 	"k8s.io/apimachinery/pkg/types"
+
+	"github.com/arangodb/kube-arangodb/pkg/util"
 )
 
 type ArangoMemberSpec struct {
@@ -38,4 +40,14 @@ type ArangoMemberSpec struct {
 
 	// Template keeps template which is gonna be applied on the Pod.
 	Template *ArangoMemberPodTemplate `json:"template,omitempty"`
+
+	DeletionPriority *int `json:"deletion_priority,omitempty"`
+}
+
+func (a *ArangoMemberSpec) GetDeletionPriority() int {
+	if a == nil {
+		return 0
+	}
+
+	return util.TypeOrDefault[int](a.DeletionPriority, 0)
 }

pkg/apis/deployment/v2alpha1/arango_member_spec.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -140,6 +140,14 @@ func (t *TopologyStatus) IsTopologyEvenlyDistributed(group ServerGroup) bool {
 	return min+1 >= max
 }
 
+func (t *TopologyStatus) GetID() types.UID {
+	if t == nil {
+		return ""
+	}
+
+	return t.ID
+}
+
 func (t *TopologyStatus) Enabled() bool {
 	return t != nil
 }

pkg/apis/deployment/v2alpha1/topology_status.go

@@ -26,9 +26,7 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/apis/deployment"
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/actions"
-	"github.com/arangodb/kube-arangodb/pkg/deployment/agency/state"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/reconcile/shared"
-	"github.com/arangodb/kube-arangodb/pkg/deployment/reconciler"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 )
 
@@ -94,12 +92,7 @@ func (r *Reconciler) createScalePlan(status api.DeploymentStatus, members api.Me
 	} else if len(members) > count {
 		// Note, we scale down 1 member at a time
 
-		if m, err := members.SelectMemberToRemove(
-			getCleanedServer(context),
-			getToBeCleanedServer(context),
-			topologyMissingMemberToRemoveSelector(status.Topology),
-			topologyAwarenessMemberToRemoveSelector(group, status.Topology),
-			getDbServerWithLowestShards(context, group)); err != nil {
+		if m, err := planBuilderScaleDownFilter(context, status, group, members); err != nil {
 			r.planLogger.Err(err).Str("role", group.AsRole()).Warn("Failed to select member to remove")
 		} else {
 			ready, message := groupReadyForRestart(context, status, m, group)
@@ -174,68 +167,6 @@ func filterScaleUP(a api.Action) bool {
 	return a.Type == api.ActionTypeAddMember
 }
 
-func getCleanedServer(ctx reconciler.ArangoAgencyGet) api.MemberToRemoveSelector {
-	return func(m api.MemberStatusList) (string, error) {
-		if a, ok := ctx.GetAgencyCache(); ok {
-			for _, member := range m {
-				if a.Target.CleanedServers.Contains(state.Server(member.ID)) {
-					return member.ID, nil
-				}
-			}
-		}
-		return "", nil
-	}
-}
-
-func getToBeCleanedServer(ctx reconciler.ArangoAgencyGet) api.MemberToRemoveSelector {
-	return func(m api.MemberStatusList) (string, error) {
-		if a, ok := ctx.GetAgencyCache(); ok {
-			for _, member := range m {
-				if a.Target.ToBeCleanedServers.Contains(state.Server(member.ID)) {
-					return member.ID, nil
-				}
-			}
-		}
-		return "", nil
-	}
-}
-
-func getDbServerWithLowestShards(ctx reconciler.ArangoAgencyGet, g api.ServerGroup) api.MemberToRemoveSelector {
-	return func(m api.MemberStatusList) (string, error) {
-		if g != api.ServerGroupDBServers {
-			return "", nil
-		}
-
-		a, ok := ctx.GetAgencyCache()
-		if !ok {
-			return "", nil
-		}
-
-		dbServersShards := a.ShardsByDBServers()
-		for _, member := range m {
-			if _, ok := dbServersShards[state.Server(member.ID)]; !ok {
-				// member is not in agency cache, so it has no shards
-				return member.ID, nil
-			}
-		}
-
-		var resultServer state.Server = ""
-		var resultShards int
-
-		for server, shards := range dbServersShards {
-			// init first server as result
-			if resultServer == "" {
-				resultServer = server
-				resultShards = shards
-			} else if shards < resultShards {
-				resultServer = server
-				resultShards = shards
-			}
-		}
-		return string(resultServer), nil
-	}
-}
-
 func (r *Reconciler) scaleDownCandidate(ctx context.Context, apiObject k8sutil.APIObject,
 	spec api.DeploymentSpec, status api.DeploymentStatus,
 	context PlanBuilderContext) api.Plan {

pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go

@@ -0,0 +1,33 @@
+//
+// DISCLAIMER
+//
+// Copyright 2023 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+//go:build !enterprise
+
+package reconcile
+
+import api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+
+func planBuilderScaleDownTopologyMissing(context PlanBuilderContext, status api.DeploymentStatus, group api.ServerGroup, in api.MemberStatusList) (api.MemberStatus, bool, error) {
+	return api.MemberStatus{}, false, nil
+}
+
+func planBuilderScaleDownTopologyAwarenessMember(context PlanBuilderContext, status api.DeploymentStatus, group api.ServerGroup, in api.MemberStatusList) (api.MemberStatus, bool, error) {
+	return api.MemberStatus{}, false, nil
+}