Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changes/unreleased/added-20261012-120909.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Added
body: |-
PodGroup safeToDisrupt field and kai.scheduler/safe-to-disrupt annotation key to opt out of consolidation
custom:
Author: "sam-huang1223"
Issue: "22"
8 changes: 8 additions & 0 deletions config/crd/scheduling.run.ai_podgroups.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,14 @@ spec:
Queue defines the queue to allocate resource for PodGroup; if queue does not exist,
the PodGroup will not be scheduled.
type: string
safeToDisrupt:
description: |-
SafeToDisrupt is whether the scheduler may evict this PodGroup's pods for reasons other than
fairness or priority. Today that is consolidation, which moves pods to other nodes to make room
for a pending workload. A workload that does not survive a restart (for example replicas with
restartPolicy Never, or bare pods) can set it to false. It does not affect preempt or reclaim.
Defaults to true.
type: boolean
schedulingBackoff:
description: The number of scheduling cycles to try before marking
the pod group as UnschedulableOnNodePool. Currently only supporting
Expand Down
1 change: 1 addition & 0 deletions podgrouper/constants/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ const (

PreemptionDelayAnnotationKey = "kai.scheduler/preemption-delay"
StalenessGracePeriodAnnotationKey = "kai.scheduler/staleness-grace-period"
SafeToDisruptAnnotationKey = "kai.scheduler/safe-to-disrupt"

BuildPriorityClass = "build"
TrainPriorityClass = "train"
Expand Down
19 changes: 19 additions & 0 deletions scheduling/v2alpha2/podgroup_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ package v2alpha2

import (
"fmt"
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -90,6 +91,14 @@ type PodGroupSpec struct {
// eviction for this PodGroup. Defaults to the scheduler's global staleness grace period.
// +optional
StalenessGracePeriod *metav1.Duration `json:"stalenessGracePeriod,omitempty" protobuf:"bytes,10,opt,name=stalenessGracePeriod"`

// SafeToDisrupt is whether the scheduler may evict this PodGroup's pods for reasons other than
// fairness or priority. Today that is consolidation, which moves pods to other nodes to make room
// for a pending workload. A workload that does not survive a restart (for example replicas with
// restartPolicy Never, or bare pods) can set it to false. It does not affect preempt or reclaim.
// Defaults to true.
// +optional
SafeToDisrupt *bool `json:"safeToDisrupt,omitempty" protobuf:"varint,11,opt,name=safeToDisrupt"`

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi @sam-huang1223, we discussed this as a team and decided to start with an annotation only. If it gets wide adoption, we can later promote it to a PodGroup field.

}

// Preemptibility defines whether this PodGroup can be preempted
Expand Down Expand Up @@ -151,6 +160,16 @@ func ParseStalenessGracePeriod(value string) (*metav1.Duration, error) {
return &metav1.Duration{Duration: stale}, nil
}

// ParseSafeToDisrupt parses a safe-to-disrupt value ("true" or "false").
// Returns an error for anything strconv.ParseBool does not accept.
func ParseSafeToDisrupt(value string) (*bool, error) {
safe, err := strconv.ParseBool(value)
if err != nil {
return nil, err
}
return &safe, nil
}

type SubGroup struct {
// Name uniquely identifies the SubGroup within the PodGroup.
// Must be a valid DNS label (RFC 1123).
Expand Down
40 changes: 40 additions & 0 deletions scheduling/v2alpha2/podgroup_types_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
// Copyright 2025 NVIDIA CORPORATION
// SPDX-License-Identifier: Apache-2.0

package v2alpha2

import (
"testing"
)

func TestParseSafeToDisrupt(t *testing.T) {
tests := []struct {
value string
want bool
wantErr bool
}{
{value: "true", want: true},
{value: "false", want: false},
{value: "False", want: false},
{value: "0", want: false},
{value: "", wantErr: true},
{value: "no", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.value, func(t *testing.T) {
got, err := ParseSafeToDisrupt(tt.value)
if tt.wantErr {
if err == nil {
t.Fatalf("ParseSafeToDisrupt(%q) = %v, want an error", tt.value, *got)
}
return
}
if err != nil {
t.Fatalf("ParseSafeToDisrupt(%q) returned error %v", tt.value, err)
}
if got == nil || *got != tt.want {
t.Fatalf("ParseSafeToDisrupt(%q) = %v, want %v", tt.value, got, tt.want)
}
})
}
}
5 changes: 5 additions & 0 deletions scheduling/v2alpha2/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading