diff --git a/build/components/versions.yml b/build/components/versions.yml index fa5a5e00e8..515a0fa56a 100644 --- a/build/components/versions.yml +++ b/build/components/versions.yml @@ -3,7 +3,7 @@ firmware: libvirt: v10.9.0 edk2: stable202411 core: - 3p-kubevirt: v1.6.2-v12n.42 + 3p-kubevirt: feat/inplace-resize # v1.6.2-v12n.42 3p-containerized-data-importer: v1.60.3-v12n.19 distribution: 2.8.3 package: diff --git a/images/virt-artifact/werf.inc.yaml b/images/virt-artifact/werf.inc.yaml index 542d241165..1ebb80d767 100644 --- a/images/virt-artifact/werf.inc.yaml +++ b/images/virt-artifact/werf.inc.yaml @@ -9,6 +9,7 @@ image: {{ .ModuleNamePrefix }}{{ .ImageName }}-src-artifact final: false fromImage: builder/src +fromCacheVersion: "005" secrets: - id: SOURCE_REPO value: {{ $.SOURCE_REPO }} diff --git a/images/virtualization-artifact/pkg/common/annotations/annotations.go b/images/virtualization-artifact/pkg/common/annotations/annotations.go index 10ab494bbd..38f9dcb6cd 100644 --- a/images/virtualization-artifact/pkg/common/annotations/annotations.go +++ b/images/virtualization-artifact/pkg/common/annotations/annotations.go @@ -231,6 +231,8 @@ const ( DefaultUSBDeviceGroup = "64535" // DefaultUSBDeviceUser is the default device user ID for USB devices. DefaultUSBDeviceUser = "64535" + + AnnVirtualMachineInstanceInPlaceResizeInProgress = "internal.virtualization.deckhouse.io/in-place-resize-in-progress" ) // AddAnnotation adds an annotation to an object diff --git a/images/virtualization-artifact/pkg/controller/vmchange/comparator_cpu.go b/images/virtualization-artifact/pkg/controller/vmchange/comparator_cpu.go index 71d2f919aa..3cb30cc3ed 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/comparator_cpu.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/comparator_cpu.go @@ -53,7 +53,7 @@ func (c *comparatorCPU) Compare(current, desired *v1alpha2.VirtualMachineSpec) [ fractionChangedAction := ActionApplyImmediate // Require reboot if CPU hotplug is not enabled. - if !c.featureGate.Enabled(featuregates.HotplugCPUWithLiveMigration) { + if !c.featureGate.Enabled(featuregates.HotplugCPUWithLiveMigration) && !c.featureGate.Enabled(featuregates.HotplugCPUInPlaceResize) { coresChangedAction = ActionRestart fractionChangedAction = ActionRestart } diff --git a/images/virtualization-artifact/pkg/controller/vmchange/comparator_memory.go b/images/virtualization-artifact/pkg/controller/vmchange/comparator_memory.go index c00d33aa5c..3cb0d084c7 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/comparator_memory.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/comparator_memory.go @@ -57,7 +57,7 @@ func (c *comparatorMemory) Compare(current, desired *v1alpha2.VirtualMachineSpec } // Require reboot if memory hotplug is not enabled. - if !c.featureGate.Enabled(featuregates.HotplugMemoryWithLiveMigration) { + if !c.featureGate.Enabled(featuregates.HotplugMemoryWithLiveMigration) && !c.featureGate.Enabled(featuregates.HotplugMemoryInPlaceResize) { actionType = ActionRestart } diff --git a/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug.go b/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug.go index e1ac2fb1c5..a40e7f7e35 100644 --- a/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug.go +++ b/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug.go @@ -30,6 +30,8 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" "github.com/deckhouse/virtualization-controller/pkg/logger" "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization-controller/pkg/featuregates" + "github.com/deckhouse/virtualization-controller/pkg/common/kvvm" ) const hotplugHandler = "HotplugHandler" @@ -56,6 +58,18 @@ func (h *HotplugHandler) Handle(ctx context.Context, vm *v1alpha2.VirtualMachine return reconcile.Result{}, client.IgnoreNotFound(err) } + if h.inplaceResizeInProgress(kvvmi) { + possible, complete, err := h.inPlaceResizePossible(ctx, kvvmi) + if err != nil { + return reconcile.Result{}, err + } + if possible || complete { + return reconcile.Result{}, nil + } + // inplace resize is not possible, but it is not complete + // switch to resize via live migration + } + cond, _ := conditions.GetKVVMICondition(virtv1.VirtualMachineInstanceMemoryChange, kvvmi.Status.Conditions) isMemoryHotplug := cond.Status == corev1.ConditionTrue @@ -81,6 +95,46 @@ func (h *HotplugHandler) Name() string { return hotplugHandler } +func (h *HotplugHandler) inplaceResizeInProgress(kvvmi *virtv1.VirtualMachineInstance) bool { + if featuregates.Default().Enabled(featuregates.HotplugCPUInPlaceResize) || featuregates.Default().Enabled(featuregates.HotplugMemoryInPlaceResize) { + return kvvmi.GetAnnotations()[annotations.AnnVirtualMachineInstanceInPlaceResizeInProgress] == "true" + } + + return false +} + +func (h *HotplugHandler) inPlaceResizePossible(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) (bool, bool, error) { + cond, exists := conditions.GetKVVMICondition("PodResourceResizeInProgress", kvvmi.Status.Conditions) + if !exists { + return false, false, fmt.Errorf("failed to get PodResourceResizeInProgress condition") + } + + switch cond.Reason { + case "PodResizeCompleted": + return false, true, nil + case "PodResizePending", "PodResizeInProgress": + default: + return false, false, fmt.Errorf("unexpected PodResourceResizeInProgress condition reason: %s", cond.Reason) + } + + pod, err := kvvm.FindPodByKVVMI(ctx, h.client, kvvmi) + if err != nil { + return false, false, err + } + + podResizePending, _ := conditions.GetPodCondition(corev1.PodResizePending, pod.Status.Conditions) + // https://github.com/kubernetes/kubernetes/blob/48d5bff65612a50ba5149319810a71ee151ec076/pkg/kubelet/events/event.go#L37 + if podResizePending.Reason == "ResizeDeferred" || podResizePending.Reason == "ResizeInfeasible" { + return false, false, nil + } + podResizeInProgress, _ := conditions.GetPodCondition(corev1.PodResizeInProgress, pod.Status.Conditions) + if podResizeInProgress.Reason == "ResizeError" { + return false, false, nil + } + + return true, false, nil +} + func getHotplugResourcesSum(vm *v1alpha2.VirtualMachine) string { return fmt.Sprintf("cpu.cores=%d,cpu.coreFraction=%s,memory.size=%s", vm.Spec.CPU.Cores, vm.Spec.CPU.CoreFraction, vm.Spec.Memory.Size.String()) } diff --git a/images/virtualization-artifact/pkg/featuregates/featuregate.go b/images/virtualization-artifact/pkg/featuregates/featuregate.go index 3358b0a85a..0e6c2c9337 100644 --- a/images/virtualization-artifact/pkg/featuregates/featuregate.go +++ b/images/virtualization-artifact/pkg/featuregates/featuregate.go @@ -32,6 +32,8 @@ const ( USB featuregate.Feature = "USB" HotplugCPUWithLiveMigration featuregate.Feature = "HotplugCPUWithLiveMigration" HotplugMemoryWithLiveMigration featuregate.Feature = "HotplugMemoryWithLiveMigration" + HotplugCPUInPlaceResize featuregate.Feature = "HotplugCPUInPlaceResize" + HotplugMemoryInPlaceResize featuregate.Feature = "HotplugMemoryInPlaceResize" ) var featureSpecs = map[featuregate.Feature]featuregate.FeatureSpec{ @@ -69,6 +71,16 @@ var featureSpecs = map[featuregate.Feature]featuregate.FeatureSpec{ LockToDefault: version.GetEdition() == version.EditionCE, PreRelease: featuregate.Alpha, }, + HotplugCPUInPlaceResize: { + Default: false, + LockToDefault: version.GetEdition() == version.EditionCE, + PreRelease: featuregate.Alpha, + }, + HotplugMemoryInPlaceResize: { + Default: false, + LockToDefault: version.GetEdition() == version.EditionCE, + PreRelease: featuregate.Alpha, + }, } var ( diff --git a/openapi/config-values.yaml b/openapi/config-values.yaml index e46d2362fc..0d460675d3 100644 --- a/openapi/config-values.yaml +++ b/openapi/config-values.yaml @@ -282,10 +282,15 @@ properties: description: | Enable experimental or early access features. - - `HotplugCPUWithLiveMigration` — enable live changing of cpu cores number. (Not available in CE); - - `HotplugMemoryWithLiveMigration` — enable live changing of memory size. (Not available in CE); + - `HotplugCPUWithLiveMigration` — enable live changing of cpu cores number via LiveMigration. (Not available in CE); + - `HotplugMemoryWithLiveMigration` — enable live changing of memory size via LiveMigration. (Not available in CE); + - `HotplugCPUInPlaceResize` - enable live changing of cpu cores number via InPlaceResize. (Not available in CE); + - `HotplugMemoryInPlaceResize` - enable live changing of memory size via InPlaceResize. (Not available in CE); items: type: string enum: - "HotplugCPUWithLiveMigration" - "HotplugMemoryWithLiveMigration" + - "HotplugCPUInPlaceResize" + - "HotplugMemoryInPlaceResize" + diff --git a/openapi/doc-ru-config-values.yaml b/openapi/doc-ru-config-values.yaml index 5aaf65aae6..0fe5831493 100644 --- a/openapi/doc-ru-config-values.yaml +++ b/openapi/doc-ru-config-values.yaml @@ -179,7 +179,9 @@ properties: description: | Включение экспериментальных или недостаточно обкатанных возможностей. - - `HotplugCPUWithLiveMigration` — включить изменение количества ядер процессора без перезагрузки. (Не доступно в CE); - - `HotplugMemoryWithLiveMigration` — включить изменение размера памяти без перезагрузки. (Не доступно в CE); + - `HotplugCPUWithLiveMigration` — включить изменение количества ядер процессора без перезагрузки через живую миграцию. (Не доступно в CE); + - `HotplugMemoryWithLiveMigration` — включить изменение размера памяти без перезагрузки через живую миграцию. (Не доступно в CE); + - `HotplugCPUInPlaceResize` - включить изменение количества ядер процессора без перезагрузки через InPlaceResize (Не доступно в CE) + - `HotplugMemoryInPlaceResize` - включить изменение размера памяти без перезагрузки через InPlaceResie (Не доступно в CE)) items: type: string diff --git a/templates/kubevirt/kubevirt.yaml b/templates/kubevirt/kubevirt.yaml index be76ae70bc..a704572151 100644 --- a/templates/kubevirt/kubevirt.yaml +++ b/templates/kubevirt/kubevirt.yaml @@ -71,6 +71,7 @@ spec: - HostDevicesWithDRA - HostDevices - HotplugHostDevicesWithDRA # custom feature gate - added in our KubeVirt fork, not present in upstream + - InPlaceResize # custom feature gate - added in our KubeVirt fork, not present in upstream virtualMachineOptions: disableSerialConsoleLog: {} customizeComponents: diff --git a/templates/kubevirt/virt-operator/rbac-for-us.yaml b/templates/kubevirt/virt-operator/rbac-for-us.yaml index c3c7b0119a..38213700da 100644 --- a/templates/kubevirt/virt-operator/rbac-for-us.yaml +++ b/templates/kubevirt/virt-operator/rbac-for-us.yaml @@ -514,6 +514,12 @@ rules: - pods/status verbs: - patch +- apiGroups: + - "" + resources: + - pods/resize + verbs: + - update - apiGroups: - "" resources: