diff --git a/assets/state-vgpu-device-manager/0600_daemonset.yaml b/assets/state-vgpu-device-manager/0600_daemonset.yaml index fb3385b3d..53b5b0999 100644 --- a/assets/state-vgpu-device-manager/0600_daemonset.yaml +++ b/assets/state-vgpu-device-manager/0600_daemonset.yaml @@ -28,14 +28,29 @@ spec: - name: vgpu-manager-validation image: "FILLED BY THE OPERATOR" command: ['sh', '-c'] - # The validator writes vgpu-manager-ready when the vGPU Manager is - # deployed as a container, and host-vgpu-manager-ready when the vGPU - # Manager driver is pre-installed on the host. Wait for either so this - # operand starts in both cases. - args: ["until [ -f /run/nvidia/validations/vgpu-manager-ready ] || [ -f /run/nvidia/validations/host-vgpu-manager-ready ]; do echo waiting for NVIDIA vGPU Manager to be setup; sleep 5; done"] + args: ["nvidia-validator"] + env: + - name: WITH_WAIT + value: "true" + - name: COMPONENT + value: vgpu-manager + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName securityContext: privileged: true + seLinuxOptions: + level: "s0" volumeMounts: + - name: host-root + mountPath: /host + readOnly: true + mountPropagation: HostToContainer + - name: driver-install-dir + mountPath: /run/nvidia/driver + readOnly: true + mountPropagation: HostToContainer - name: run-nvidia-validations mountPath: /run/nvidia/validations mountPropagation: Bidirectional diff --git a/controllers/object_controls.go b/controllers/object_controls.go index c1bf59e4c..146e32765 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -3935,6 +3935,21 @@ func transformValidationInitContainer(obj *appsv1.DaemonSet, config *gpuv1.Clust } } + // Pass env for vgpu-manager-validation init container + if strings.HasPrefix(initContainer.Name, "vgpu-manager") { + setContainerEnv(&(obj.Spec.Template.Spec.InitContainers[i]), "DEFAULT_GPU_WORKLOAD_CONFIG", defaultGPUWorkloadConfig) + if len(config.Validator.VGPUManager.Env) > 0 { + for _, env := range config.Validator.VGPUManager.Env { + // Preserve the required execution settings defined by the manifest. + switch env.Name { + case "COMPONENT", "NODE_NAME", "WITH_WAIT": + continue + } + setContainerEnv(&(obj.Spec.Template.Spec.InitContainers[i]), env.Name, env.Value) + } + } + } + // update validation image image, err := gpuv1.ImagePath(&config.Validator) if err != nil { diff --git a/controllers/object_controls_test.go b/controllers/object_controls_test.go index cdf20701c..127a819d2 100644 --- a/controllers/object_controls_test.go +++ b/controllers/object_controls_test.go @@ -1259,14 +1259,10 @@ func TestVGPUManagerAssets(t *testing.T) { } } -// TestVGPUDeviceManagerReadinessGate verifies that the vGPU Device Manager's -// vgpu-manager-validation init container waits for the vGPU Manager readiness -// status file written in BOTH deployment modes: vgpu-manager-ready (vGPU -// Manager deployed as a container) and host-vgpu-manager-ready (vGPU Manager -// driver pre-installed on the host). The validator writes only the host- -// prefixed file when the driver is pre-installed, so a gate that waits for the -// container-managed file alone hangs indefinitely when driver.enabled=false. -func TestVGPUDeviceManagerReadinessGate(t *testing.T) { +// TestVGPUDeviceManagerValidationInitContainer verifies that the vGPU Device +// Manager validates the vGPU Manager directly instead of depending on a status +// file produced by the sandbox validator. +func TestVGPUDeviceManagerValidationInitContainer(t *testing.T) { manifestPath := filepath.Join(cfg.root, vGPUDeviceManagerAssetsPath, "0600_daemonset.yaml") buffer, err := os.ReadFile(manifestPath) require.NoError(t, err, "unable to read vGPU Device Manager daemonset asset") @@ -1286,16 +1282,63 @@ func TestVGPUDeviceManagerReadinessGate(t *testing.T) { } require.NotNil(t, initCtr, "vgpu-manager-validation init container not found") - args := strings.Join(initCtr.Args, " ") - require.Contains(t, args, "/run/nvidia/validations/vgpu-manager-ready", - "readiness gate must wait for the container-managed vGPU Manager status file") - require.Contains(t, args, "/run/nvidia/validations/host-vgpu-manager-ready", - "readiness gate must also wait for the host-installed vGPU Manager status file (driver.enabled=false)") - // The two files must be combined with OR: the gate must pass when EITHER - // status file exists, since the validator only ever writes one of them. - // Guard against an accidental AND, which would re-break both modes. - require.Contains(t, args, "|| [ -f /run/nvidia/validations/host-vgpu-manager-ready", - "the two status files must be combined with OR, not AND, so the gate passes when either is present") + require.Equal(t, []string{"sh", "-c"}, initCtr.Command) + require.Equal(t, []string{"nvidia-validator"}, initCtr.Args) + + withWait := findEnvVarByName(initCtr.Env, "WITH_WAIT") + require.NotNil(t, withWait) + require.Equal(t, "true", withWait.Value) + component := findEnvVarByName(initCtr.Env, "COMPONENT") + require.NotNil(t, component) + require.Equal(t, "vgpu-manager", component.Value) + + nodeName := findEnvVarByName(initCtr.Env, "NODE_NAME") + require.NotNil(t, nodeName) + require.NotNil(t, nodeName.ValueFrom) + require.NotNil(t, nodeName.ValueFrom.FieldRef) + require.Equal(t, "spec.nodeName", nodeName.ValueFrom.FieldRef.FieldPath) + + require.NotNil(t, initCtr.SecurityContext) + require.NotNil(t, initCtr.SecurityContext.SELinuxOptions) + require.Equal(t, "s0", initCtr.SecurityContext.SELinuxOptions.Level) + + hostRoot := findVolumeMountByName(initCtr.VolumeMounts, "host-root") + require.NotNil(t, hostRoot) + require.Equal(t, "/host", hostRoot.MountPath) + require.True(t, hostRoot.ReadOnly) + require.NotNil(t, hostRoot.MountPropagation) + require.Equal(t, corev1.MountPropagationHostToContainer, *hostRoot.MountPropagation) + + driverInstallDir := findVolumeMountByName(initCtr.VolumeMounts, "driver-install-dir") + require.NotNil(t, driverInstallDir) + require.Equal(t, "/run/nvidia/driver", driverInstallDir.MountPath) + require.True(t, driverInstallDir.ReadOnly) + require.NotNil(t, driverInstallDir.MountPropagation) + require.Equal(t, corev1.MountPropagationHostToContainer, *driverInstallDir.MountPropagation) + + validations := findVolumeMountByName(initCtr.VolumeMounts, "run-nvidia-validations") + require.NotNil(t, validations) + require.Equal(t, "/run/nvidia/validations", validations.MountPath) + require.NotNil(t, validations.MountPropagation) + require.Equal(t, corev1.MountPropagationBidirectional, *validations.MountPropagation) +} + +func findEnvVarByName(envVars []corev1.EnvVar, name string) *corev1.EnvVar { + for i := range envVars { + if envVars[i].Name == name { + return &envVars[i] + } + } + return nil +} + +func findVolumeMountByName(volumeMounts []corev1.VolumeMount, name string) *corev1.VolumeMount { + for i := range volumeMounts { + if volumeMounts[i].Name == name { + return &volumeMounts[i] + } + } + return nil } // getSandboxDevicePluginTestInput return a ClusterPolicy instance for a particular diff --git a/controllers/transforms_test.go b/controllers/transforms_test.go index 8931298a4..8f117ac1b 100644 --- a/controllers/transforms_test.go +++ b/controllers/transforms_test.go @@ -2279,6 +2279,12 @@ func TestTransformCCManager(t *testing.T) { } func TestTransformVGPUDeviceManager(t *testing.T) { + originalDefaultWorkload := defaultGPUWorkloadConfig + defaultGPUWorkloadConfig = "container" + t.Cleanup(func() { + defaultGPUWorkloadConfig = originalDefaultWorkload + }) + resources := corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("100m"), @@ -2301,9 +2307,38 @@ func TestTransformVGPUDeviceManager(t *testing.T) { { description: "transform vgpu device manager", daemonset: NewDaemonset(). + WithInitContainer(corev1.Container{ + Name: "vgpu-manager-validation", + Env: []corev1.EnvVar{ + {Name: "WITH_WAIT", Value: "true"}, + {Name: "COMPONENT", Value: "vgpu-manager"}, + { + Name: "NODE_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + }, + }). WithContainer(corev1.Container{Name: "nvidia-vgpu-device-manager"}). WithContainer(corev1.Container{Name: "sidecar"}), clusterPolicySpec: &gpuv1.ClusterPolicySpec{ + Validator: gpuv1.ValidatorSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "gpu-operator-validator", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + ImagePullSecrets: []string{secret}, + Resources: &gpuv1.ResourceRequirements{Limits: resources.Limits, Requests: resources.Requests}, + VGPUManager: gpuv1.VGPUManagerValidatorSpec{ + Env: []gpuv1.EnvVar{ + {Name: "foo", Value: "bar"}, + {Name: "COMPONENT", Value: "cuda"}, + {Name: "NODE_NAME", Value: "invalid-override"}, + {Name: "WITH_WAIT", Value: "false"}, + }, + }, + }, VGPUDeviceManager: gpuv1.VGPUDeviceManagerSpec{ Repository: "nvcr.io/nvidia/cloud-native", Image: "vgpu-device-manager", @@ -2320,6 +2355,27 @@ func TestTransformVGPUDeviceManager(t *testing.T) { }, }, expectedDaemonset: NewDaemonset(). + WithInitContainer(corev1.Container{ + Name: "vgpu-manager-validation", + Image: "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + {Name: "WITH_WAIT", Value: "true"}, + {Name: "COMPONENT", Value: "vgpu-manager"}, + { + Name: "NODE_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + {Name: "DEFAULT_GPU_WORKLOAD_CONFIG", Value: "container"}, + {Name: "foo", Value: "bar"}, + }, + Resources: resources, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: rootUID, + }, + }). WithContainer(corev1.Container{ Name: "nvidia-vgpu-device-manager", Image: "nvcr.io/nvidia/cloud-native/vgpu-device-manager:v1.0.0",