diff --git a/api/nvidia/v1/clusterpolicy_types.go b/api/nvidia/v1/clusterpolicy_types.go index 16d95220c..350c64e30 100644 --- a/api/nvidia/v1/clusterpolicy_types.go +++ b/api/nvidia/v1/clusterpolicy_types.go @@ -431,6 +431,15 @@ type ValidatorSpec struct { // PluginValidatorSpec defines validator spec for NVIDIA Device Plugin type PluginValidatorSpec struct { + // Skip indicates whether to skip the NVIDIA Device Plugin validation. When set to true, the + // plugin-validation init container does not run the validation and only creates the + // component readiness status file, so that other operands which depend on it are not blocked. + // +kubebuilder:validation:Optional + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Skip NVIDIA Device Plugin validation" + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch" + Skip *bool `json:"skip,omitempty"` + // Optional: List of environment variables // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Environment Variables" @@ -440,6 +449,15 @@ type PluginValidatorSpec struct { // ToolkitValidatorSpec defines validator spec for NVIDIA Container Toolkit type ToolkitValidatorSpec struct { + // Skip indicates whether to skip the NVIDIA Container Toolkit validation. When set to true, the + // toolkit-validation init container does not run the validation and only creates the + // component readiness status file, so that other operands which depend on it are not blocked. + // +kubebuilder:validation:Optional + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Skip NVIDIA Container Toolkit validation" + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch" + Skip *bool `json:"skip,omitempty"` + // Optional: List of environment variables // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Environment Variables" @@ -458,6 +476,15 @@ type DriverValidatorSpec struct { // CUDAValidatorSpec defines validator spec for CUDA validation workload pod type CUDAValidatorSpec struct { + // Skip indicates whether to skip the CUDA validation. When set to true, the + // cuda-validation init container does not run the validation and only creates the + // component readiness status file, so that other operands which depend on it are not blocked. + // +kubebuilder:validation:Optional + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Skip CUDA validation" + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch" + Skip *bool `json:"skip,omitempty"` + // Optional: List of environment variables // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Environment Variables" @@ -2224,6 +2251,33 @@ func (d *DriverSpec) IsAutoUpgradeEnabled() bool { return d.UpgradePolicy.AutoUpgrade } +// IsSkipped returns true if the NVIDIA Device Plugin validation is skipped +func (p *PluginValidatorSpec) IsSkipped() bool { + if p.Skip == nil { + // default is false if not specified by user + return false + } + return *p.Skip +} + +// IsSkipped returns true if the NVIDIA Container Toolkit validation is skipped +func (t *ToolkitValidatorSpec) IsSkipped() bool { + if t.Skip == nil { + // default is false if not specified by user + return false + } + return *t.Skip +} + +// IsSkipped returns true if the CUDA validation is skipped +func (c *CUDAValidatorSpec) IsSkipped() bool { + if c.Skip == nil { + // default is false if not specified by user + return false + } + return *c.Skip +} + // IsEnabled returns true if device-plugin is enabled(default) through gpu-operator func (p *DevicePluginSpec) IsEnabled() bool { if p.Enabled == nil { diff --git a/api/nvidia/v1/zz_generated.deepcopy.go b/api/nvidia/v1/zz_generated.deepcopy.go index 9e936de60..07e03e995 100644 --- a/api/nvidia/v1/zz_generated.deepcopy.go +++ b/api/nvidia/v1/zz_generated.deepcopy.go @@ -107,6 +107,11 @@ func (in *CDIConfigSpec) DeepCopy() *CDIConfigSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CUDAValidatorSpec) DeepCopyInto(out *CUDAValidatorSpec) { *out = *in + if in.Skip != nil { + in, out := &in.Skip, &out.Skip + *out = new(bool) + **out = **in + } if in.Env != nil { in, out := &in.Env, &out.Env *out = make([]EnvVar, len(*in)) @@ -1374,6 +1379,11 @@ func (in *PSPSpec) DeepCopy() *PSPSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PluginValidatorSpec) DeepCopyInto(out *PluginValidatorSpec) { *out = *in + if in.Skip != nil { + in, out := &in.Skip, &out.Skip + *out = new(bool) + **out = **in + } if in.Env != nil { in, out := &in.Env, &out.Env *out = make([]EnvVar, len(*in)) @@ -1591,6 +1601,11 @@ func (in *ToolkitSpec) DeepCopy() *ToolkitSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ToolkitValidatorSpec) DeepCopyInto(out *ToolkitValidatorSpec) { *out = *in + if in.Skip != nil { + in, out := &in.Skip, &out.Skip + *out = new(bool) + **out = **in + } if in.Env != nil { in, out := &in.Env, &out.Env *out = make([]EnvVar, len(*in)) diff --git a/bundle/manifests/nvidia.com_clusterpolicies.yaml b/bundle/manifests/nvidia.com_clusterpolicies.yaml index 6257b44d5..a30edeba0 100644 --- a/bundle/manifests/nvidia.com_clusterpolicies.yaml +++ b/bundle/manifests/nvidia.com_clusterpolicies.yaml @@ -2363,6 +2363,12 @@ spec: - name type: object type: array + skip: + description: |- + Skip indicates whether to skip the CUDA validation. When set to true, the + cuda-validation init container does not run the validation and only creates the + component readiness status file, so that other operands which depend on it are not blocked. + type: boolean type: object driver: description: Toolkit validator spec @@ -2435,6 +2441,12 @@ spec: - name type: object type: array + skip: + description: |- + Skip indicates whether to skip the NVIDIA Device Plugin validation. When set to true, the + plugin-validation init container does not run the validation and only creates the + component readiness status file, so that other operands which depend on it are not blocked. + type: boolean type: object repository: description: Validator image repository @@ -2487,6 +2499,12 @@ spec: - name type: object type: array + skip: + description: |- + Skip indicates whether to skip the NVIDIA Container Toolkit validation. When set to true, the + toolkit-validation init container does not run the validation and only creates the + component readiness status file, so that other operands which depend on it are not blocked. + type: boolean type: object version: description: Validator image tag diff --git a/cmd/nvidia-validator/main.go b/cmd/nvidia-validator/main.go index 8d31f9d34..03adea907 100644 --- a/cmd/nvidia-validator/main.go +++ b/cmd/nvidia-validator/main.go @@ -138,6 +138,7 @@ var ( driverInstallDirFlag string driverInstallDirCtrPathFlag string driverValidationSkipGPUInitFlag bool + skipValidationFlag bool ) // defaultGPUWorkloadConfig is "vm-passthrough" unless @@ -385,6 +386,13 @@ func main() { Destination: &driverValidationSkipGPUInitFlag, Sources: cli.EnvVars("DRIVER_VALIDATION_SKIP_GPU_INIT"), }, + &cli.BoolFlag{ + Name: "skip-validation", + Value: false, + Usage: "skip the validation of the component and only create its readiness status file. Supported for the 'toolkit', 'cuda' and 'plugin' components.", + Destination: &skipValidationFlag, + Sources: cli.EnvVars("SKIP_VALIDATION"), + }, } // Log version info @@ -540,6 +548,10 @@ func start(ctx context.Context, cli *cli.Command) error { } func validateComponent(ctx context.Context, componentFlag string) error { + if skipValidationFlag { + return skipComponentValidation(componentFlag) + } + switch componentFlag { case "driver": driver := &Driver{ @@ -655,6 +667,32 @@ func validateComponent(ctx context.Context, componentFlag string) error { } } +// skipComponentValidation skips the validation of the given component and only creates +// its readiness status file, so that other operands waiting on the status file are not blocked. +// This is only supported for components whose status file carries no additional information. +func skipComponentValidation(component string) error { + var statusFile string + switch component { + case "toolkit": + statusFile = toolkitStatusFile + case "cuda": + statusFile = cudaStatusFile + case "plugin": + statusFile = pluginStatusFile + default: + return fmt.Errorf("skipping validation is not supported for component: %s", component) + } + + log.Warnf("skipping validation of component %q as requested; creating status file %s without validation", component, statusFile) + + // delete status file if already present + err := deleteStatusFile(outputDirFlag + "/" + statusFile) + if err != nil { + return err + } + return createStatusFile(outputDirFlag + "/" + statusFile) +} + func runCommand(command string, args []string, silent bool) error { cmd := exec.Command(command, args...) if !silent { diff --git a/cmd/nvidia-validator/main_test.go b/cmd/nvidia-validator/main_test.go index 7a521a687..94e9daef2 100644 --- a/cmd/nvidia-validator/main_test.go +++ b/cmd/nvidia-validator/main_test.go @@ -345,3 +345,39 @@ func TestCountNvidiaDevices(t *testing.T) { }) } } + +func TestSkipComponentValidation(t *testing.T) { + testCases := []struct { + name string + component string + statusFile string + errorExpected bool + }{ + {name: "toolkit", component: "toolkit", statusFile: toolkitStatusFile}, + {name: "cuda", component: "cuda", statusFile: cudaStatusFile}, + {name: "plugin", component: "plugin", statusFile: pluginStatusFile}, + {name: "driver is not supported", component: "driver", errorExpected: true}, + {name: "invalid component", component: "foo", errorExpected: true}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tmpDir := t.TempDir() + origOutputDir := outputDirFlag + outputDirFlag = tmpDir + defer func() { outputDirFlag = origOutputDir }() + + err := skipComponentValidation(tc.component) + if tc.errorExpected { + require.Error(t, err) + entries, readErr := os.ReadDir(tmpDir) + require.NoError(t, readErr) + require.Empty(t, entries, "no status file should be created") + return + } + require.NoError(t, err) + _, err = os.Stat(tmpDir + "/" + tc.statusFile) + require.NoError(t, err, "status file %s should be created", tc.statusFile) + }) + } +} diff --git a/config/crd/bases/nvidia.com_clusterpolicies.yaml b/config/crd/bases/nvidia.com_clusterpolicies.yaml index 6257b44d5..a30edeba0 100644 --- a/config/crd/bases/nvidia.com_clusterpolicies.yaml +++ b/config/crd/bases/nvidia.com_clusterpolicies.yaml @@ -2363,6 +2363,12 @@ spec: - name type: object type: array + skip: + description: |- + Skip indicates whether to skip the CUDA validation. When set to true, the + cuda-validation init container does not run the validation and only creates the + component readiness status file, so that other operands which depend on it are not blocked. + type: boolean type: object driver: description: Toolkit validator spec @@ -2435,6 +2441,12 @@ spec: - name type: object type: array + skip: + description: |- + Skip indicates whether to skip the NVIDIA Device Plugin validation. When set to true, the + plugin-validation init container does not run the validation and only creates the + component readiness status file, so that other operands which depend on it are not blocked. + type: boolean type: object repository: description: Validator image repository @@ -2487,6 +2499,12 @@ spec: - name type: object type: array + skip: + description: |- + Skip indicates whether to skip the NVIDIA Container Toolkit validation. When set to true, the + toolkit-validation init container does not run the validation and only creates the + component readiness status file, so that other operands which depend on it are not blocked. + type: boolean type: object version: description: Validator image tag diff --git a/controllers/object_controls.go b/controllers/object_controls.go index c1bf59e4c..9c460fef5 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -107,6 +107,9 @@ const ( ValidatorImagePullSecretsEnvName = "VALIDATOR_IMAGE_PULL_SECRETS" // ValidatorRuntimeClassEnvName indicates env name of runtime class to be applied to validator pods ValidatorRuntimeClassEnvName = "VALIDATOR_RUNTIME_CLASS" + // ValidatorSkipValidationEnvName indicates env name used to instruct a component validator to skip + // the actual validation and only create the component readiness status file + ValidatorSkipValidationEnvName = "SKIP_VALIDATION" // MigStrategyEnvName indicates env name for passing MIG strategy MigStrategyEnvName = "MIG_STRATEGY" // MigPartedDefaultConfigMapName indicates name of ConfigMap containing default mig-parted config @@ -2460,6 +2463,11 @@ func TransformValidatorComponent(config *gpuv1.ClusterPolicySpec, podSpec *corev setContainerEnv(&(podSpec.InitContainers[i]), env.Name, env.Value) } } + // skip cuda validation if requested. Applied after the user-provided env so an + // explicit 'skip: true' cannot be undone by a conflicting SKIP_VALIDATION entry. + if config.Validator.CUDA.IsSkipped() { + setContainerEnv(&(podSpec.InitContainers[i]), ValidatorSkipValidationEnvName, "true") + } case "plugin": // remove plugin init container from validator Daemonset if it is not enabled if !config.DevicePlugin.IsEnabled() { @@ -2485,6 +2493,11 @@ func TransformValidatorComponent(config *gpuv1.ClusterPolicySpec, podSpec *corev setContainerEnv(&(podSpec.InitContainers[i]), env.Name, env.Value) } } + // skip plugin validation if requested. Applied after the user-provided env so an + // explicit 'skip: true' cannot be undone by a conflicting SKIP_VALIDATION entry. + if config.Validator.Plugin.IsSkipped() { + setContainerEnv(&(podSpec.InitContainers[i]), ValidatorSkipValidationEnvName, "true") + } case "driver": // set/append environment variables for driver-validation container if len(config.Validator.Driver.Env) > 0 { @@ -2505,6 +2518,14 @@ func TransformValidatorComponent(config *gpuv1.ClusterPolicySpec, podSpec *corev setContainerEnv(&(podSpec.InitContainers[i]), env.Name, env.Value) } } + // skip toolkit validation if requested. Note that the init container is intentionally + // not removed: it still creates the 'toolkit-ready' status file which other operands + // (e.g. device-plugin, gpu-feature-discovery, dcgm-exporter, mig-manager) wait for. + // Applied after the user-provided env so an explicit 'skip: true' cannot be undone by + // a conflicting SKIP_VALIDATION entry. + if config.Validator.Toolkit.IsSkipped() { + setContainerEnv(&(podSpec.InitContainers[i]), ValidatorSkipValidationEnvName, "true") + } case "vfio-pci": // set/append environment variables for vfio-pci-validation container setContainerEnv(&(podSpec.InitContainers[i]), "DEFAULT_GPU_WORKLOAD_CONFIG", defaultGPUWorkloadConfig) diff --git a/controllers/transforms_test.go b/controllers/transforms_test.go index 8931298a4..a42aed9a5 100644 --- a/controllers/transforms_test.go +++ b/controllers/transforms_test.go @@ -2860,6 +2860,186 @@ func TestTransformValidatorComponent(t *testing.T) { component: "cc-manager", expectedPod: NewPod().WithInitContainer(corev1.Container{Name: "dummy"}), }, + { + description: "cuda validation skipped", + pod: NewPod(). + WithInitContainer(corev1.Container{Name: "cuda-validation"}). + WithRuntimeClassName("nvidia"), + cpSpec: &gpuv1.ClusterPolicySpec{ + Validator: gpuv1.ValidatorSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "gpu-operator-validator", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + CUDA: gpuv1.CUDAValidatorSpec{ + Skip: newBoolPtr(true), + Env: []gpuv1.EnvVar{{Name: "foo", Value: "bar"}}, + }, + }, + }, + component: "cuda", + expectedPod: NewPod().WithInitContainer(corev1.Container{ + Name: "cuda-validation", + Image: "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + {Name: ValidatorImageEnvName, Value: "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v1.0.0"}, + {Name: ValidatorImagePullPolicyEnvName, Value: "IfNotPresent"}, + {Name: ValidatorRuntimeClassEnvName, Value: "nvidia"}, + {Name: "foo", Value: "bar"}, + {Name: ValidatorSkipValidationEnvName, Value: "true"}, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: rootUID, + }, + }).WithRuntimeClassName("nvidia"), + }, + { + description: "plugin validation skipped", + pod: NewPod(). + WithInitContainer(corev1.Container{Name: "plugin-validation"}). + WithRuntimeClassName("nvidia"), + cpSpec: &gpuv1.ClusterPolicySpec{ + Validator: gpuv1.ValidatorSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "gpu-operator-validator", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + Plugin: gpuv1.PluginValidatorSpec{ + Skip: newBoolPtr(true), + }, + }, + MIG: gpuv1.MIGSpec{ + Strategy: gpuv1.MIGStrategySingle, + }, + }, + component: "plugin", + expectedPod: NewPod().WithInitContainer(corev1.Container{ + Name: "plugin-validation", + Image: "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + {Name: ValidatorImageEnvName, Value: "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v1.0.0"}, + {Name: ValidatorImagePullPolicyEnvName, Value: "IfNotPresent"}, + {Name: ValidatorRuntimeClassEnvName, Value: "nvidia"}, + {Name: MigStrategyEnvName, Value: string(gpuv1.MIGStrategySingle)}, + {Name: ValidatorSkipValidationEnvName, Value: "true"}, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: rootUID, + }, + }).WithRuntimeClassName("nvidia"), + }, + { + description: "plugin validation removed when plugin is disabled even if skipped", + pod: NewPod(). + WithInitContainer(corev1.Container{Name: "plugin-validation"}). + WithInitContainer(corev1.Container{Name: "dummy"}), + cpSpec: &gpuv1.ClusterPolicySpec{ + Validator: gpuv1.ValidatorSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "gpu-operator-validator", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + Plugin: gpuv1.PluginValidatorSpec{ + Skip: newBoolPtr(true), + }, + }, + DevicePlugin: gpuv1.DevicePluginSpec{Enabled: newBoolPtr(false)}, + }, + component: "plugin", + expectedPod: NewPod().WithInitContainer(corev1.Container{Name: "dummy"}), + }, + { + description: "toolkit validation skipped", + pod: NewPod(). + WithInitContainer(corev1.Container{Name: "driver-validation"}). + WithInitContainer(corev1.Container{Name: "toolkit-validation"}), + cpSpec: &gpuv1.ClusterPolicySpec{ + Validator: gpuv1.ValidatorSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "gpu-operator-validator", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + Toolkit: gpuv1.ToolkitValidatorSpec{ + Skip: newBoolPtr(true), + Env: []gpuv1.EnvVar{{Name: "foo", Value: "bar"}}, + }, + }, + Toolkit: gpuv1.ToolkitSpec{Enabled: newBoolPtr(false)}, + }, + component: "toolkit", + expectedPod: NewPod(). + WithInitContainer(corev1.Container{Name: "driver-validation"}). + WithInitContainer(corev1.Container{ + Name: "toolkit-validation", + Image: "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + {Name: "foo", Value: "bar"}, + {Name: ValidatorSkipValidationEnvName, Value: "true"}, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: rootUID, + }, + }), + }, + { + description: "toolkit skip wins over a conflicting SKIP_VALIDATION user env entry", + pod: NewPod().WithInitContainer(corev1.Container{Name: "toolkit-validation"}), + cpSpec: &gpuv1.ClusterPolicySpec{ + Validator: gpuv1.ValidatorSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "gpu-operator-validator", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + Toolkit: gpuv1.ToolkitValidatorSpec{ + Skip: newBoolPtr(true), + Env: []gpuv1.EnvVar{ + {Name: "foo", Value: "bar"}, + {Name: ValidatorSkipValidationEnvName, Value: "false"}, + }, + }, + }, + }, + component: "toolkit", + expectedPod: NewPod().WithInitContainer(corev1.Container{ + Name: "toolkit-validation", + Image: "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + {Name: "foo", Value: "bar"}, + {Name: ValidatorSkipValidationEnvName, Value: "true"}, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: rootUID, + }, + }), + }, + { + description: "toolkit validation not skipped when skip is explicitly false", + pod: NewPod().WithInitContainer(corev1.Container{Name: "toolkit-validation"}), + cpSpec: &gpuv1.ClusterPolicySpec{ + Validator: gpuv1.ValidatorSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "gpu-operator-validator", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + Toolkit: gpuv1.ToolkitValidatorSpec{ + Skip: newBoolPtr(false), + }, + }, + }, + component: "toolkit", + expectedPod: NewPod().WithInitContainer(corev1.Container{ + Name: "toolkit-validation", + Image: "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: rootUID, + }, + }), + }, { description: "toolkit validation", pod: NewPod().WithInitContainer(corev1.Container{Name: "toolkit-validation"}), @@ -3136,6 +3316,73 @@ func TestTransformValidator(t *testing.T) { }). WithPullSecret("pull-secret"), }, + { + // e.g. GKE with preinstalled toolkit and device plugin: the toolkit-validation init + // container must be kept (it creates the 'toolkit-ready' status file for other operands) + // but must not run the actual validation, and plugin-validation must be removed. + description: "toolkit validation skipped with toolkit and device plugin disabled", + ds: NewDaemonset(). + WithInitContainer(corev1.Container{Name: "driver-validation"}). + WithInitContainer(corev1.Container{Name: "toolkit-validation"}). + WithInitContainer(corev1.Container{Name: "cuda-validation"}). + WithInitContainer(corev1.Container{Name: "plugin-validation"}). + WithContainer(corev1.Container{Name: "dummy"}), + cpSpec: &gpuv1.ClusterPolicySpec{ + Validator: gpuv1.ValidatorSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "gpu-operator-validator", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + Toolkit: gpuv1.ToolkitValidatorSpec{ + Skip: newBoolPtr(true), + }, + }, + Toolkit: gpuv1.ToolkitSpec{Enabled: newBoolPtr(false)}, + DevicePlugin: gpuv1.DevicePluginSpec{Enabled: newBoolPtr(false)}, + }, + expectedDs: NewDaemonset(). + WithInitContainer(corev1.Container{ + Name: "driver-validation", + Image: "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: rootUID, + }, + }). + WithInitContainer(corev1.Container{ + Name: "toolkit-validation", + Image: "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + {Name: ValidatorSkipValidationEnvName, Value: "true"}, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: rootUID, + }, + }). + WithInitContainer(corev1.Container{ + Name: "cuda-validation", + Image: "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + {Name: ValidatorImageEnvName, Value: "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v1.0.0"}, + {Name: ValidatorImagePullPolicyEnvName, Value: "IfNotPresent"}, + {Name: ValidatorRuntimeClassEnvName, Value: "nvidia"}, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: rootUID, + }, + }). + WithContainer(corev1.Container{ + Name: "dummy", + Image: "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: rootUID, + }, + }). + WithRuntimeClassName("nvidia"), + }, } for _, tc := range testCases { diff --git a/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml b/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml index 6257b44d5..a30edeba0 100644 --- a/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml +++ b/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml @@ -2363,6 +2363,12 @@ spec: - name type: object type: array + skip: + description: |- + Skip indicates whether to skip the CUDA validation. When set to true, the + cuda-validation init container does not run the validation and only creates the + component readiness status file, so that other operands which depend on it are not blocked. + type: boolean type: object driver: description: Toolkit validator spec @@ -2435,6 +2441,12 @@ spec: - name type: object type: array + skip: + description: |- + Skip indicates whether to skip the NVIDIA Device Plugin validation. When set to true, the + plugin-validation init container does not run the validation and only creates the + component readiness status file, so that other operands which depend on it are not blocked. + type: boolean type: object repository: description: Validator image repository @@ -2487,6 +2499,12 @@ spec: - name type: object type: array + skip: + description: |- + Skip indicates whether to skip the NVIDIA Container Toolkit validation. When set to true, the + toolkit-validation init container does not run the validation and only creates the + component readiness status file, so that other operands which depend on it are not blocked. + type: boolean type: object version: description: Validator image tag diff --git a/deployments/gpu-operator/templates/clusterpolicy.yaml b/deployments/gpu-operator/templates/clusterpolicy.yaml index e156f5b7a..1590b259b 100644 --- a/deployments/gpu-operator/templates/clusterpolicy.yaml +++ b/deployments/gpu-operator/templates/clusterpolicy.yaml @@ -87,6 +87,9 @@ spec: {{- else }} env: [] {{- end }} + {{- if .Values.validator.plugin.skip }} + skip: {{ .Values.validator.plugin.skip }} + {{- end }} {{- end }} {{- if .Values.validator.cuda }} cuda: @@ -95,6 +98,9 @@ spec: {{- else }} env: [] {{- end }} + {{- if .Values.validator.cuda.skip }} + skip: {{ .Values.validator.cuda.skip }} + {{- end }} {{- end }} {{- if .Values.validator.driver }} driver: @@ -111,6 +117,9 @@ spec: {{- else }} env: [] {{- end }} + {{- if .Values.validator.toolkit.skip }} + skip: {{ .Values.validator.toolkit.skip }} + {{- end }} {{- end }} {{- if .Values.validator.vfioPCI }} vfioPCI: diff --git a/deployments/gpu-operator/values.yaml b/deployments/gpu-operator/values.yaml index f6222b0d5..9b287be0f 100644 --- a/deployments/gpu-operator/values.yaml +++ b/deployments/gpu-operator/values.yaml @@ -68,6 +68,18 @@ validator: hostNetwork: false plugin: env: [] + # skip the device plugin validation (the 'plugin-ready' status file is still created) + skip: false + toolkit: + env: [] + # skip the container toolkit validation, e.g. when the toolkit is preinstalled on the host and + # 'nvidia-smi' is not injected into containers that do not request a GPU resource (e.g. GKE COS). + # The 'toolkit-ready' status file is still created so that other operands are not blocked. + skip: false + cuda: + env: [] + # skip the CUDA workload validation (the 'cuda-ready' status file is still created) + skip: false operator: repository: nvcr.io/nvidia