Add option to skip individual component validators (toolkit, cuda, plugin)

api/nvidia/v1/clusterpolicy_types.go

@@ -431,6 +431,15 @@ type ValidatorSpec struct {
 
 // PluginValidatorSpec defines validator spec for NVIDIA Device Plugin
 type PluginValidatorSpec struct {
+	// Skip indicates whether to skip the NVIDIA Device Plugin validation. When set to true, the
+	// plugin-validation init container does not run the validation and only creates the
+	// component readiness status file, so that other operands which depend on it are not blocked.
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Skip NVIDIA Device Plugin validation"
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch"
+	Skip *bool `json:"skip,omitempty"`
+
 	// Optional: List of environment variables
 	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
 	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Environment Variables"
@@ -440,6 +449,15 @@ type PluginValidatorSpec struct {
 
 // ToolkitValidatorSpec defines validator spec for NVIDIA Container Toolkit
 type ToolkitValidatorSpec struct {
+	// Skip indicates whether to skip the NVIDIA Container Toolkit validation. When set to true, the
+	// toolkit-validation init container does not run the validation and only creates the
+	// component readiness status file, so that other operands which depend on it are not blocked.
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Skip NVIDIA Container Toolkit validation"
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch"
+	Skip *bool `json:"skip,omitempty"`
+
 	// Optional: List of environment variables
 	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
 	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Environment Variables"
@@ -458,6 +476,15 @@ type DriverValidatorSpec struct {
 
 // CUDAValidatorSpec defines validator spec for CUDA validation workload pod
 type CUDAValidatorSpec struct {
+	// Skip indicates whether to skip the CUDA validation. When set to true, the
+	// cuda-validation init container does not run the validation and only creates the
+	// component readiness status file, so that other operands which depend on it are not blocked.
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Skip CUDA validation"
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch"
+	Skip *bool `json:"skip,omitempty"`
+
 	// Optional: List of environment variables
 	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
 	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Environment Variables"
@@ -2224,6 +2251,33 @@ func (d *DriverSpec) IsAutoUpgradeEnabled() bool {
 	return d.UpgradePolicy.AutoUpgrade
 }
 
+// IsSkipped returns true if the NVIDIA Device Plugin validation is skipped
+func (p *PluginValidatorSpec) IsSkipped() bool {
+	if p.Skip == nil {
+		// default is false if not specified by user
+		return false
+	}
+	return *p.Skip
+}
+
+// IsSkipped returns true if the NVIDIA Container Toolkit validation is skipped
+func (t *ToolkitValidatorSpec) IsSkipped() bool {
+	if t.Skip == nil {
+		// default is false if not specified by user
+		return false
+	}
+	return *t.Skip
+}
+
+// IsSkipped returns true if the CUDA validation is skipped
+func (c *CUDAValidatorSpec) IsSkipped() bool {
+	if c.Skip == nil {
+		// default is false if not specified by user
+		return false
+	}
+	return *c.Skip
+}
+
 // IsEnabled returns true if device-plugin is enabled(default) through gpu-operator
 func (p *DevicePluginSpec) IsEnabled() bool {
 	if p.Enabled == nil {

bundle/manifests/nvidia.com_clusterpolicies.yaml

@@ -2363,6 +2363,12 @@ spec:
                           - name
                           type: object
                         type: array
+                      skip:
+                        description: |-
+                          Skip indicates whether to skip the CUDA validation. When set to true, the
+                          cuda-validation init container does not run the validation and only creates the
+                          component readiness status file, so that other operands which depend on it are not blocked.
+                        type: boolean
                     type: object
                   driver:
                     description: Toolkit validator spec
@@ -2435,6 +2441,12 @@ spec:
                           - name
                           type: object
                         type: array
+                      skip:
+                        description: |-
+                          Skip indicates whether to skip the NVIDIA Device Plugin validation. When set to true, the
+                          plugin-validation init container does not run the validation and only creates the
+                          component readiness status file, so that other operands which depend on it are not blocked.
+                        type: boolean
                     type: object
                   repository:
                     description: Validator image repository
@@ -2487,6 +2499,12 @@ spec:
                           - name
                           type: object
                         type: array
+                      skip:
+                        description: |-
+                          Skip indicates whether to skip the NVIDIA Container Toolkit validation. When set to true, the
+                          toolkit-validation init container does not run the validation and only creates the
+                          component readiness status file, so that other operands which depend on it are not blocked.
+                        type: boolean
                     type: object
                   version:
                     description: Validator image tag

cmd/nvidia-validator/main.go

@@ -138,6 +138,7 @@ var (
 	driverInstallDirFlag            string
 	driverInstallDirCtrPathFlag     string
 	driverValidationSkipGPUInitFlag bool
+	skipValidationFlag              bool
 )
 
 // defaultGPUWorkloadConfig is "vm-passthrough" unless
@@ -385,6 +386,13 @@ func main() {
 			Destination: &driverValidationSkipGPUInitFlag,
 			Sources:     cli.EnvVars("DRIVER_VALIDATION_SKIP_GPU_INIT"),
 		},
+		&cli.BoolFlag{
+			Name:        "skip-validation",
+			Value:       false,
+			Usage:       "skip the validation of the component and only create its readiness status file. Supported for the 'toolkit', 'cuda' and 'plugin' components.",
+			Destination: &skipValidationFlag,
+			Sources:     cli.EnvVars("SKIP_VALIDATION"),
+		},
 	}
 
 	// Log version info
@@ -540,6 +548,10 @@ func start(ctx context.Context, cli *cli.Command) error {
 }
 
 func validateComponent(ctx context.Context, componentFlag string) error {
+	if skipValidationFlag {
+		return skipComponentValidation(componentFlag)
+	}
+
 	switch componentFlag {
 	case "driver":
 		driver := &Driver{
@@ -655,6 +667,32 @@ func validateComponent(ctx context.Context, componentFlag string) error {
 	}
 }
 
+// skipComponentValidation skips the validation of the given component and only creates
+// its readiness status file, so that other operands waiting on the status file are not blocked.
+// This is only supported for components whose status file carries no additional information.
+func skipComponentValidation(component string) error {
+	var statusFile string
+	switch component {
+	case "toolkit":
+		statusFile = toolkitStatusFile
+	case "cuda":
+		statusFile = cudaStatusFile
+	case "plugin":
+		statusFile = pluginStatusFile
+	default:
+		return fmt.Errorf("skipping validation is not supported for component: %s", component)
+	}
+
+	log.Warnf("skipping validation of component %q as requested; creating status file %s without validation", component, statusFile)
+
+	// delete status file if already present
+	err := deleteStatusFile(outputDirFlag + "/" + statusFile)
+	if err != nil {
+		return err
+	}
+	return createStatusFile(outputDirFlag + "/" + statusFile)
+}
+
 func runCommand(command string, args []string, silent bool) error {
 	cmd := exec.Command(command, args...)
 	if !silent {

cmd/nvidia-validator/main_test.go

@@ -345,3 +345,39 @@ func TestCountNvidiaDevices(t *testing.T) {
 		})
 	}
 }
+
+func TestSkipComponentValidation(t *testing.T) {
+	testCases := []struct {
+		name          string
+		component     string
+		statusFile    string
+		errorExpected bool
+	}{
+		{name: "toolkit", component: "toolkit", statusFile: toolkitStatusFile},
+		{name: "cuda", component: "cuda", statusFile: cudaStatusFile},
+		{name: "plugin", component: "plugin", statusFile: pluginStatusFile},
+		{name: "driver is not supported", component: "driver", errorExpected: true},
+		{name: "invalid component", component: "foo", errorExpected: true},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			tmpDir := t.TempDir()
+			origOutputDir := outputDirFlag
+			outputDirFlag = tmpDir
+			defer func() { outputDirFlag = origOutputDir }()
+
+			err := skipComponentValidation(tc.component)
+			if tc.errorExpected {
+				require.Error(t, err)
+				entries, readErr := os.ReadDir(tmpDir)
+				require.NoError(t, readErr)
+				require.Empty(t, entries, "no status file should be created")
+				return
+			}
+			require.NoError(t, err)
+			_, err = os.Stat(tmpDir + "/" + tc.statusFile)
+			require.NoError(t, err, "status file %s should be created", tc.statusFile)
+		})
+	}
+}

config/crd/bases/nvidia.com_clusterpolicies.yaml

@@ -2363,6 +2363,12 @@ spec:
                           - name
                           type: object
                         type: array
+                      skip:
+                        description: |-
+                          Skip indicates whether to skip the CUDA validation. When set to true, the
+                          cuda-validation init container does not run the validation and only creates the
+                          component readiness status file, so that other operands which depend on it are not blocked.
+                        type: boolean
                     type: object
                   driver:
                     description: Toolkit validator spec
@@ -2435,6 +2441,12 @@ spec:
                           - name
                           type: object
                         type: array
+                      skip:
+                        description: |-
+                          Skip indicates whether to skip the NVIDIA Device Plugin validation. When set to true, the
+                          plugin-validation init container does not run the validation and only creates the
+                          component readiness status file, so that other operands which depend on it are not blocked.
+                        type: boolean
                     type: object
                   repository:
                     description: Validator image repository
@@ -2487,6 +2499,12 @@ spec:
                           - name
                           type: object
                         type: array
+                      skip:
+                        description: |-
+                          Skip indicates whether to skip the NVIDIA Container Toolkit validation. When set to true, the
+                          toolkit-validation init container does not run the validation and only creates the
+                          component readiness status file, so that other operands which depend on it are not blocked.
+                        type: boolean
                     type: object
                   version:
                     description: Validator image tag

controllers/object_controls.go

@@ -107,6 +107,9 @@ const (
 	ValidatorImagePullSecretsEnvName = "VALIDATOR_IMAGE_PULL_SECRETS"
 	// ValidatorRuntimeClassEnvName indicates env name of runtime class to be applied to validator pods
 	ValidatorRuntimeClassEnvName = "VALIDATOR_RUNTIME_CLASS"
+	// ValidatorSkipValidationEnvName indicates env name used to instruct a component validator to skip
+	// the actual validation and only create the component readiness status file
+	ValidatorSkipValidationEnvName = "SKIP_VALIDATION"
 	// MigStrategyEnvName indicates env name for passing MIG strategy
 	MigStrategyEnvName = "MIG_STRATEGY"
 	// MigPartedDefaultConfigMapName indicates name of ConfigMap containing default mig-parted config
@@ -2460,6 +2463,11 @@ func TransformValidatorComponent(config *gpuv1.ClusterPolicySpec, podSpec *corev
 					setContainerEnv(&(podSpec.InitContainers[i]), env.Name, env.Value)
 				}
 			}
+			// skip cuda validation if requested. Applied after the user-provided env so an
+			// explicit 'skip: true' cannot be undone by a conflicting SKIP_VALIDATION entry.
+			if config.Validator.CUDA.IsSkipped() {
+				setContainerEnv(&(podSpec.InitContainers[i]), ValidatorSkipValidationEnvName, "true")
+			}
 		case "plugin":
 			// remove plugin init container from validator Daemonset if it is not enabled
 			if !config.DevicePlugin.IsEnabled() {
@@ -2485,6 +2493,11 @@ func TransformValidatorComponent(config *gpuv1.ClusterPolicySpec, podSpec *corev
 					setContainerEnv(&(podSpec.InitContainers[i]), env.Name, env.Value)
 				}
 			}
+			// skip plugin validation if requested. Applied after the user-provided env so an
+			// explicit 'skip: true' cannot be undone by a conflicting SKIP_VALIDATION entry.
+			if config.Validator.Plugin.IsSkipped() {
+				setContainerEnv(&(podSpec.InitContainers[i]), ValidatorSkipValidationEnvName, "true")
+			}
 		case "driver":
 			// set/append environment variables for driver-validation container
 			if len(config.Validator.Driver.Env) > 0 {
@@ -2505,6 +2518,14 @@ func TransformValidatorComponent(config *gpuv1.ClusterPolicySpec, podSpec *corev
 					setContainerEnv(&(podSpec.InitContainers[i]), env.Name, env.Value)
 				}
 			}
+			// skip toolkit validation if requested. Note that the init container is intentionally
+			// not removed: it still creates the 'toolkit-ready' status file which other operands
+			// (e.g. device-plugin, gpu-feature-discovery, dcgm-exporter, mig-manager) wait for.
+			// Applied after the user-provided env so an explicit 'skip: true' cannot be undone by
+			// a conflicting SKIP_VALIDATION entry.
+			if config.Validator.Toolkit.IsSkipped() {
+				setContainerEnv(&(podSpec.InitContainers[i]), ValidatorSkipValidationEnvName, "true")
+			}
 		case "vfio-pci":
 			// set/append environment variables for vfio-pci-validation container
 			setContainerEnv(&(podSpec.InitContainers[i]), "DEFAULT_GPU_WORKLOAD_CONFIG", defaultGPUWorkloadConfig)