diff --git a/crates/api-core/src/cfg/file.rs b/crates/api-core/src/cfg/file.rs index 0fe2b23095..ba3a89d778 100644 --- a/crates/api-core/src/cfg/file.rs +++ b/crates/api-core/src/cfg/file.rs @@ -1174,9 +1174,17 @@ pub struct DpfServiceConfig { /// `bf3` field of [`DpfDeploymentsConfig`]. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct DpfDeploymentConfig { - /// URL to the BlueField firmware bundle (BFB) for DPU provisioning. - #[serde(default = "default_dpf_bfb_url")] - pub bfb_url: String, + /// URL to the BlueField firmware bundle (BFB) for DPU provisioning + /// (BF3-class DPUs). Exactly one of `bfb_url` or `bluefield_software` + /// must be set per deployment (see + /// [`DpfDeploymentsConfig::validate_provisioning_sources`]). + #[serde(default)] + pub bfb_url: Option, + /// BlueFieldSoftware spec for BF4-class DPUs. When set, a `BlueFieldSoftware` + /// CR is created and referenced by the DPUDeployment instead of a BFB. + /// Mutually exclusive with `bfb_url`. + #[serde(default)] + pub bluefield_software: Option, /// Kubernetes DPUFlavor CR name. pub flavor_name: String, /// Kubernetes DPUDeployment CR name. @@ -1194,7 +1202,8 @@ pub struct DpfDeploymentConfig { impl Default for DpfDeploymentConfig { fn default() -> Self { Self { - bfb_url: default_dpf_bfb_url(), + bfb_url: Some(default_dpf_bfb_url()), + bluefield_software: None, flavor_name: default_dpf_flavor_name(), deployment_name: default_dpf_deployment_name(), node_label_key: default_dpf_node_label_key(), @@ -1203,6 +1212,25 @@ impl Default for DpfDeploymentConfig { } } +/// BlueFieldSoftware spec for BF4-class DPU provisioning. Mirrors the `spec` of +/// the `provisioning.dpu.nvidia.com/v1alpha1` `BlueFieldSoftware` CR. +/// +/// The PLDM firmware bundle is PSID-specific, so `pldm_fw_bundle` maps each PSID +/// to its bundle URL. One `BlueFieldSoftware` CR and one DPUDeployment are +/// created per PSID (see +/// [`DpfDeploymentConfig::per_psid_deployment_name`] and +/// [`DpfDeploymentConfig::per_psid_node_label_key`]). +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct DpfBlueFieldSoftwareConfig { + /// OS ISO URL used by the DPU OS installation flow (`spec.osIso`). Shared + /// across all PSIDs. + pub os_iso: String, + /// Map of PSID → PLDM firmware bundle URL (`spec.pldmFwBundle`). Each entry + /// fans out to its own `BlueFieldSoftware` CR and DPUDeployment. + #[serde(default)] + pub pldm_fw_bundle: BTreeMap, +} + /// Named DPUDeployment configurations under `[dpf.deployments]`. /// Each entry creates its own BFB, DPUFlavor, and DPUDeployment CR at startup. #[derive(Clone, Debug, Default, Serialize, Deserialize)] @@ -1271,6 +1299,72 @@ impl DpfDeploymentsConfig { )) } } + + /// Validates that each active deployment specifies exactly one provisioning + /// source: either `bfb_url` (BF3) or `bluefield_software` (BF4), never both + /// and never neither. This mirrors the DPUDeployment CRD rule requiring + /// exactly one of `spec.dpus.bfb` / `spec.dpus.blueFieldSoftware`. Returns an + /// error listing every offending deployment so they can be fixed in one pass. + /// + /// Additionally enforces the hard rule that the `bf3` deployment is BFB-only: + /// it must use `bfb_url` and must never set `bluefield_software` (BF4-only). + pub fn validate_provisioning_sources(&self) -> eyre::Result<()> { + let mut errors: Vec = Vec::new(); + + // BF3 is BFB-only. `bluefield_software` is BF4-specific and is never + // valid on the bf3 deployment, regardless of whether bfb_url is also set. + if self.bf3.bluefield_software.is_some() { + errors.push( + "deployment \"bf3\" must not set bluefield_software; BF3 uses bfb_url only" + .to_string(), + ); + } + + // BF4 is BlueFieldSoftware-only. `bfb_url` is BF3-specific; a bf4_generic + // deployment must use `bluefield_software`. Reject the BFB-only case here + // so it fails at config validation rather than later at SDK startup, + // which unconditionally requires `bluefield_software` for bf4_generic. + if self + .bf4_generic + .as_ref() + .is_some_and(|cfg| cfg.bfb_url.is_some() && cfg.bluefield_software.is_none()) + { + errors.push( + "deployment \"bf4_generic\" must set bluefield_software; BF4 does not support bfb_url" + .to_string(), + ); + } + + for (name, cfg) in self.all() { + match (&cfg.bfb_url, &cfg.bluefield_software) { + (Some(_), Some(_)) => errors.push(format!( + "deployment {name:?} sets both bfb_url and bluefield_software; set exactly one" + )), + (None, None) => errors.push(format!( + "deployment {name:?} sets neither bfb_url nor bluefield_software; set exactly one" + )), + // Exactly one PSID entry is allowed for now. Multi-PSID support + // is pending a DPF change that lets one `BlueFieldSoftware` CR + // carry a PSID→PLDM map; until then a single BF4 deployment uses + // the one entry's PLDM bundle. + (None, Some(bfs)) if bfs.pldm_fw_bundle.len() != 1 => errors.push(format!( + "deployment {name:?} bluefield_software.pldm_fw_bundle must have exactly one \ + PSID → PLDM bundle URL entry (found {}).", + bfs.pldm_fw_bundle.len() + )), + _ => {} + } + } + + if errors.is_empty() { + Ok(()) + } else { + Err(eyre::eyre!( + "DPF deployment configuration has invalid provisioning sources:\n - {}", + errors.join("\n - ") + )) + } + } } /// Machine identity (SPIFFE JWT-SVID) configuration. @@ -4829,4 +4923,128 @@ firmware_url = "https://firmware.example.com/fw-b.bin" assert!(config.secrets.is_none()); } + + fn bf4_config( + bfb_url: Option<&str>, + bfs: Option, + ) -> DpfDeploymentConfig { + DpfDeploymentConfig { + bfb_url: bfb_url.map(str::to_string), + bluefield_software: bfs, + flavor_name: "bf4-flavor".to_string(), + deployment_name: "bf4-dep".to_string(), + node_label_key: "carbide.nvidia.com/bf4".to_string(), + services: None, + } + } + + #[test] + fn validate_provisioning_sources_accepts_exactly_one() { + // bf3 default has bfb_url; bf4 has bluefield_software with one PSID. + let deployments = DpfDeploymentsConfig { + bf3: DpfDeploymentConfig::default(), + bf4_generic: Some(bf4_config( + None, + Some(DpfBlueFieldSoftwareConfig { + os_iso: "http://example.com/os.iso".to_string(), + pldm_fw_bundle: BTreeMap::from([( + "MT_0000000884".to_string(), + "http://example.com/fw.pldm".to_string(), + )]), + }), + )), + }; + assert!(deployments.validate_provisioning_sources().is_ok()); + } + + #[test] + fn validate_provisioning_sources_rejects_both_and_neither_and_empty_map() { + // Both sources set. + let both = DpfDeploymentsConfig { + bf3: DpfDeploymentConfig::default(), + bf4_generic: Some(bf4_config( + Some("http://example.com/test.bfb"), + Some(DpfBlueFieldSoftwareConfig { + os_iso: "http://example.com/os.iso".to_string(), + pldm_fw_bundle: BTreeMap::from([( + "MT_0000000884".to_string(), + "http://example.com/fw.pldm".to_string(), + )]), + }), + )), + }; + assert!(both.validate_provisioning_sources().is_err()); + + // Neither source set. + let neither = DpfDeploymentsConfig { + bf3: DpfDeploymentConfig::default(), + bf4_generic: Some(bf4_config(None, None)), + }; + assert!(neither.validate_provisioning_sources().is_err()); + + // bluefield_software set but empty PSID map. + let empty_map = DpfDeploymentsConfig { + bf3: DpfDeploymentConfig::default(), + bf4_generic: Some(bf4_config( + None, + Some(DpfBlueFieldSoftwareConfig { + os_iso: "http://example.com/os.iso".to_string(), + pldm_fw_bundle: BTreeMap::new(), + }), + )), + }; + assert!(empty_map.validate_provisioning_sources().is_err()); + } + + #[test] + fn validate_provisioning_sources_rejects_bf3_bluefield_software() { + // bf3 is BFB-only: setting bluefield_software on it is always invalid, + // even though the same block would be valid on bf4_generic. + let deployments = DpfDeploymentsConfig { + bf3: bf4_config(None, Some(bf4_with_psids(&["MT_0000000884"]))), + bf4_generic: None, + }; + assert!(deployments.validate_provisioning_sources().is_err()); + } + + fn bf4_with_psids(psids: &[&str]) -> DpfBlueFieldSoftwareConfig { + DpfBlueFieldSoftwareConfig { + os_iso: "http://example.com/os.iso".to_string(), + pldm_fw_bundle: psids + .iter() + .map(|p| (p.to_string(), format!("http://example.com/{p}.pldm"))) + .collect(), + } + } + + #[test] + fn validate_provisioning_sources_rejects_bf4_bfb_url() { + // bf4_generic is BlueFieldSoftware-only: bfb_url without bluefield_software + // passes the exactly-one check but fails at SDK startup, so reject it here. + let deployments = DpfDeploymentsConfig { + bf3: DpfDeploymentConfig::default(), + bf4_generic: Some(bf4_config(Some("http://example.com/test.bfb"), None)), + }; + assert!(deployments.validate_provisioning_sources().is_err()); + } + + #[test] + fn validate_provisioning_sources_requires_exactly_one_psid() { + // Exactly one PSID entry is accepted. + let one = DpfDeploymentsConfig { + bf3: DpfDeploymentConfig::default(), + bf4_generic: Some(bf4_config(None, Some(bf4_with_psids(&["MT_0000000884"])))), + }; + assert!(one.validate_provisioning_sources().is_ok()); + + // More than one PSID is rejected (multi-PSID support is pending a DPF change). + let many = DpfDeploymentsConfig { + bf3: DpfDeploymentConfig::default(), + bf4_generic: Some(bf4_config( + None, + Some(bf4_with_psids(&["MT_0000000884", "MT_0000000992"])), + )), + }; + assert!(many.validate_provisioning_sources().is_err()); + } } diff --git a/crates/api-core/src/setup.rs b/crates/api-core/src/setup.rs index 75daf0a36a..03262e506f 100644 --- a/crates/api-core/src/setup.rs +++ b/crates/api-core/src/setup.rs @@ -667,6 +667,12 @@ async fn initialize_dpf_sdk( .validate_unique_identifiers() .map_err(|err| eyre::eyre!("Invalid DPF deployment configuration: {err}"))?; + carbide_config + .dpf + .deployments + .validate_provisioning_sources() + .map_err(|err| eyre::eyre!("Invalid DPF deployment configuration: {err}"))?; + // This is just temporary code until we make v2 only option. (just 2 weeks) // Soon v2 flag will be removed and will become only mode for dpf handling. let deployment_type_labels = build_deployment_type_labels(carbide_config); @@ -682,30 +688,51 @@ async fn initialize_dpf_sdk( .await .map_err(|err| eyre::eyre!("Failed to initialize DPF SDK: {err}"))?; - let make_init_config = |deployment: &crate::cfg::file::DpfDeploymentConfig, - deployment_type: DpuDeploymentType| { - let services = carbide_config.dpf.resolved_services_for(deployment); - carbide_dpf::InitDpfResourcesConfig { - bfb_url: deployment.bfb_url.clone(), - flavor_name: deployment.flavor_name.clone(), - deployment_name: deployment.deployment_name.clone(), - services: crate::dpf_services::mandatory_services(&services), - proxy: carbide_config.dpf.proxy.clone(), - deployment_type, - } - }; + // Builds the SDK init config for one DPUDeployment. BF4 uses a single + // `BlueFieldSoftware` source (the CR itself carries the PSID→PLDM mapping); + // config validation guarantees exactly one PSID entry. + let make_init_config = + |deployment: &crate::cfg::file::DpfDeploymentConfig, + deployment_type: DpuDeploymentType, + bluefield_software: Option| { + let services = carbide_config.dpf.resolved_services_for(deployment); + carbide_dpf::InitDpfResourcesConfig { + bfb_url: deployment.bfb_url.clone().unwrap_or_default(), + bluefield_software, + flavor_name: deployment.flavor_name.clone(), + deployment_name: deployment.deployment_name.clone(), + services: crate::dpf_services::mandatory_services(&services), + proxy: carbide_config.dpf.proxy.clone(), + deployment_type, + } + }; - sdk.create_initialization_objects(&make_init_config( - &carbide_config.dpf.deployments.bf3, - DpuDeploymentType::Bf3, - )) - .await - .map_err(|err| eyre::eyre!("Failed to initialize bf3 DPF deployment: {err}"))?; + let bf3 = &carbide_config.dpf.deployments.bf3; + sdk.create_initialization_objects(&make_init_config(bf3, DpuDeploymentType::Bf3, None)) + .await + .map_err(|err| eyre::eyre!("Failed to initialize bf3 DPF deployment: {err}"))?; if let Some(bf4) = &carbide_config.dpf.deployments.bf4_generic { - sdk.create_initialization_objects(&make_init_config(bf4, DpuDeploymentType::Bf4Generic)) - .await - .map_err(|err| eyre::eyre!("Failed to initialize bf4_generic DPF deployment: {err}"))?; + // Validation guarantees `bluefield_software` is set with exactly one PSID + // entry for a BF4 deployment. + let bfs = bf4.bluefield_software.as_ref().ok_or_else(|| { + eyre::eyre!("bf4_generic DPF deployment is missing bluefield_software") + })?; + let pldm_url = + bfs.pldm_fw_bundle.values().next().ok_or_else(|| { + eyre::eyre!("bf4_generic DPF deployment has an empty pldm_fw_bundle") + })?; + let params = carbide_dpf::BlueFieldSoftwareParams { + os_iso: bfs.os_iso.clone(), + pldm_fw_bundle: Some(pldm_url.clone()), + }; + sdk.create_initialization_objects(&make_init_config( + bf4, + DpuDeploymentType::Bf4Generic, + Some(params), + )) + .await + .map_err(|err| eyre::eyre!("Failed to initialize bf4_generic DPF deployment: {err}"))?; } Ok(Some(Arc::new(DpfSdkOps::new( diff --git a/crates/api-core/src/tests/dpf/duplicate_events.rs b/crates/api-core/src/tests/dpf/duplicate_events.rs index a47e3bc90a..e5d7fe661c 100644 --- a/crates/api-core/src/tests/dpf/duplicate_events.rs +++ b/crates/api-core/src/tests/dpf/duplicate_events.rs @@ -60,7 +60,7 @@ fn dpf_config() -> crate::cfg::file::DpfConfig { enabled: true, deployments: crate::cfg::file::DpfDeploymentsConfig { bf3: crate::cfg::file::DpfDeploymentConfig { - bfb_url: "http://example.com/test.bfb".to_string(), + bfb_url: Some("http://example.com/test.bfb".to_string()), ..Default::default() }, ..Default::default() diff --git a/crates/api-core/src/tests/dpf/happy_path.rs b/crates/api-core/src/tests/dpf/happy_path.rs index 54883ea3c1..43f38576e2 100644 --- a/crates/api-core/src/tests/dpf/happy_path.rs +++ b/crates/api-core/src/tests/dpf/happy_path.rs @@ -56,7 +56,7 @@ async fn test_dpu_and_host_till_ready(pool: sqlx::PgPool) { enabled: true, deployments: crate::cfg::file::DpfDeploymentsConfig { bf3: crate::cfg::file::DpfDeploymentConfig { - bfb_url: "http://example.com/test.bfb".to_string(), + bfb_url: Some("http://example.com/test.bfb".to_string()), ..Default::default() }, ..Default::default() diff --git a/crates/api-core/src/tests/dpf/reprovisioning.rs b/crates/api-core/src/tests/dpf/reprovisioning.rs index 13fe8f162f..f2e2534830 100644 --- a/crates/api-core/src/tests/dpf/reprovisioning.rs +++ b/crates/api-core/src/tests/dpf/reprovisioning.rs @@ -98,7 +98,7 @@ fn dpf_config() -> crate::cfg::file::DpfConfig { enabled: true, deployments: crate::cfg::file::DpfDeploymentsConfig { bf3: crate::cfg::file::DpfDeploymentConfig { - bfb_url: "http://example.com/test.bfb".to_string(), + bfb_url: Some("http://example.com/test.bfb".to_string()), ..Default::default() }, ..Default::default() diff --git a/crates/api-core/src/tests/dpf/stale_labels.rs b/crates/api-core/src/tests/dpf/stale_labels.rs index bf21641ab7..f4d41f91af 100644 --- a/crates/api-core/src/tests/dpf/stale_labels.rs +++ b/crates/api-core/src/tests/dpf/stale_labels.rs @@ -45,7 +45,7 @@ fn dpf_config() -> crate::cfg::file::DpfConfig { enabled: true, deployments: crate::cfg::file::DpfDeploymentsConfig { bf3: crate::cfg::file::DpfDeploymentConfig { - bfb_url: "http://example.com/test.bfb".to_string(), + bfb_url: Some("http://example.com/test.bfb".to_string()), ..Default::default() }, ..Default::default() diff --git a/crates/api-core/src/tests/dpf/waiting_for_ready.rs b/crates/api-core/src/tests/dpf/waiting_for_ready.rs index e571c1333d..d75e142667 100644 --- a/crates/api-core/src/tests/dpf/waiting_for_ready.rs +++ b/crates/api-core/src/tests/dpf/waiting_for_ready.rs @@ -67,7 +67,7 @@ fn dpf_config() -> crate::cfg::file::DpfConfig { enabled: true, deployments: crate::cfg::file::DpfDeploymentsConfig { bf3: crate::cfg::file::DpfDeploymentConfig { - bfb_url: "http://example.com/test.bfb".to_string(), + bfb_url: Some("http://example.com/test.bfb".to_string()), ..Default::default() }, ..Default::default() diff --git a/crates/api-core/src/tests/machine_admin_force_delete.rs b/crates/api-core/src/tests/machine_admin_force_delete.rs index 995812e529..afba567552 100644 --- a/crates/api-core/src/tests/machine_admin_force_delete.rs +++ b/crates/api-core/src/tests/machine_admin_force_delete.rs @@ -774,7 +774,7 @@ async fn test_admin_force_delete_with_dpf_uses_bmc_mac(pool: sqlx::PgPool) { enabled: true, deployments: crate::cfg::file::DpfDeploymentsConfig { bf3: crate::cfg::file::DpfDeploymentConfig { - bfb_url: "http://example.com/test.bfb".to_string(), + bfb_url: Some("http://example.com/test.bfb".to_string()), ..Default::default() }, ..Default::default() diff --git a/crates/dpf/src/bin/api_harness.rs b/crates/dpf/src/bin/api_harness.rs index 7c825c5d86..45aea6c873 100644 --- a/crates/dpf/src/bin/api_harness.rs +++ b/crates/dpf/src/bin/api_harness.rs @@ -729,7 +729,6 @@ async fn run_provisioning_flow( serial_number: dpu.serial_number.clone(), dpu_machine_id: String::new(), is_primary: true, - deployment_type: DpuDeploymentType::Bf3, }; sdk.register_dpu_device(info).await?; tracing::info!(device_name = %dpu.device_name, serial = %dpu.serial_number, "Registered device"); diff --git a/crates/dpf/src/lib.rs b/crates/dpf/src/lib.rs index 38d435ad32..47c44687a2 100644 --- a/crates/dpf/src/lib.rs +++ b/crates/dpf/src/lib.rs @@ -72,17 +72,17 @@ mod test; pub use error::DpfError; pub use repository::{DpfRepository, KubeRepository}; pub use sdk::{ - DpfSdk, DpfSdkBuilder, NoLabels, ResourceLabeler, build_deployment, + DpfSdk, DpfSdkBuilder, DpuProvisioningSource, NoLabels, ResourceLabeler, build_deployment, build_service_configuration, build_service_interface, build_service_nad, build_service_template, dpu_cr_name, dpu_device_cr_name, dpu_node_cr_name, node_id_from_dpu_node_cr_name, }; pub use services::{DEFAULT_DOCA_HELM_REGISTRY, ServiceRegistryConfig}; pub use types::{ - BmcPasswordProvider, ConfigPortsServiceType, DpuDeploymentType, DpuDeviceInfo, DpuErrorEvent, - DpuEvent, DpuMismatch, DpuNodeInfo, DpuPhase, DpuReadyEvent, InitDpfResourcesConfig, - MaintenanceEvent, RebootRequiredEvent, ServiceChainSwitch, ServiceConfigPort, - ServiceConfigPortProtocol, ServiceDefinition, ServiceInterface, ServiceNAD, + BlueFieldSoftwareParams, BmcPasswordProvider, ConfigPortsServiceType, DpuDeploymentType, + DpuDeviceInfo, DpuErrorEvent, DpuEvent, DpuMismatch, DpuNodeInfo, DpuPhase, DpuReadyEvent, + InitDpfResourcesConfig, MaintenanceEvent, RebootRequiredEvent, ServiceChainSwitch, + ServiceConfigPort, ServiceConfigPortProtocol, ServiceDefinition, ServiceInterface, ServiceNAD, ServiceNADResourceType, }; pub use watcher::{DpuWatcher, DpuWatcherBuilder}; diff --git a/crates/dpf/src/repository/kube.rs b/crates/dpf/src/repository/kube.rs index 4e50c1956a..029eb738ad 100644 --- a/crates/dpf/src/repository/kube.rs +++ b/crates/dpf/src/repository/kube.rs @@ -33,6 +33,7 @@ use tokio_util::sync::CancellationToken; use super::traits::*; use crate::crds::bfbs_generated::BFB; +use crate::crds::bluefieldsoftwares_generated::BlueFieldSoftware; use crate::crds::dpuclusters_generated::DPUCluster; use crate::crds::dpudeployments_generated::DPUDeployment; use crate::crds::dpudevices_generated::DPUDevice; @@ -114,6 +115,36 @@ impl BfbRepository for KubeRepository { } } +#[async_trait] +impl BlueFieldSoftwareRepository for KubeRepository { + async fn get( + &self, + name: &str, + namespace: &str, + ) -> Result, DpfError> { + let api = self.api(namespace); + Ok(api.get_opt(name).await?) + } + + async fn list(&self, namespace: &str) -> Result, DpfError> { + let api = self.api(namespace); + let list = api.list(&ListParams::default()).await?; + Ok(list.items) + } + + async fn create(&self, bfs: &BlueFieldSoftware) -> Result { + let namespace = bfs.meta().namespace.as_deref().unwrap_or("default"); + let api = self.api(namespace); + Ok(api.create(&PostParams::default(), bfs).await?) + } + + async fn delete(&self, name: &str, namespace: &str) -> Result<(), DpfError> { + let api: Api = self.api(namespace); + api.delete(name, &Default::default()).await?; + Ok(()) + } +} + #[async_trait] impl DpuRepository for KubeRepository { async fn get(&self, name: &str, namespace: &str) -> Result, DpfError> { diff --git a/crates/dpf/src/repository/traits.rs b/crates/dpf/src/repository/traits.rs index 5faba9f6ba..12ce0424ad 100644 --- a/crates/dpf/src/repository/traits.rs +++ b/crates/dpf/src/repository/traits.rs @@ -24,6 +24,7 @@ use std::sync::Arc; use async_trait::async_trait; use crate::crds::bfbs_generated::BFB; +use crate::crds::bluefieldsoftwares_generated::BlueFieldSoftware; use crate::crds::dpuclusters_generated::DPUCluster; use crate::crds::dpudeployments_generated::DPUDeployment; use crate::crds::dpudevices_generated::DPUDevice; @@ -49,6 +50,20 @@ pub trait BfbRepository: Send + Sync { async fn delete(&self, name: &str, namespace: &str) -> Result<(), DpfError>; } +/// Repository for BlueFieldSoftware resources. +/// +/// BF4-class DPUs are provisioned from a `BlueFieldSoftware` CR (OS ISO + +/// firmware bundle) rather than a BFB, so this repository mirrors +/// [`BfbRepository`] for that resource. +#[async_trait] +pub trait BlueFieldSoftwareRepository: Send + Sync { + async fn get(&self, name: &str, namespace: &str) + -> Result, DpfError>; + async fn list(&self, namespace: &str) -> Result, DpfError>; + async fn create(&self, bfs: &BlueFieldSoftware) -> Result; + async fn delete(&self, name: &str, namespace: &str) -> Result<(), DpfError>; +} + /// Repository for DPU resources. #[async_trait] pub trait DpuRepository: Send + Sync { @@ -266,6 +281,7 @@ pub trait DpfOperatorConfigRepository: Send + Sync { /// enabling the SDK to work with any backend (real K8s, mock, etc.). pub trait DpfRepository: BfbRepository + + BlueFieldSoftwareRepository + DpuRepository + DpuDeviceRepository + DpuNodeRepository diff --git a/crates/dpf/src/sdk.rs b/crates/dpf/src/sdk.rs index bc8ca47dee..80580102af 100644 --- a/crates/dpf/src/sdk.rs +++ b/crates/dpf/src/sdk.rs @@ -26,6 +26,7 @@ use serde_json::json; use sha2::{Digest, Sha256}; use crate::crds::bfbs_generated::{BFB, BfbSpec}; +use crate::crds::bluefieldsoftwares_generated::{BlueFieldSoftware, BlueFieldSoftwareSpec}; use crate::crds::dpudeployments_generated::{ DPUDeployment, DpuDeploymentDpus, DpuDeploymentDpusDpuSetStrategy, DpuDeploymentDpusDpuSetStrategyType, DpuDeploymentDpusDpuSets, @@ -69,16 +70,17 @@ use crate::crds::dpuservicetemplates_generated::{ }; use crate::error::DpfError; use crate::repository::{ - BfbRepository, DpfOperatorConfigRepository, DpuDeploymentRepository, DpuDeviceRepository, - DpuFlavorRepository, DpuNodeMaintenanceRepository, DpuNodeRepository, DpuRepository, + BfbRepository, BlueFieldSoftwareRepository, DpfOperatorConfigRepository, + DpuDeploymentRepository, DpuDeviceRepository, DpuFlavorRepository, + DpuNodeMaintenanceRepository, DpuNodeRepository, DpuRepository, DpuServiceConfigurationRepository, DpuServiceNADRepository, DpuServiceTemplateRepository, K8sConfigRepository, }; use crate::types::{ - BmcPasswordProvider, ConfigPortsServiceType, DHCP_SERVER_SERVICE_NAME, DOCA_HBN_SERVICE_NAME, - DPU_AGENT_SERVICE_NAME, DTS_SERVICE_NAME, DpfProxyDetails, DpuDeploymentType, DpuDeviceInfo, - DpuDeviceSummary, DpuMismatch, DpuNodeInfo, DpuNodeSummary, DpuPhase, - DpuServiceInterfaceTemplateDefinition, DpuServiceInterfaceTemplateType, DpuSummary, + BlueFieldSoftwareParams, BmcPasswordProvider, ConfigPortsServiceType, DHCP_SERVER_SERVICE_NAME, + DOCA_HBN_SERVICE_NAME, DPU_AGENT_SERVICE_NAME, DTS_SERVICE_NAME, DpfProxyDetails, + DpuDeploymentType, DpuDeviceInfo, DpuDeviceSummary, DpuMismatch, DpuNodeInfo, DpuNodeSummary, + DpuPhase, DpuServiceInterfaceTemplateDefinition, DpuServiceInterfaceTemplateType, DpuSummary, FMDS_SERVICE_NAME, HostDpfSnapshot, InitDpfResourcesConfig, OTEL_COLLECTOR_SERVICE_NAME, ServiceConfigPortProtocol, ServiceDefinition, ServiceNADResourceType, ServiceTemplateVersion, }; @@ -86,6 +88,7 @@ use crate::watcher::DpuWatcherBuilder; const SECRET_NAME: &str = "bmc-shared-password"; const BFB_NAME_PREFIX: &str = "bf-bundle"; +const BLUEFIELD_SOFTWARE_NAME_PREFIX: &str = "bf-software"; /// Label set by the DPF operator on each DPU CR pointing back to its owning /// DPUDeployment. Value format: `_`. const DPU_OWNED_BY_DEPLOYMENT_LABEL: &str = "svc.dpu.nvidia.com/owned-by-dpudeployment"; @@ -269,6 +272,7 @@ where impl DpfSdkBuilder<'_, R, P, L> where R: BfbRepository + + BlueFieldSoftwareRepository + DpuFlavorRepository + DpuDeploymentRepository + DpuServiceTemplateRepository @@ -458,6 +462,75 @@ async fn create_bfb( } } +/// Reference to the resource a DPUDeployment provisions DPUs from. Exactly one +/// variant is populated per deployment, matching the DPUDeployment CRD rule that +/// exactly one of `spec.dpus.bfb` / `spec.dpus.blueFieldSoftware` be set. +pub enum DpuProvisioningSource { + /// Name of a `BFB` CR (BF3-class DPUs). + Bfb(String), + /// Name of a `BlueFieldSoftware` CR (BF4-class DPUs). + BlueFieldSoftware(String), +} + +/// Creates a `BlueFieldSoftware` CR with a hash-derived name +/// (`{prefix}-{sha256(os_iso[+pldm_fw_bundle])}`). Like [`create_bfb`], any +/// change to the spec produces a new name so DPUs are detected as outdated. +/// Idempotent: an already-existing CR with the same name is reused. +async fn create_bluefield_software( + repo: &R, + namespace: &str, + params: &BlueFieldSoftwareParams, +) -> Result { + let mut hasher = Sha256::new(); + hasher.update(params.os_iso.as_bytes()); + if let Some(pldm) = params.pldm_fw_bundle.as_deref() { + hasher.update(b"\0"); + hasher.update(pldm.as_bytes()); + } + let name = format!( + "{}-{}", + BLUEFIELD_SOFTWARE_NAME_PREFIX, + hex::encode(hasher.finalize()) + ); + + let bfs = BlueFieldSoftware { + metadata: ObjectMeta { + name: Some(name.clone()), + namespace: Some(namespace.to_string()), + ..Default::default() + }, + spec: BlueFieldSoftwareSpec { + os_iso: params.os_iso.clone(), + pldm_fw_bundle: params.pldm_fw_bundle.clone(), + nic_fw: None, + platform_pldm_fw_bundle: None, + }, + status: None, + }; + match BlueFieldSoftwareRepository::create(repo, &bfs).await { + Ok(_) => Ok(name), + Err(DpfError::KubeError(kube::Error::Api(ref err))) + if err.is_already_exists() || err.is_conflict() => + { + // Reuse the existing CR only if it is not being torn down; otherwise + // the DPUDeployment would reference a source that is disappearing. + let existing = BlueFieldSoftwareRepository::get(repo, &name, namespace).await?; + if existing + .as_ref() + .is_some_and(|b| b.metadata.deletion_timestamp.is_some()) + { + return Err(DpfError::InvalidState(format!( + "BlueFieldSoftware {name} is being deleted (has deletionTimestamp); \ + cannot reuse until the old resource is fully removed" + ))); + } + tracing::debug!(bluefield_software = %name, "BlueFieldSoftware already exists, reusing"); + Ok(name) + } + Err(e) => Err(e), + } +} + /// Creates a DPUFlavor with a hash-derived name (`{default_flavor_name}-{spec_hash}`). /// Any change in the spec produces a different hash and therefore a new flavor name, which /// causes MachineUpdateManager to detect the DPUs as outdated and trigger reprovisioning. @@ -662,7 +735,7 @@ pub fn build_service_nad(svc: &ServiceDefinition, namespace: &str) -> Option Some(name.clone()), + DpuProvisioningSource::BlueFieldSoftware(_) => None, + }, dpu_sets: Some(vec![DpuDeploymentDpusDpuSets { dpu_annotations: None, dpu_selector: None, @@ -799,7 +875,10 @@ pub fn build_deployment( }, secure_boot: None, astra_enabled: None, - blue_field_software: None, + blue_field_software: match source { + DpuProvisioningSource::Bfb(_) => None, + DpuProvisioningSource::BlueFieldSoftware(name) => Some(name.clone()), + }, flavor_template: None, }, revision_history_limit: None, @@ -1084,7 +1163,7 @@ async fn create_flavor_services_and_deployment< labeler: &L, services: &[ServiceDefinition], deployment_name: &str, - bfb_name: &str, + source: &DpuProvisioningSource, default_flavor_name: &str, proxy: &Option, deployment_type: DpuDeploymentType, @@ -1112,7 +1191,7 @@ async fn create_flavor_services_and_deployment< let deployment = build_deployment( services, deployment_name, - bfb_name, + source, &flavor_name, namespace, &interfaces, @@ -1124,6 +1203,7 @@ async fn create_flavor_services_and_deployment< impl< R: BfbRepository + + BlueFieldSoftwareRepository + DpuFlavorRepository + DpuDeploymentRepository + DpuServiceTemplateRepository @@ -1137,16 +1217,24 @@ impl< { /// Create all initialization CRDs for the "Provision a DPU" flow. /// - /// Order: BFB (BFB controller downloads), DPUFlavor, DPUDeployment with - /// `dpu_sets` referencing BFB and DPUFlavor. The operator then creates - /// DPU objects and drives provisioning. + /// Order: provisioning source (BFB for BF3, or BlueFieldSoftware for BF4 — + /// the controller downloads either), DPUFlavor, DPUDeployment with + /// `dpu_sets` referencing the source and DPUFlavor. The operator then + /// creates DPU objects and drives provisioning. /// /// See: https://docs.nvidia.com/networking/display/dpf2507/component+description#ProvisionaDPU pub async fn create_initialization_objects( &self, config: &InitDpfResourcesConfig, ) -> Result<(), DpfError> { - let bfb_name = create_bfb(&*self.repo, &self.namespace, &config.bfb_url).await?; + let source = match &config.bluefield_software { + Some(params) => DpuProvisioningSource::BlueFieldSoftware( + create_bluefield_software(&*self.repo, &self.namespace, params).await?, + ), + None => DpuProvisioningSource::Bfb( + create_bfb(&*self.repo, &self.namespace, &config.bfb_url).await?, + ), + }; let services = if config.services.is_empty() { crate::services::default_services(&crate::services::ServiceRegistryConfig::default()) } else { @@ -1158,7 +1246,7 @@ impl< &self.labeler, &services, &config.deployment_name, - &bfb_name, + &source, &config.flavor_name, &config.proxy, config.deployment_type, @@ -1521,9 +1609,27 @@ impl DpfSdk { return None; }; - // TODO: Compare either bfb or bluefield_software - let expected_bfb_cr_name = deployment.spec.dpus.bfb.clone().unwrap_or_default(); let expected_flavor = deployment.spec.dpus.flavor.clone().unwrap_or_default(); + let flavor_matches = dpu.spec.dpu_flavor == expected_flavor; + + // BF4-class deployments provision from a BlueFieldSoftware CR + // (`spec.dpus.bfb` is unset) rather than a BFB. We have no + // BFB filename to compare against in that case, so only the + // flavor is checked to avoid falsely flagging every BF4 DPU as + // outdated. + // TODO: compare the installed BlueFieldSoftware version once the + // DPU status exposes it, so BF4 software changes trigger reprovisioning. + let Some(expected_bfb_cr_name) = deployment.spec.dpus.bfb.clone() else { + if flavor_matches { + return None; + } + return Some(DpuMismatch { + dpu_cr_name: cr_name, + dpu_labels: dpu.metadata.labels.clone().unwrap_or_default(), + target_bfb: String::new(), + }); + }; + let expected_filename = format!("{}-{}.bfb", self.namespace, expected_bfb_cr_name); let current_basename = dpu @@ -1532,7 +1638,6 @@ impl DpfSdk { .and_then(|s| s.bfb_file.as_deref()) .map(bfb_file_basename); let bfb_matches = current_basename == Some(expected_filename.as_str()); - let flavor_matches = dpu.spec.dpu_flavor == expected_flavor; if bfb_matches && flavor_matches { return None; } @@ -1890,7 +1995,7 @@ mod tests { let deployment = build_deployment( &services, "dep", - "bfb", + &DpuProvisioningSource::Bfb("bfb".to_string()), "flavor", TEST_NAMESPACE, &[], @@ -2181,7 +2286,6 @@ mod tests { serial_number: "SN123456".to_string(), dpu_machine_id: "dpu-bbb".to_string(), is_primary: true, - deployment_type: DpuDeploymentType::Bf3, }; sdk.register_dpu_device(info).await.unwrap(); @@ -2233,7 +2337,6 @@ mod tests { serial_number: "SN123456".to_string(), dpu_machine_id: "dpu-bbb".to_string(), is_primary: true, - deployment_type: DpuDeploymentType::Bf3, }; sdk.register_dpu_device(info).await.unwrap(); @@ -2327,7 +2430,6 @@ mod tests { serial_number: "SN123456".to_string(), dpu_machine_id: "dpu-bbb".to_string(), is_primary: true, - deployment_type: DpuDeploymentType::Bf3, }; sdk.register_dpu_device(info).await.unwrap(); @@ -2364,7 +2466,6 @@ mod tests { serial_number: "SN123456".to_string(), dpu_machine_id: "dpu-bbb".to_string(), is_primary: true, - deployment_type: DpuDeploymentType::Bf3, }; sdk.register_dpu_device(info).await.unwrap(); @@ -2484,7 +2585,6 @@ mod tests { serial_number: "SN123".to_string(), dpu_machine_id: "dpu-bbb".to_string(), is_primary: true, - deployment_type: DpuDeploymentType::Bf3, }; sdk.register_dpu_device(device_info).await.unwrap(); @@ -2588,7 +2688,6 @@ mod tests { serial_number: "SN111".to_string(), dpu_machine_id: "dpu-111".to_string(), is_primary: true, - deployment_type: DpuDeploymentType::Bf3, }; let info2 = DpuDeviceInfo { @@ -2598,7 +2697,6 @@ mod tests { serial_number: "SN222".to_string(), dpu_machine_id: "dpu-222".to_string(), is_primary: false, - deployment_type: DpuDeploymentType::Bf3, }; sdk1.register_dpu_device(info1).await.unwrap(); @@ -2762,7 +2860,6 @@ mod tests { serial_number: "SN123456".to_string(), dpu_machine_id: "dpu-bbb".to_string(), is_primary: true, - deployment_type: DpuDeploymentType::Bf3, }; let err = sdk.register_dpu_device(info).await.unwrap_err(); assert!( @@ -2812,7 +2909,6 @@ mod tests { serial_number: "SN123456".to_string(), dpu_machine_id: "dpu-bbb".to_string(), is_primary: true, - deployment_type: DpuDeploymentType::Bf3, }; sdk.register_dpu_device(info).await.unwrap(); } diff --git a/crates/dpf/src/test/sdk_device_registration.rs b/crates/dpf/src/test/sdk_device_registration.rs index 17723f249d..788f4b37ba 100644 --- a/crates/dpf/src/test/sdk_device_registration.rs +++ b/crates/dpf/src/test/sdk_device_registration.rs @@ -200,7 +200,6 @@ async fn test_register_devices_node_and_force_delete() { serial_number: format!("SN-{}", i), dpu_machine_id: format!("dpu-{}-id", i), is_primary: true, - deployment_type: DpuDeploymentType::Bf3, }; sdk.register_dpu_device(info).await.unwrap(); } diff --git a/crates/dpf/src/test/sdk_initialization.rs b/crates/dpf/src/test/sdk_initialization.rs index ed0db349c6..c03aec15a0 100644 --- a/crates/dpf/src/test/sdk_initialization.rs +++ b/crates/dpf/src/test/sdk_initialization.rs @@ -25,6 +25,7 @@ use dashmap::DashMap; use kube::Resource; use crate::crds::bfbs_generated::BFB; +use crate::crds::bluefieldsoftwares_generated::BlueFieldSoftware; use crate::crds::dpudeployments_generated::DPUDeployment; use crate::crds::dpuflavors_generated::DPUFlavor; use crate::crds::dpuserviceconfigurations_generated::DPUServiceConfiguration; @@ -33,9 +34,10 @@ use crate::crds::dpuservicenads_generated::DPUServiceNAD; use crate::crds::dpuservicetemplates_generated::DPUServiceTemplate; use crate::error::DpfError; use crate::repository::{ - BfbRepository, DpfOperatorConfigRepository, DpuDeploymentRepository, DpuFlavorRepository, - DpuServiceConfigurationRepository, DpuServiceInterfaceRepository, DpuServiceNADRepository, - DpuServiceTemplateRepository, K8sConfigRepository, + BfbRepository, BlueFieldSoftwareRepository, DpfOperatorConfigRepository, + DpuDeploymentRepository, DpuFlavorRepository, DpuServiceConfigurationRepository, + DpuServiceInterfaceRepository, DpuServiceNADRepository, DpuServiceTemplateRepository, + K8sConfigRepository, }; use crate::types::*; @@ -56,6 +58,7 @@ fn resource_key(r: &T) -> String { #[derive(Clone, Default)] struct InitializationMock { bfbs: Arc>, + bluefield_softwares: Arc>, flavors: Arc>, deployments: Arc>, service_templates: Arc>, @@ -100,6 +103,34 @@ impl BfbRepository for InitializationMock { } } +#[async_trait] +impl BlueFieldSoftwareRepository for InitializationMock { + async fn get(&self, name: &str, ns: &str) -> Result, DpfError> { + Ok(self + .bluefield_softwares + .get(&ns_key(ns, name)) + .map(|r| r.clone())) + } + async fn list(&self, ns: &str) -> Result, DpfError> { + let prefix = format!("{}/", ns); + Ok(self + .bluefield_softwares + .iter() + .filter(|entry| entry.key().starts_with(&prefix)) + .map(|entry| entry.value().clone()) + .collect()) + } + async fn create(&self, bfs: &BlueFieldSoftware) -> Result { + self.bluefield_softwares + .insert(resource_key(bfs), bfs.clone()); + Ok(bfs.clone()) + } + async fn delete(&self, name: &str, ns: &str) -> Result<(), DpfError> { + self.bluefield_softwares.remove(&ns_key(ns, name)); + Ok(()) + } +} + #[async_trait] impl DpuFlavorRepository for InitializationMock { async fn get(&self, name: &str, ns: &str) -> Result, DpfError> { @@ -318,3 +349,52 @@ async fn test_create_initialization_objects() { drop(sdk); } + +#[tokio::test] +async fn test_create_initialization_objects_bluefield_software() { + let mock = InitializationMock::default(); + + let config = InitDpfResourcesConfig { + bluefield_software: Some(BlueFieldSoftwareParams { + os_iso: "http://example.com/os.iso".to_string(), + pldm_fw_bundle: Some("http://example.com/fw.pldm".to_string()), + }), + deployment_name: "bf4-dep".to_string(), + deployment_type: DpuDeploymentType::Bf4Generic, + ..Default::default() + }; + + let sdk = crate::sdk::DpfSdkBuilder::new(mock.clone(), TEST_NS, "test-password".to_string()) + .initialize(&config) + .await + .unwrap(); + + // A BlueFieldSoftware CR is created; no BFB is. + let bfbs = BfbRepository::list(&mock, TEST_NS).await.unwrap(); + assert!( + bfbs.is_empty(), + "no BFB should be created for a BF4 deployment" + ); + let bfsw = BlueFieldSoftwareRepository::list(&mock, TEST_NS) + .await + .unwrap(); + assert_eq!(bfsw.len(), 1); + assert_eq!(bfsw[0].spec.os_iso, "http://example.com/os.iso"); + assert_eq!( + bfsw[0].spec.pldm_fw_bundle.as_deref(), + Some("http://example.com/fw.pldm") + ); + + // The DPUDeployment references the BlueFieldSoftware CR, not a BFB. + let deployment = DpuDeploymentRepository::get(&mock, "bf4-dep", TEST_NS) + .await + .unwrap() + .expect("bf4 deployment created"); + assert_eq!( + deployment.spec.dpus.blue_field_software.as_deref(), + Some(bfsw[0].metadata.name.as_deref().unwrap()) + ); + assert!(deployment.spec.dpus.bfb.is_none()); + + drop(sdk); +} diff --git a/crates/dpf/src/types.rs b/crates/dpf/src/types.rs index f7044533b6..feaa3a9ca9 100644 --- a/crates/dpf/src/types.rs +++ b/crates/dpf/src/types.rs @@ -52,8 +52,14 @@ pub const DTS_SERVICE_NAME: &str = "dts"; /// DPUDeployment, service templates, etc.) during initialization. #[derive(Debug, Clone)] pub struct InitDpfResourcesConfig { - /// URL for the BFB (BlueField Bundle) image. + /// URL for the BFB (BlueField Bundle) image. Used for BF3-class DPUs. + /// Ignored when [`bluefield_software`](Self::bluefield_software) is set. pub bfb_url: String, + /// BlueFieldSoftware spec for BF4-class DPUs. When set, a `BlueFieldSoftware` + /// CR is created and referenced by the DPUDeployment instead of a BFB, and + /// [`bfb_url`](Self::bfb_url) is ignored. Exactly one provisioning source + /// (BFB or BlueFieldSoftware) is expected per deployment. + pub bluefield_software: Option, /// Name of the DPUDeployment CR. pub deployment_name: String, /// Name of the DPUFlavor CR. @@ -67,10 +73,23 @@ pub struct InitDpfResourcesConfig { pub deployment_type: DpuDeploymentType, } +/// Parameters for a `BlueFieldSoftware` CR, used to provision BF4-class DPUs. +/// Mirrors the `spec` of the `provisioning.dpu.nvidia.com/v1alpha1` +/// `BlueFieldSoftware` resource. +#[derive(Debug, Clone)] +pub struct BlueFieldSoftwareParams { + /// OS ISO URL used by the DPU OS installation flow (`spec.osIso`). + pub os_iso: String, + /// Optional PLDM firmware bundle URL for baseline firmware updates + /// (`spec.pldmFwBundle`). + pub pldm_fw_bundle: Option, +} + impl Default for InitDpfResourcesConfig { fn default() -> Self { Self { bfb_url: String::new(), + bluefield_software: None, deployment_name: "dpu-deployment".to_string(), flavor_name: crate::flavor::DEFAULT_FLAVOR_NAME.to_string(), services: Vec::new(), @@ -244,6 +263,9 @@ pub struct DpuFlavorBridgeDefinition { /// Deployment type of a DPU — used to route devices to the correct /// DPUDeployment and select the appropriate DPUFlavor configuration. +/// +/// BF4-class DPUs are provisioned from a single `BlueFieldSoftware` CR (the CR +/// itself carries the PSID→PLDM mapping), so there is one BF4 deployment. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum DpuDeploymentType { Bf3, @@ -267,8 +289,6 @@ pub struct DpuDeviceInfo { pub dpu_machine_id: String, /// is _primary dpu? pub is_primary: bool, - /// Deployment type for this DPU — used to look up deployment-specific labels. - pub deployment_type: DpuDeploymentType, } /// Information about a DPU node (host with DPUs). diff --git a/crates/machine-controller/src/dpf.rs b/crates/machine-controller/src/dpf.rs index 93efd42b18..cac8cc972f 100644 --- a/crates/machine-controller/src/dpf.rs +++ b/crates/machine-controller/src/dpf.rs @@ -87,8 +87,9 @@ pub trait DpfOperations: Send + Sync + std::fmt::Debug { async fn reboot_complete(&self, node_name: &str) -> Result<(), DpfError>; /// Resolve the deployment type of a DPU based on its hardware (BF3 vs BF4). - /// Returns `Err` when the part number is absent or does not match any known generation, - /// so unrecognized hardware never silently routes to a wrong deployment. + /// Returns `Err` when the part number is absent or does not match any known + /// generation, so unrecognized hardware never silently routes to a wrong + /// deployment. fn deployment_type_for_dpu(&self, dpu: &Machine) -> Result; /// Check that a DPUNode's labels match the current expected labels. diff --git a/crates/machine-controller/src/handler/dpf.rs b/crates/machine-controller/src/handler/dpf.rs index 940e4500d8..a26e8f7cde 100644 --- a/crates/machine-controller/src/handler/dpf.rs +++ b/crates/machine-controller/src/handler/dpf.rs @@ -202,7 +202,6 @@ async fn create_and_register_dpudevices_and_dpunode( serial_number: serial_number.to_string(), dpu_machine_id: dpu.id.to_string(), is_primary: dpu.id == primary_dpu_id, - deployment_type: dpf_sdk.deployment_type_for_dpu(dpu).map_err(dpf_error)?, }; dpf_sdk .register_dpu_device(device_info)