From 6af30608461d065a10170656583b39a27ca2ecaa Mon Sep 17 00:00:00 2001 From: Chet Nichols III Date: Thu, 9 Jul 2026 17:08:35 -0700 Subject: [PATCH] fix: separate BlueField SKU from operating mode in mlx-devices `site-explorer mlx-devices` now reports what its part-number classification actually is -- the card's factory SKU -- and leaves the operating mode to `nic_mode`, the value read from the DPU's own BMC. The old labels presented the SKU as the mode ("BlueField-3 (DPU mode)" / "(NIC mode)"), built on a false premise: part numbers never change when a card flips between DPU and NIC mode. QA2 has the live counterexample -- a `900-9D3B6` DPU flipped into NIC mode (OEM `Mode = NicMode`, Arm complex `Paused`, host booted through its PF) that the CLI printed as `kind: "BlueField-3 (DPU mode)"` next to `nic_mode: "NIC"`, which read as NICo contradicting itself and got filed as a mode-mismatch bug. `--nic-mode-only` now follows the authoritative signal too: a flipped `900-9D3B6` running as a NIC (exactly the card whose firmware scout can't see) is included, a `900-9D3B4` running as a DPU is not, and the SuperNIC SKUs still match when the mode is unknown. The new `kind` labels are just NVIDIA's product names ("BlueField-3 DPU" / "BlueField-3 SuperNIC"), with the part-number column as the discriminator. The `MlxDeviceKind` doc comments (api-model + proto, with the regenerated REST snapshots) explain the SKU-vs-mode split, and the dead `MlxDeviceKind::is_nic_mode()` is dropped. The `nic_mode` NICo reads is the OEM extension (`Systems/Bluefield/Oem/Nvidia` `Mode`) -- the surface the BlueField BMC manual documents for querying the mode, and the same OEM-for-BF3 / BIOS-for-BF2 precedence `bmc-explorer` already codifies. Tests added! This supports https://github.com/NVIDIA/infra-controller/issues/3345 Signed-off-by: Chet Nichols III --- .../src/site_explorer/mlx_devices/args.rs | 4 +- .../src/site_explorer/mlx_devices/cmd.rs | 92 ++++++++++++++++++- crates/api-model/src/site_explorer/mod.rs | 43 +++++---- crates/rpc/proto/site_explorer.proto | 13 ++- .../core/gen/v1/site_explorer_nico.pb.go | 13 ++- .../core/src/v1/site_explorer_nico.proto | 13 ++- 6 files changed, 139 insertions(+), 39 deletions(-) diff --git a/crates/admin-cli/src/site_explorer/mlx_devices/args.rs b/crates/admin-cli/src/site_explorer/mlx_devices/args.rs index 8e13d3dfa9..9273feaa5a 100644 --- a/crates/admin-cli/src/site_explorer/mlx_devices/args.rs +++ b/crates/admin-cli/src/site_explorer/mlx_devices/args.rs @@ -27,7 +27,7 @@ List Mellanox/BlueField devices across all explored hosts: List the devices found under one host BMC: $ nico-admin-cli site-explorer mlx-devices --host 192.0.2.20 -Find DPUs in NIC mode whose NIC firmware is below the desired version: +Find devices operating as NICs whose firmware is below the desired version: $ nico-admin-cli site-explorer mlx-devices --nic-mode-only --expected-version 32.42.1000 ")] @@ -36,7 +36,7 @@ pub struct Args { pub host: Option, #[clap( long, - help = "Only BlueField-3 devices operating in NIC mode (part number 900-9D3B4)" + help = "Only devices operating as NICs: their DPU BMC reports NIC mode, or they have a SuperNIC SKU and the mode is unknown" )] pub nic_mode_only: bool, #[clap( diff --git a/crates/admin-cli/src/site_explorer/mlx_devices/cmd.rs b/crates/admin-cli/src/site_explorer/mlx_devices/cmd.rs index 8605207f42..2b15542245 100644 --- a/crates/admin-cli/src/site_explorer/mlx_devices/cmd.rs +++ b/crates/admin-cli/src/site_explorer/mlx_devices/cmd.rs @@ -39,8 +39,8 @@ pub async fn mlx_devices( .get_all_explored_mlx_devices(page_size, opts.host) .await? .into_iter() - // Only NIC-mode DPUs, when asked. - .filter(|d| !nic_mode_only || d.device_kind == MlxDeviceKind::Bf3NicMode as i32) + // Only devices operating as NICs, when asked. + .filter(|d| !nic_mode_only || operating_as_nic(d)) // Only devices whose NIC firmware is below the desired version, when given // one. A device with no firmware, or a version we can't parse, is surfaced // rather than hidden. @@ -100,11 +100,31 @@ impl From for MlxDeviceRow { } } +/// Whether a device is operating as a plain NIC -- its Arm OS is down, so scout +/// can't report its firmware and this inventory is the only one that sees it. +/// +/// The authoritative signal is `nic_mode`, the mode the DPU's own BMC reports; +/// a `900-9D3B6` DPU flipped into NIC mode passes on that signal even though +/// its SKU says DPU. When the mode is unknown (`nic_mode` unset -- no DPU BMC +/// matched, or its firmware predates mode reporting), fall back to the SKU: +/// the SuperNIC families ship running as NICs, and like the firmware filter +/// below we err toward surfacing a device rather than hiding it. +fn operating_as_nic(device: &ExploredMlxDevice) -> bool { + match device.nic_mode { + Some(mode) => mode == NicMode::Nic as i32, + None => { + device.device_kind == MlxDeviceKind::Bf3NicMode as i32 + || device.device_kind == MlxDeviceKind::Bf3SuperNic as i32 + } + } +} + fn kind_label(device_kind: i32) -> String { match MlxDeviceKind::try_from(device_kind) { - Ok(MlxDeviceKind::Bf3NicMode) => "BlueField-3 (NIC mode)", - Ok(MlxDeviceKind::Bf3DpuMode) => "BlueField-3 (DPU mode)", - Ok(MlxDeviceKind::Bf3SuperNic) => "BlueField-3 SuperNIC", + // Both SuperNIC SKU families render under NVIDIA's product name; the + // part-number column alongside is the discriminator. + Ok(MlxDeviceKind::Bf3NicMode) | Ok(MlxDeviceKind::Bf3SuperNic) => "BlueField-3 SuperNIC", + Ok(MlxDeviceKind::Bf3DpuMode) => "BlueField-3 DPU", Ok(MlxDeviceKind::Bf2Dpu) => "BlueField-2 DPU", Ok(MlxDeviceKind::Unknown) | Err(_) => "Unknown", } @@ -172,6 +192,68 @@ fn parse_version(version: &str) -> Option> { mod tests { use super::*; + #[test] + fn operating_as_nic_prefers_reported_mode_over_sku() { + struct Case { + name: &'static str, + device_kind: MlxDeviceKind, + nic_mode: Option, + expect: bool, + } + let cases = [ + Case { + name: "dpu sku flipped into nic mode", + device_kind: MlxDeviceKind::Bf3DpuMode, + nic_mode: Some(NicMode::Nic), + expect: true, + }, + Case { + name: "dpu sku running as a dpu", + device_kind: MlxDeviceKind::Bf3DpuMode, + nic_mode: Some(NicMode::Dpu), + expect: false, + }, + Case { + name: "supernic sku flipped into dpu mode", + device_kind: MlxDeviceKind::Bf3NicMode, + nic_mode: Some(NicMode::Dpu), + expect: false, + }, + Case { + name: "unmatched 9d3b4 supernic falls back to sku", + device_kind: MlxDeviceKind::Bf3NicMode, + nic_mode: None, + expect: true, + }, + Case { + name: "unmatched 9d3d4 supernic falls back to sku", + device_kind: MlxDeviceKind::Bf3SuperNic, + nic_mode: None, + expect: true, + }, + Case { + name: "unmatched dpu sku stays a dpu", + device_kind: MlxDeviceKind::Bf3DpuMode, + nic_mode: None, + expect: false, + }, + Case { + name: "unmatched bf2 stays a dpu", + device_kind: MlxDeviceKind::Bf2Dpu, + nic_mode: None, + expect: false, + }, + ]; + for case in cases { + let device = ExploredMlxDevice { + device_kind: case.device_kind as i32, + nic_mode: case.nic_mode.map(|mode| mode as i32), + ..Default::default() + }; + assert_eq!(operating_as_nic(&device), case.expect, "{}", case.name); + } + } + #[test] fn firmware_below_compares_dotted_versions() { // Outdated -> below the target. diff --git a/crates/api-model/src/site_explorer/mod.rs b/crates/api-model/src/site_explorer/mod.rs index 2212df9143..0fb6900354 100644 --- a/crates/api-model/src/site_explorer/mod.rs +++ b/crates/api-model/src/site_explorer/mod.rs @@ -1815,16 +1815,22 @@ pub fn is_bluefield_part_number(part_number: &str) -> bool { /// The kind of BlueField/Mellanox device, classified from its Redfish part number. /// -/// A BlueField-3's part number records the mode it is currently operating in: -/// `900-9D3B4` is a card running as a NIC, `900-9D3B6` is the same generation -/// running as a DPU, and `900-9D3D4` is a dedicated SuperNIC. That split is what -/// lets us pick out a DPU operating in NIC mode -- whose NIC firmware is -/// otherwise invisible while its Arm OS is down -- apart from a native SuperNIC. +/// The part number identifies the card's factory SKU, not the mode it is +/// operating in: `900-9D3B6` is a BlueField-3 DPU product, while `900-9D3B4` +/// and `900-9D3D4` are BlueField-3 SuperNIC products that ship running as +/// NICs. Reconfiguring a card between DPU and NIC mode (the DPU BMC's +/// `Mode.Set` action) does not change its part number -- a flipped `900-9D3B6` +/// still classifies as [`MlxDeviceKind::Bf3DpuMode`] here. For the mode a +/// device is actually operating in, read [`ExploredMlxDevice::nic_mode`], +/// which comes from the DPU's own BMC. +/// +/// The `*Mode` variant names are frozen: they mirror the wire enum from when +/// this classification was believed to track the operating mode. #[derive(Copy, Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] pub enum MlxDeviceKind { - /// BlueField-3 operating as a NIC (part number `900-9D3B4...`). + /// BlueField-3 SuperNIC (part number `900-9D3B4...`). Bf3NicMode, - /// BlueField-3 operating as a DPU (part number `900-9D3B6...`). + /// BlueField-3 DPU (part number `900-9D3B6...`). Bf3DpuMode, /// BlueField-3 SuperNIC (part number `900-9D3D4...`). Bf3SuperNic, @@ -1843,9 +1849,9 @@ impl MlxDeviceKind { return Self::Unknown; }; let part_number = part_number.trim().to_lowercase(); - // `is_bf3_supernic_part_number` deliberately groups `900-9d3b4` and `900-9d3d4`; here - // we split them, because a NIC-mode DPU (`b4`) and a native SuperNIC - // (`d4`) are exactly what an operator needs told apart. + // `is_bf3_supernic_part_number` deliberately groups `900-9d3b4` and + // `900-9d3d4`; here we keep them apart because the wire enum + // distinguishes the two SuperNIC SKU families. if part_number.starts_with("900-9d3b6") || part_number == "sn37b36732" { Self::Bf3DpuMode } else if part_number.starts_with("900-9d3b4") { @@ -1858,20 +1864,15 @@ impl MlxDeviceKind { Self::Unknown } } - - /// Whether this is a BlueField-3 operating in NIC mode -- the devices whose - /// NIC firmware most needs auditing, since their Arm OS can't report it. - pub fn is_nic_mode(&self) -> bool { - matches!(self, Self::Bf3NicMode) - } } impl Display for MlxDeviceKind { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let label = match self { - Self::Bf3NicMode => "BlueField-3 (NIC mode)", - Self::Bf3DpuMode => "BlueField-3 (DPU mode)", - Self::Bf3SuperNic => "BlueField-3 SuperNIC", + // Both SuperNIC SKU families render under NVIDIA's product name; + // the part number alongside is the discriminator. + Self::Bf3NicMode | Self::Bf3SuperNic => "BlueField-3 SuperNIC", + Self::Bf3DpuMode => "BlueField-3 DPU", Self::Bf2Dpu => "BlueField-2 DPU", Self::Unknown => "Unknown", }; @@ -1919,7 +1920,9 @@ pub struct ExploredMlxDevice { #[serde(default, skip_serializing_if = "Option::is_none")] pub dpu_bmc_ip: Option, /// The DPU's authoritative operating mode, read from its own Redfish endpoint - /// when matched -- corroborates the part-number-derived `device_kind`. + /// when matched. This is the mode the card is running in right now; + /// `device_kind` is its factory SKU, and the two legitimately differ for a + /// DPU reconfigured to run as a NIC. #[serde(default, skip_serializing_if = "Option::is_none")] pub nic_mode: Option, } diff --git a/crates/rpc/proto/site_explorer.proto b/crates/rpc/proto/site_explorer.proto index 65868857a7..31ea986f04 100644 --- a/crates/rpc/proto/site_explorer.proto +++ b/crates/rpc/proto/site_explorer.proto @@ -193,13 +193,18 @@ message ExploredMlxDevice { // DPU we have explored -- the address to target for a firmware push. optional string dpu_bmc_ip = 9; // The DPU's authoritative operating mode, read from its own Redfish endpoint - // when matched. + // when matched. This is the mode the card is running in right now; it + // legitimately differs from the SKU-derived device_kind for a DPU + // reconfigured to run as a NIC. optional NicMode nic_mode = 10; } -// The kind of BlueField/Mellanox device, classified from its part number. A -// BlueField-3's part number records the mode it is operating in: 900-9D3B4 -// (NIC), 900-9D3B6 (DPU), 900-9D3D4 (SuperNIC). +// The kind of BlueField/Mellanox device, classified from its part number. The +// part number identifies the factory SKU, not the operating mode: 900-9D3B6 is +// a BlueField-3 DPU product, 900-9D3B4 and 900-9D3D4 are BlueField-3 SuperNIC +// products. A DPU/NIC mode flip does not change it -- nic_mode carries the +// operating mode. The *_MODE value names are frozen from when this was +// believed to track the operating mode. enum MlxDeviceKind { MLX_DEVICE_KIND_UNKNOWN = 0; MLX_DEVICE_KIND_BF3_NIC_MODE = 1; diff --git a/rest-api/proto/core/gen/v1/site_explorer_nico.pb.go b/rest-api/proto/core/gen/v1/site_explorer_nico.pb.go index 647a2aa995..f5e9e029bc 100644 --- a/rest-api/proto/core/gen/v1/site_explorer_nico.pb.go +++ b/rest-api/proto/core/gen/v1/site_explorer_nico.pb.go @@ -25,9 +25,12 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// The kind of BlueField/Mellanox device, classified from its part number. A -// BlueField-3's part number records the mode it is operating in: 900-9D3B4 -// (NIC), 900-9D3B6 (DPU), 900-9D3D4 (SuperNIC). +// The kind of BlueField/Mellanox device, classified from its part number. The +// part number identifies the factory SKU, not the operating mode: 900-9D3B6 is +// a BlueField-3 DPU product, 900-9D3B4 and 900-9D3D4 are BlueField-3 SuperNIC +// products. A DPU/NIC mode flip does not change it -- nic_mode carries the +// operating mode. The *_MODE value names are frozen from when this was +// believed to track the operating mode. type MlxDeviceKind int32 const ( @@ -1322,7 +1325,9 @@ type ExploredMlxDevice struct { // DPU we have explored -- the address to target for a firmware push. DpuBmcIp *string `protobuf:"bytes,9,opt,name=dpu_bmc_ip,json=dpuBmcIp,proto3,oneof" json:"dpu_bmc_ip,omitempty"` // The DPU's authoritative operating mode, read from its own Redfish endpoint - // when matched. + // when matched. This is the mode the card is running in right now; it + // legitimately differs from the SKU-derived device_kind for a DPU + // reconfigured to run as a NIC. NicMode *NicMode `protobuf:"varint,10,opt,name=nic_mode,json=nicMode,proto3,enum=site_explorer.NicMode,oneof" json:"nic_mode,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache diff --git a/rest-api/proto/core/src/v1/site_explorer_nico.proto b/rest-api/proto/core/src/v1/site_explorer_nico.proto index 4033b30ab7..076ab9df74 100644 --- a/rest-api/proto/core/src/v1/site_explorer_nico.proto +++ b/rest-api/proto/core/src/v1/site_explorer_nico.proto @@ -198,13 +198,18 @@ message ExploredMlxDevice { // DPU we have explored -- the address to target for a firmware push. optional string dpu_bmc_ip = 9; // The DPU's authoritative operating mode, read from its own Redfish endpoint - // when matched. + // when matched. This is the mode the card is running in right now; it + // legitimately differs from the SKU-derived device_kind for a DPU + // reconfigured to run as a NIC. optional NicMode nic_mode = 10; } -// The kind of BlueField/Mellanox device, classified from its part number. A -// BlueField-3's part number records the mode it is operating in: 900-9D3B4 -// (NIC), 900-9D3B6 (DPU), 900-9D3D4 (SuperNIC). +// The kind of BlueField/Mellanox device, classified from its part number. The +// part number identifies the factory SKU, not the operating mode: 900-9D3B6 is +// a BlueField-3 DPU product, 900-9D3B4 and 900-9D3D4 are BlueField-3 SuperNIC +// products. A DPU/NIC mode flip does not change it -- nic_mode carries the +// operating mode. The *_MODE value names are frozen from when this was +// believed to track the operating mode. enum MlxDeviceKind { MLX_DEVICE_KIND_UNKNOWN = 0; MLX_DEVICE_KIND_BF3_NIC_MODE = 1;