diff --git a/Cargo.lock b/Cargo.lock index d437173187..8e3e638d7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1982,6 +1982,7 @@ dependencies = [ "arc-swap", "async-trait", "base64", + "bmc-mock", "bytes", "carbide-health-report", "carbide-instrument", diff --git a/crates/api-core/src/auth/internal_rbac_rules.rs b/crates/api-core/src/auth/internal_rbac_rules.rs index dcf4974919..f331e5d4ed 100644 --- a/crates/api-core/src/auth/internal_rbac_rules.rs +++ b/crates/api-core/src/auth/internal_rbac_rules.rs @@ -608,7 +608,10 @@ impl InternalRBACRules { x.perm("VerifySkuForMachine", vec![ForgeAdminCLI]); x.perm("RemoveSkuAssociation", vec![ForgeAdminCLI]); x.perm("GetAllSkuIds", vec![ForgeAdminCLI, SiteAgent, Flow]); - x.perm("FindSkusByIds", vec![ForgeAdminCLI, SiteAgent, Flow]); + x.perm( + "FindSkusByIds", + vec![ForgeAdminCLI, Health, SiteAgent, Flow], + ); x.perm("DeleteSku", vec![ForgeAdminCLI]); x.perm("UpdateSkuMetadata", vec![ForgeAdminCLI]); x.perm("UpdateMachineHardwareInfo", vec![ForgeAdminCLI]); diff --git a/crates/health/Cargo.toml b/crates/health/Cargo.toml index 67997cada4..bd896ec34a 100644 --- a/crates/health/Cargo.toml +++ b/crates/health/Cargo.toml @@ -108,6 +108,7 @@ tonic-prost-build = { workspace = true } bench-hooks = [] [dev-dependencies] +bmc-mock = { path = "../bmc-mock" } carbide-instrument = { path = "../instrument", features = ["test-support"] } carbide-test-support = { path = "../test-support" } criterion = { workspace = true } diff --git a/crates/health/example/config.example.toml b/crates/health/example/config.example.toml index c327f35fb6..a02a17330d 100644 --- a/crates/health/example/config.example.toml +++ b/crates/health/example/config.example.toml @@ -174,6 +174,13 @@ firmware_refresh_interval = "30m" poll_interval = "1m" state_refresh_interval = "30m" +# GPU inventory: out-of-band GPU count vs the machine's assigned SKU (issue #301). +# Disabled by default. Requires [endpoint_sources.carbide_api] (to resolve the +# expected count from the SKU) and [sinks.health_report] (to deliver alerts) to be +# enabled, or config validation fails. Uncomment to enable: +# [collectors.gpu_inventory] +# interval = "5m" + [collectors.logs] # "auto": (default) # - spawn SSE per endpoint diff --git a/crates/health/src/api_client.rs b/crates/health/src/api_client.rs index de83283648..6a059a3b17 100644 --- a/crates/health/src/api_client.rs +++ b/crates/health/src/api_client.rs @@ -150,6 +150,42 @@ impl ApiClientWrapper { Ok(()) } + /// Fetch SKU manifests by id — the expected-hardware source of truth used + /// to validate out-of-band GPU count against the assigned SKU. + pub async fn find_skus_by_ids( + &self, + ids: Vec, + ) -> Result, HealthError> { + let request = rpc::forge::SkusByIdsRequest { ids }; + + let response = self + .client + .find_skus_by_ids(request) + .await + .map_err(HealthError::ApiInvocationError)?; + + Ok(response.skus) + } + + /// Fetch a machine's currently-assigned SKU id, re-read live each call so SKU + /// assignments/changes after a collector starts are picked up (no caching). + pub async fn machine_hw_sku( + &self, + machine_id: carbide_uuid::machine::MachineId, + ) -> Result, HealthError> { + let request = rpc::forge::MachinesByIdsRequest { + machine_ids: vec![machine_id], + ..Default::default() + }; + + let response = self + .client + .find_machines_by_ids(request) + .await + .map_err(HealthError::ApiInvocationError)?; + + Ok(response.machines.into_iter().next().and_then(|m| m.hw_sku)) + } } #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/crates/health/src/collectors/gpu_inventory.rs b/crates/health/src/collectors/gpu_inventory.rs new file mode 100644 index 0000000000..829ad99984 --- /dev/null +++ b/crates/health/src/collectors/gpu_inventory.rs @@ -0,0 +1,412 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +use std::sync::Arc; + +use carbide_uuid::machine::MachineId; +use nv_redfish::core::{Bmc, ToSnakeCase}; +use nv_redfish::{Resource, ServiceRoot}; + +use crate::HealthError; +use crate::api_client::ApiClientWrapper; +use crate::collectors::runtime::{IterationResult, PeriodicCollector}; +use crate::endpoint::{BmcEndpoint, EndpointMetadata}; +use crate::sink::{ + Classification, CollectorEvent, DataSink, EventContext, HealthReport, HealthReportAlert, + HealthReportSuccess, HealthReportTarget, Probe, ReportSource, +}; + +/// Resolved expected-GPU state for the endpoint's assigned SKU. +#[derive(Clone, Debug)] +enum Expected { + /// No SKU assigned yet — nothing to validate. + NoSku, + /// The assigned SKU id, but its manifest could not be found. + SkuMissing(String), + /// Expected GPU count from the SKU manifest. + Count(u32), +} + +pub struct GpuInventoryCollectorConfig { + pub data_sink: Option>, + pub api_client: Arc, +} + +pub struct GpuInventoryCollector { + endpoint: Arc, + bmc: Arc, + event_context: EventContext, + data_sink: Option>, + api_client: Arc, + /// Machine id for this endpoint. The assigned SKU is re-read live each iteration + /// (not cached) so SKU assignments/changes after start are honored. + machine_id: Option, +} + +impl GpuInventoryCollector { + /// Resolve the expected GPU count from the machine's currently-assigned SKU. + /// Re-reads the SKU live every call (no caching) so assignments/changes after + /// the collector starts are picked up. + async fn resolve_expected(&self) -> Result { + let Some(machine_id) = self.machine_id else { + return Ok(Expected::NoSku); + }; + let Some(sku_id) = self.api_client.machine_hw_sku(machine_id).await? else { + return Ok(Expected::NoSku); + }; + let skus = self + .api_client + .find_skus_by_ids(vec![sku_id.clone()]) + .await?; + Ok(match skus.into_iter().next() { + None => Expected::SkuMissing(sku_id), + Some(sku) => Expected::Count( + sku.components + .map(|c| c.gpus.iter().map(|g| g.count).sum()) + .unwrap_or(0), + ), + }) + } + + /// Count GPUs out-of-band via Redfish. + /// + /// GPUs surface in two ways depending on how they attach, so we count both and + /// take the max — no vendor table, works across Dell / Lenovo / Supermicro / + /// NVIDIA regardless of GPU form: + /// - **SXM / HGX baseboards** (H100 SXM, GB200, GB300, GH200) → one + /// `HGX_GPU_*` chassis per GPU (`count_hgx_gpu_chassis`). + /// - **PCIe cards** (L40 / L40S, H100 PCIe) → Redfish `Processors` with + /// `ProcessorType == GPU` (`count_gpu_processors`). + /// + /// Some platforms (e.g. GB200) expose the *same* GPUs as both chassis and + /// processors, so we take the max rather than the sum to avoid double-counting. + /// `max` is deliberate: `min` would false-alert on platforms that populate only + /// one view, and `sum` would double-count dual-view platforms. + /// + /// Limitations: + /// - Trusts the BMC's Redfish inventory: if a failed GPU still appears (stale + /// inventory), the count won't drop and no shortage alert fires. The SEL + /// GPU-fault processor is the complementary signal for present-but-faulty GPUs. + /// - GPUs exposed ONLY under `PCIeDevices` (neither HGX chassis nor a GPU + /// Processor) are not yet counted. nv-redfish exposes no PCI device-class, so + /// such a path would have to match on model/name — add it once a platform that + /// needs it is confirmed via Redfish. + async fn count_gpus(&self) -> Result { + let root = ServiceRoot::new(self.bmc.clone()).await?; + let chassis_gpus = Self::count_hgx_gpu_chassis(&root).await?; + let processor_gpus = Self::count_gpu_processors(&root).await?; + Ok(chassis_gpus.max(processor_gpus)) + } + + /// HGX path: each GPU module is one `HGX_GPU_*` chassis + /// (`HGX_GPU_SXM_*` on Viking/H100, `HGX_GPU_*` on GH200/GB200/GB300). + /// NVSwitch trays excluded. + async fn count_hgx_gpu_chassis(root: &ServiceRoot) -> Result { + let Some(chassis_list) = root.chassis().await? else { + return Ok(0); + }; + let mut count = 0u32; + for c in chassis_list.members().await? { + let id = c.id().into_inner(); + if id.starts_with("HGX_GPU_") && !id.contains("NVSwitch") { + count += 1; + } + } + Ok(count) + } + + /// Standard-server path: count GPUs exposed as Redfish Processors + /// (`ProcessorType == GPU`). Vendor-neutral across Dell / Lenovo / HPE / + /// Supermicro, and model-agnostic — it counts every GPU (H100 PCIe, L40 / + /// L40S, A100, …), which is what "fewer GPUs than expected" needs. + /// + /// Caveat: this relies on the BMC exposing GPUs as `Processors`. A BMC that + /// only lists GPUs under `PCIeDevices` would count 0 here; add a PCIe fallback + /// if such firmware shows up in the fleet. Confirmed on hardware that the known + /// PCIe fleet platforms do expose GPUs this way — e.g. a Lenovo ThinkSystem + /// SR670 V2 with 8x NVIDIA L40 reports 8 `ProcessorType=GPU` processors via XCC. + async fn count_gpu_processors(root: &ServiceRoot) -> Result { + let Some(systems) = root.systems().await? else { + return Ok(0); + }; + let mut count = 0u32; + for system in systems.members().await? { + let processors = system.processors().await?.unwrap_or_default(); + for processor in processors { + let is_gpu = processor + .raw() + .processor_type + .flatten() + .is_some_and(|pt| pt.to_snake_case() == "gpu"); + if is_gpu { + count += 1; + } + } + } + Ok(count) + } + + fn emit_alert(&self, message: String) { + tracing::warn!(bmc = %self.endpoint.addr.mac, %message, "GPU inventory alert"); + let report = HealthReport { + source: ReportSource::GpuInventory, + target: Some(HealthReportTarget::Machine), + observed_at: Some(chrono::Utc::now()), + successes: Vec::new(), + alerts: vec![HealthReportAlert { + probe_id: Probe::GpuInventory, + target: None, + message, + classifications: vec![Classification::PreventAllocations], + }], + }; + self.emit(report); + } + + fn emit(&self, report: HealthReport) { + if let Some(sink) = &self.data_sink { + sink.handle_event( + &self.event_context, + &CollectorEvent::HealthReport(Arc::new(report)), + ); + } + } +} + +/// Build the health report for a GPU-count comparison — the core of issue #301: +/// alert when the BMC sees fewer GPUs than the SKU expects, success otherwise. +fn gpu_count_report(expected: u32, actual: u32) -> HealthReport { + if actual < expected { + HealthReport { + source: ReportSource::GpuInventory, + target: Some(HealthReportTarget::Machine), + observed_at: Some(chrono::Utc::now()), + successes: Vec::new(), + alerts: vec![HealthReportAlert { + probe_id: Probe::GpuInventory, + target: None, + message: format!( + "Expected gpu count ({expected}) does not match actual ({actual}) \ + as seen out-of-band via BMC" + ), + classifications: vec![Classification::PreventAllocations], + }], + } + } else { + HealthReport { + source: ReportSource::GpuInventory, + target: Some(HealthReportTarget::Machine), + observed_at: Some(chrono::Utc::now()), + successes: vec![HealthReportSuccess { + probe_id: Probe::GpuInventory, + target: None, + }], + alerts: Vec::new(), + } + } +} + +impl PeriodicCollector for GpuInventoryCollector { + type Config = GpuInventoryCollectorConfig; + + fn new_runner( + bmc: Arc, + endpoint: Arc, + config: Self::Config, + ) -> Result { + let event_context = + EventContext::from_endpoint(endpoint.as_ref(), "gpu_inventory_collector"); + let machine_id = match &endpoint.metadata { + Some(EndpointMetadata::Machine(m)) => Some(m.machine_id), + _ => None, + }; + Ok(Self { + endpoint, + bmc, + event_context, + data_sink: config.data_sink, + api_client: config.api_client, + machine_id, + }) + } + + async fn run_iteration(&mut self) -> Result { + let expected_count = match self.resolve_expected().await? { + // No SKU assigned, or the SKU declares zero GPUs (e.g. a CPU-only node): + // nothing to validate. Emit a success so any prior shortage alert on + // this machine clears (recovery), rather than lingering forever. + Expected::NoSku | Expected::Count(0) => { + self.emit(gpu_count_report(0, 0)); + return Ok(IterationResult { + refresh_triggered: false, + entity_count: None, + fetch_failures: 0, + }); + } + Expected::SkuMissing(sku_id) => { + self.emit_alert(format!("The assigned sku {sku_id} does not exist")); + return Ok(IterationResult { + refresh_triggered: false, + // Actual GPU count was never queried (bad SKU reference), so it + // is unknown here — report None rather than a misleading 0. + entity_count: None, + fetch_failures: 0, + }); + } + Expected::Count(n) => n, + }; + + let actual = self.count_gpus().await?; + let report = gpu_count_report(expected_count, actual); + if !report.alerts.is_empty() { + tracing::warn!( + bmc = %self.endpoint.addr.mac, + expected = expected_count, + actual, + "GPU count below SKU expectation" + ); + } + self.emit(report); + + Ok(IterationResult { + refresh_triggered: false, + entity_count: Some(actual as usize), + fetch_failures: 0, + }) + } + + fn collector_type(&self) -> &'static str { + "gpu_inventory_collector" + } +} + +/// Integration tests that drive the GPU-counting logic against realistic Redfish +/// trees served in-process by `bmc-mock` (no socket, no reqwest). +/// +/// `count_gpus` takes `max(count_hgx_gpu_chassis, count_gpu_processors)`, so both +/// counters are exercised here: +/// - `count_hgx_gpu_chassis` (`HGX_GPU_*`) is the same code for H100 SXM, GH200, +/// GB200 and GB300 — GB300 pins it to an exact count. No in-process H100/GH200 +/// helper exists (H100 is a Rust fixture without a helper; GH200 is a tarball +/// served over HTTP), so those are covered by the shared predicate. +/// - `count_gpu_processors` (`ProcessorType == GPU`) is validated against GB200, +/// the only in-process fixture that models GPUs as Redfish Processors — the same +/// signal PCIe GPUs (L40/L40S, H100 PCIe) produce on Dell/Lenovo/HPE. +#[cfg(test)] +mod bmc_mock_integration_tests { + use bmc_mock::test_support::{ + TestBmc, dell_poweredge_r750_bmc, dgx_gb300_bmc, wiwynn_gb200_bmc, + }; + + use super::{GpuInventoryCollector, gpu_count_report}; + use crate::sink::{Classification, Probe}; + + #[test] + fn alerts_when_fewer_gpus_than_sku_expects() { + // Issue #301 core: BMC sees 6 GPUs, the SKU expects 8 -> shortage alert. + let report = gpu_count_report(8, 6); + assert!(report.successes.is_empty()); + assert_eq!(report.alerts.len(), 1); + let alert = &report.alerts[0]; + assert_eq!(alert.probe_id, Probe::GpuInventory); + assert_eq!( + alert.classifications, + vec![Classification::PreventAllocations] + ); + assert!( + alert.message.contains('6') && alert.message.contains('8'), + "message should name actual and expected: {}", + alert.message + ); + } + + #[test] + fn success_when_gpu_count_matches_or_exceeds_sku() { + // Exact match -> success, no alert. + let matched = gpu_count_report(8, 8); + assert!(matched.alerts.is_empty()); + assert_eq!(matched.successes.len(), 1); + // More GPUs than the SKU expects is not a shortage -> still success. + assert!(gpu_count_report(4, 5).alerts.is_empty()); + // Clear/no-op case (no SKU or SKU declares 0 GPUs) -> success, no alert, + // which clears any prior shortage alert on the machine. + let cleared = gpu_count_report(0, 0); + assert!(cleared.alerts.is_empty()); + assert_eq!(cleared.successes.len(), 1); + } + + #[tokio::test] + async fn gb300_shortage_vs_sku_raises_alert_end_to_end() { + // Real Redfish count feeding the #301 decision: DGX GB300 exposes 4 GPU + // chassis; if its SKU expected 8, that must raise a shortage alert. + let h = dgx_gb300_bmc().await; + let actual = + GpuInventoryCollector::::count_hgx_gpu_chassis(h.service_root.as_ref()) + .await + .expect("count"); + assert_eq!(actual, 4); + assert_eq!(gpu_count_report(8, actual).alerts.len(), 1); + assert!(gpu_count_report(4, actual).alerts.is_empty()); + } + + #[tokio::test] + async fn dgx_gb300_counts_four_hgx_gpu_chassis() { + let h = dgx_gb300_bmc().await; + let root = h.service_root.as_ref(); + + let count = GpuInventoryCollector::::count_hgx_gpu_chassis(root) + .await + .expect("count_hgx_gpu_chassis"); + // The dgx_gb300 fixture models a 4-GPU compute tray (HGX_GPU_0..=3). + assert_eq!(count, 4, "expected 4 HGX_GPU_ chassis on DGX GB300"); + } + + #[tokio::test] + async fn gb200_exposes_gpus_as_both_chassis_and_processors() { + let h = wiwynn_gb200_bmc().await; + let root = h.service_root.as_ref(); + + let chassis = GpuInventoryCollector::::count_hgx_gpu_chassis(root) + .await + .expect("count_hgx_gpu_chassis"); + let processors = GpuInventoryCollector::::count_gpu_processors(root) + .await + .expect("count_gpu_processors"); + + // GB200 lists the same GPUs both ways. Both counters must find them; because + // count_gpus() takes the max (not the sum) they are not double-counted. + assert!(chassis > 0, "expected HGX_GPU_* chassis, got {chassis}"); + assert!(processors > 0, "expected GPU processors, got {processors}"); + } + + #[tokio::test] + async fn dell_r750_gpuless_counts_zero_by_both_methods() { + let h = dell_poweredge_r750_bmc().await; + let root = h.service_root.as_ref(); + + let chassis = GpuInventoryCollector::::count_hgx_gpu_chassis(root) + .await + .expect("count_hgx_gpu_chassis"); + let processors = GpuInventoryCollector::::count_gpu_processors(root) + .await + .expect("count_gpu_processors"); + + // The R750 fixture is a GPU-less server: neither method finds a GPU. + assert_eq!(chassis, 0); + assert_eq!(processors, 0); + } +} diff --git a/crates/health/src/collectors/mod.rs b/crates/health/src/collectors/mod.rs index 44cc5594aa..7f1b21651a 100644 --- a/crates/health/src/collectors/mod.rs +++ b/crates/health/src/collectors/mod.rs @@ -18,6 +18,7 @@ mod discovery; mod entity_metrics; mod firmware; +mod gpu_inventory; // NEW pub(crate) mod inventory; mod leak_detector; mod logs; @@ -30,6 +31,7 @@ mod sensors; pub use discovery::{EntityDiscoveryCollector, EntityDiscoveryCollectorConfig}; pub use entity_metrics::{MetricsCollector, MetricsCollectorConfig}; pub use firmware::{FirmwareCollector, FirmwareCollectorConfig}; +pub use gpu_inventory::{GpuInventoryCollector, GpuInventoryCollectorConfig}; pub(crate) use inventory::SharedInventory; pub use leak_detector::{LeakDetectorCollector, LeakDetectorCollectorConfig}; pub(crate) use logs::auto::{AutoFailureBudget, BudgetDecision, FailureKind}; @@ -46,4 +48,4 @@ pub use runtime::{ IterationResult, PeriodicCollector, StreamMetrics, StreamingCollector, StreamingCollectorStartContext, StreamingConnectResult, open_sse_stream, }; -pub use sensors::{SensorCollector, SensorCollectorConfig}; +pub use sensors::{SensorCollector, SensorCollectorConfig}; // NEW diff --git a/crates/health/src/config.rs b/crates/health/src/config.rs index 11ba3fc9b1..fa8e09a3df 100644 --- a/crates/health/src/config.rs +++ b/crates/health/src/config.rs @@ -553,6 +553,9 @@ pub struct CollectorsConfig { /// NVUE collector configuration for direct NVUE HTTP(s) polling of NVLink switches pub nvue: Configurable, + + /// GPU inventory collector: compares OOB GPU count vs the assigned SKU. + pub gpu_inventory: Configurable, } impl Default for CollectorsConfig { @@ -567,6 +570,7 @@ impl Default for CollectorsConfig { nmxt: Configurable::Disabled, nmxc: Configurable::Disabled, nvue: Configurable::Disabled, + gpu_inventory: Configurable::Disabled, } } } @@ -685,6 +689,21 @@ impl Default for MetricsCollectorConfig { } } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] +pub struct GpuInventoryConfig { + #[serde(with = "humantime_serde")] + pub interval: Duration, +} + +impl Default for GpuInventoryConfig { + fn default() -> Self { + Self { + interval: Duration::from_secs(300), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] pub struct ProcessorsConfig { @@ -1439,6 +1458,23 @@ impl Config { nmxc.validate()?; } + if self.collectors.gpu_inventory.is_enabled() { + if !self.endpoint_sources.carbide_api.is_enabled() { + return Err( + "collectors.gpu_inventory requires endpoint_sources.carbide_api to be enabled \ + (expected GPU counts are resolved from the machine SKU via the Carbide API)" + .to_string(), + ); + } + if !self.sinks.health_report.is_enabled() { + return Err( + "collectors.gpu_inventory requires sinks.health_report to be enabled \ + (GPU shortage alerts are delivered through the health-report sink)" + .to_string(), + ); + } + } + if let Configurable::Enabled(ref otlp) = self.sinks.otlp { tonic::transport::Channel::from_shared(otlp.endpoint.clone()) .map_err(|_| format!("invalid sinks.otlp.endpoint: {}", otlp.endpoint))?; @@ -1733,6 +1769,27 @@ username = "root" assert!(result.is_err()); } + #[test] + fn test_gpu_inventory_requires_carbide_api_and_health_report() { + let mut config = Config::default(); + config.collectors.gpu_inventory = Configurable::Enabled(GpuInventoryConfig::default()); + + // Enabled without the Carbide API source -> invalid (can't resolve SKU). + config.endpoint_sources.carbide_api = Configurable::Disabled; + config.sinks.health_report = Configurable::Enabled(HealthReportSinkConfig::default()); + assert!(config.validate().is_err()); + + // Enabled without the health-report sink -> invalid (alerts go nowhere). + config.endpoint_sources.carbide_api = + Configurable::Enabled(CarbideApiConnectionConfig::default()); + config.sinks.health_report = Configurable::Disabled; + assert!(config.validate().is_err()); + + // Both dependencies present -> valid. + config.sinks.health_report = Configurable::Enabled(HealthReportSinkConfig::default()); + assert!(config.validate().is_ok()); + } + #[test] fn test_config_validation() { let mut config = Config::default(); diff --git a/crates/health/src/discovery/context.rs b/crates/health/src/discovery/context.rs index 0ea6fa1c21..15989c60d0 100644 --- a/crates/health/src/discovery/context.rs +++ b/crates/health/src/discovery/context.rs @@ -24,11 +24,12 @@ use arc_swap::ArcSwapOption; use prometheus::{Histogram, HistogramOpts}; use crate::HealthError; +use crate::api_client::ApiClientWrapper; use crate::bmc::BmcClient; use crate::collectors::{Collector, LogDowngradeRegistry, SharedInventory}; use crate::config::{ Config, Configurable, DiscoveryConfig, FirmwareCollectorConfig as FirmwareCollectorOptions, - LeakDetectorCollectorConfig as LeakDetectorCollectorOptions, + GpuInventoryConfig, LeakDetectorCollectorConfig as LeakDetectorCollectorOptions, LogsCollectorConfig as LogsCollectorOptions, MetricsCollectorConfig as MetricsCollectorOptions, MtlsProfileConfig, NmxcCollectorConfig as NmxcCollectorOptions, NmxtCollectorConfig as NmxtCollectorOptions, NvueCollectorConfig as NvueCollectorOptions, @@ -50,10 +51,11 @@ pub(super) enum CollectorKind { Nmxc, NvueRest, NvueGnmi, + GpuInventory, } impl CollectorKind { - pub(super) const ALL: [CollectorKind; 10] = [ + pub(super) const ALL: [CollectorKind; 11] = [ CollectorKind::Discovery, CollectorKind::Sensor, CollectorKind::Metrics, @@ -64,6 +66,7 @@ impl CollectorKind { CollectorKind::Nmxc, CollectorKind::NvueRest, CollectorKind::NvueGnmi, + CollectorKind::GpuInventory, ]; pub(super) fn stop_message(self) -> &'static str { @@ -71,6 +74,7 @@ impl CollectorKind { CollectorKind::Discovery => { "Stopping entity discovery collector for removed BMC endpoint" } + CollectorKind::GpuInventory => "Stopping GPU inventory collector", CollectorKind::Sensor => "Stopping sensor collector for removed BMC endpoint", CollectorKind::Metrics => "Stopping entity metrics collector for removed BMC endpoint", CollectorKind::Logs => "Stopping logs collector for removed BMC endpoint", @@ -99,6 +103,7 @@ pub(super) struct CollectorState { nmxc: HashMap, Collector>, nvue_rest: HashMap, Collector>, nvue_gnmi: HashMap, Collector>, + gpu_inventory: HashMap, Collector>, inventories: HashMap, SharedInventory>, } @@ -115,6 +120,7 @@ impl CollectorState { nmxc: HashMap::new(), nvue_rest: HashMap::new(), nvue_gnmi: HashMap::new(), + gpu_inventory: HashMap::new(), inventories: HashMap::new(), } } @@ -131,6 +137,7 @@ impl CollectorState { CollectorKind::Nmxc => &self.nmxc, CollectorKind::NvueRest => &self.nvue_rest, CollectorKind::NvueGnmi => &self.nvue_gnmi, + CollectorKind::GpuInventory => &self.gpu_inventory, } } @@ -149,6 +156,7 @@ impl CollectorState { CollectorKind::Nmxc => &mut self.nmxc, CollectorKind::NvueRest => &mut self.nvue_rest, CollectorKind::NvueGnmi => &mut self.nvue_gnmi, + CollectorKind::GpuInventory => &mut self.gpu_inventory, } } @@ -199,6 +207,7 @@ impl CollectorState { .chain(self.nmxc.keys()) .chain(self.nvue_rest.keys()) .chain(self.nvue_gnmi.keys()) + .chain(self.gpu_inventory.keys()) .filter(|key| !active_keys.contains(*key)) .cloned() .collect() @@ -239,6 +248,8 @@ pub struct DiscoveryLoopContext { /// Whether any enabled sink consumes `CollectorEvent::Log` payloads. pub(crate) log_event_sink_enabled: bool, + pub(crate) gpu_inventory_config: Configurable, + pub(crate) api_client: Option>, pub(crate) log_downgrade_registry: Arc, /// Whether log collectors should attach diagnostic payload carriers. @@ -311,6 +322,16 @@ impl DiscoveryLoopContext { tls_config, tls_http_client_provider, log_event_sink_enabled: config.sinks.includes_log_events(), + gpu_inventory_config: config.collectors.gpu_inventory.clone(), + api_client: match &config.endpoint_sources.carbide_api { + Configurable::Enabled(source_cfg) => Some(Arc::new(ApiClientWrapper::new( + source_cfg.root_ca.clone(), + source_cfg.client_cert.clone(), + source_cfg.client_key.clone(), + &source_cfg.api_url, + ))), + _ => None, + }, log_downgrade_registry: Arc::new(LogDowngradeRegistry::new()), logs_include_diagnostics: config.sinks.includes_log_diagnostics(), }) diff --git a/crates/health/src/discovery/spawn.rs b/crates/health/src/discovery/spawn.rs index 57b2abc527..e6bfcef70e 100644 --- a/crates/health/src/discovery/spawn.rs +++ b/crates/health/src/discovery/spawn.rs @@ -25,11 +25,12 @@ use crate::bmc::BmcClient; use crate::collectors::{ AutoFailureBudget, BackoffConfig, BudgetDecision, Collector, CollectorStartContext, EntityDiscoveryCollector, EntityDiscoveryCollectorConfig, FailureKind, FirmwareCollector, - FirmwareCollectorConfig, LeakDetectorCollector, LeakDetectorCollectorConfig, LogsCollector, - LogsCollectorConfig, MetricsCollector, MetricsCollectorConfig, NmxcCollector, - NmxcCollectorConfig, NmxtCollector, NmxtCollectorConfig, NvueRestCollector, - NvueRestCollectorConfig, SensorCollector, SensorCollectorConfig, SseLogCollector, - SseLogCollectorConfig, StreamingCollectorStartContext, spawn_gnmi_collector, + FirmwareCollectorConfig, GpuInventoryCollector, GpuInventoryCollectorConfig, + LeakDetectorCollector, LeakDetectorCollectorConfig, LogsCollector, LogsCollectorConfig, + MetricsCollector, MetricsCollectorConfig, NmxcCollector, NmxcCollectorConfig, NmxtCollector, + NmxtCollectorConfig, NvueRestCollector, NvueRestCollectorConfig, SensorCollector, + SensorCollectorConfig, SseLogCollector, SseLogCollectorConfig, StreamingCollectorStartContext, + spawn_gnmi_collector, }; use crate::config::{Configurable, LogCollectionMode, PeriodicLogConfig}; use crate::endpoint::{BmcEndpoint, EndpointMetadata, SwitchEndpointRole}; @@ -383,6 +384,46 @@ fn spawn_generic_redfish_collectors( } } + if let Configurable::Enabled(gpu_cfg) = &ctx.gpu_inventory_config + && let Some(api_client) = &ctx.api_client + // GPU inventory validation only applies to machine endpoints (it needs a + // machine id + assigned SKU). Skip switch / power-shelf endpoints so we + // don't emit machine-target reports that get dropped for lack of context. + && matches!(endpoint.metadata, Some(EndpointMetadata::Machine(_))) + && !ctx.collectors.contains(CollectorKind::GpuInventory, &key) + { + let collector_registry = Arc::new( + ctx.metrics_manager + .create_collector_registry(format!("gpu_inventory_{key}"), metrics_prefix)?, + ); + match Collector::start::>( + endpoint_arc.clone(), + bmc.clone(), + GpuInventoryCollectorConfig { + data_sink: data_sink.clone(), + api_client: api_client.clone(), + }, + CollectorStartContext { + limiter: ctx.limiter.clone(), + iteration_interval: gpu_cfg.interval, + collector_registry, + metrics_manager: ctx.metrics_manager.clone(), + }, + ) { + Ok(monitor) => { + ctx.collectors + .insert(CollectorKind::GpuInventory, key.clone().into(), monitor); + } + Err(error) => { + tracing::error!( + ?error, + "Could not start GPU inventory collector for: {:?}", + endpoint.addr + ); + } + } + } + if let Configurable::Enabled(leak_detector_cfg) = &ctx.leak_detector_config && !ctx.collectors.contains(CollectorKind::LeakDetector, &key) { diff --git a/crates/health/src/lib.rs b/crates/health/src/lib.rs index 73011d2e01..5134f3516f 100644 --- a/crates/health/src/lib.rs +++ b/crates/health/src/lib.rs @@ -46,8 +46,8 @@ use crate::endpoint::{CompositeEndpointSource, EndpointSource, StaticEndpointSou use crate::limiter::{BucketLimiter, NoopLimiter, RateLimiter}; use crate::metrics::{MetricsManager, run_metrics_server}; use crate::processor::{ - BmcIntrusionEventProcessor, EventProcessingPipeline, EventProcessor, HealthReportProcessor, - LeakEventProcessor, RackLeakProcessor, + BmcIntrusionEventProcessor, EventProcessingPipeline, EventProcessor, GpuFaultEventProcessor, + HealthReportProcessor, LeakEventProcessor, RackLeakProcessor, }; use crate::sharding::ShardManager; use crate::sink::event_mapper::{OpenBmcEventMapper, RedfishEventMapper}; @@ -206,6 +206,11 @@ fn build_data_sink( processors.push(Arc::new(BmcIntrusionEventProcessor::new())); } + if config.sinks.health_report.is_enabled() { + // NEW block + processors.push(Arc::new(GpuFaultEventProcessor::new())); + } + if let Configurable::Enabled(ref leak_detection_cfg) = config.processors.leak_detection { processors.push(Arc::new(LeakEventProcessor::new( leak_detection_cfg.minimum_alerts_per_report, diff --git a/crates/health/src/processor/gpu_events.rs b/crates/health/src/processor/gpu_events.rs new file mode 100644 index 0000000000..d581456f92 --- /dev/null +++ b/crates/health/src/processor/gpu_events.rs @@ -0,0 +1,210 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +use std::borrow::Cow; +use std::sync::Arc; + +use super::{CollectorEvent, EventContext, EventProcessor}; +use crate::sink::{ + Classification, HealthReport, HealthReportAlert, HealthReportTarget, LogRecord, Probe, + ReportSource, +}; + +/// Substrings that mark a GPU-related fault in a Redfish SEL / LogService entry. +/// Matched case-insensitively against the log message body. +const GPU_FAULT_KEYWORDS: &[&str] = &[ + "gpu", // "GPU_0 ...", "HGX_GPU_..." + "nvlink", // "NVLink Training Error" + "row remap", // ECC row-remap failures + "xid", // driver XID surfaced via SEL on some platforms + "sxm", // HGX_GPU_SXM_* +]; + +#[derive(Default)] +pub struct GpuFaultEventProcessor; + +impl GpuFaultEventProcessor { + pub fn new() -> Self { + Self + } + + fn attr<'a>(attributes: &'a [(Cow<'static, str>, String)], key: &str) -> Option<&'a str> { + attributes + .iter() + .find(|(name, _)| name.as_ref() == key) + .map(|(_, value)| value.as_str()) + } + + /// Severities worth alerting on; "ok"/"info" GPU log lines are ignored. + fn is_actionable(severity: &str) -> bool { + matches!( + severity.to_ascii_uppercase().as_str(), + "WARN" | "WARNING" | "ERROR" | "FATAL" | "CRITICAL" + ) + } + + /// Returns the fault message if this log record looks like a GPU fault. + fn gpu_fault(record: &LogRecord) -> Option { + if !Self::is_actionable(&record.severity) { + return None; + } + // Normalize hyphens to spaces so hyphenated variants (e.g. "row-remap") + // match the space-separated keywords below. + let haystack = record.body.to_ascii_lowercase().replace('-', " "); + if GPU_FAULT_KEYWORDS.iter().any(|kw| haystack.contains(kw)) { + Some(record.body.clone()) + } else { + None + } + } +} + +impl EventProcessor for GpuFaultEventProcessor { + fn processor_type(&self) -> &'static str { + "gpu_fault_event_processor" + } + + fn process_event( + &self, + _context: &EventContext, + event: &CollectorEvent, + ) -> Vec { + let CollectorEvent::Log(record) = event else { + return Vec::new(); + }; + + let Some(message) = Self::gpu_fault(record) else { + return Vec::new(); + }; + + // Attach the SEL entry id as the target if present, else the BMC. + let target = Self::attr(&record.attributes, "entry_id") + .map(|id| format!("GPU/{id}")) + .or_else(|| Some("HostBMC".to_string())); + + let report = HealthReport { + source: ReportSource::GpuFault, + target: Some(HealthReportTarget::Machine), + observed_at: Some(chrono::Utc::now()), + successes: Vec::new(), + alerts: vec![HealthReportAlert { + probe_id: Probe::GpuFault, + target, + message, + classifications: vec![Classification::GpuFault], + }], + }; + + vec![CollectorEvent::HealthReport(Arc::new(report))] + } +} + +#[cfg(test)] +mod tests { + use std::net::{IpAddr, Ipv4Addr}; + use std::str::FromStr; + + use mac_address::MacAddress; + + use super::*; + use crate::endpoint::BmcAddr; + + fn context() -> EventContext { + EventContext { + endpoint_key: "42:9e:b1:bd:9d:dd".to_string(), + addr: BmcAddr { + ip: IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), + port: Some(443), + mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").expect("valid mac"), + }, + collector_type: "test", + metadata: None, + rack_id: None, + } + } + + fn log_event(body: &str, severity: &str) -> CollectorEvent { + CollectorEvent::Log(Box::new(LogRecord { + body: body.to_string(), + severity: severity.to_string(), + attributes: Vec::new(), + diagnostic_record: None, + })) + } + + fn process(event: CollectorEvent) -> Vec { + GpuFaultEventProcessor::new().process_event(&context(), &event) + } + + #[test] + fn alerts_on_gpu_fault_at_actionable_severity() { + let emitted = process(log_event("GPU_0 NVLink Training Error", "ERROR")); + assert_eq!(emitted.len(), 1); + + let CollectorEvent::HealthReport(report) = &emitted[0] else { + panic!("expected health report"); + }; + assert_eq!(report.source, ReportSource::GpuFault); + assert_eq!(report.target, Some(HealthReportTarget::Machine)); + + let alert = report.alerts.first().expect("one alert"); + assert_eq!(alert.probe_id, Probe::GpuFault); + assert_eq!(alert.classifications, vec![Classification::GpuFault]); + assert!(alert.message.contains("NVLink")); + } + + #[test] + fn ignores_non_actionable_severity() { + assert!(process(log_event("GPU_0 temperature normal", "INFO")).is_empty()); + assert!(process(log_event("GPU_0 ok", "OK")).is_empty()); + } + + #[test] + fn alerts_on_warning_severity() { + // The SSE log stream emits "Warning" (not "WARN"); both must alert. + assert_eq!(process(log_event("GPU_0 NVLink error", "Warning")).len(), 1); + assert_eq!(process(log_event("GPU_0 NVLink error", "WARN")).len(), 1); + } + + #[test] + fn ignores_logs_without_gpu_keywords() { + assert!(process(log_event("Fan1 speed warning", "WARN")).is_empty()); + } + + #[test] + fn ignores_non_log_events() { + assert!(process(CollectorEvent::CollectorRemoved).is_empty()); + } + + #[test] + fn matches_various_gpu_fault_keywords() { + for body in [ + "GPU_0 error", + "NVLink down", + "ECC row remap failure", + "ECC row-remap failure", // hyphenated variant must also match + "XID 79 fault", + "HGX_GPU_SXM_1 fault", + ] { + assert_eq!( + process(log_event(body, "ERROR")).len(), + 1, + "should alert on: {body}" + ); + } + } +} diff --git a/crates/health/src/processor/mod.rs b/crates/health/src/processor/mod.rs index 0560dd0d6b..6fc7aa5b45 100644 --- a/crates/health/src/processor/mod.rs +++ b/crates/health/src/processor/mod.rs @@ -20,10 +20,12 @@ use std::collections::VecDeque; use std::sync::Arc; use std::time::Instant; +mod gpu_events; // NEW (keep alphabetical: before health_report) mod health_report; mod intrusion_events; mod leak_events; mod rack_leak; +pub use gpu_events::GpuFaultEventProcessor; // NEW pub use health_report::HealthReportProcessor; pub use intrusion_events::BmcIntrusionEventProcessor; pub use leak_events::LeakEventProcessor; diff --git a/crates/health/src/sink/events.rs b/crates/health/src/sink/events.rs index 9075892af7..7723f7f1c3 100644 --- a/crates/health/src/sink/events.rs +++ b/crates/health/src/sink/events.rs @@ -375,6 +375,8 @@ pub enum ReportSource { TrayLeakDetection, RackLeakDetection, NvueLeakage, + GpuInventory, + GpuFault, } impl ReportSource { @@ -386,6 +388,8 @@ impl ReportSource { Self::TrayLeakDetection => "tray-leak-detection", Self::RackLeakDetection => "rack-leak-detection", Self::NvueLeakage => "nvue-leakage", + Self::GpuInventory => "gpu-inventory", + Self::GpuFault => "gpu-fault", } } } @@ -396,6 +400,8 @@ pub enum Probe { IntrusionSensorTriggered, LeakDetection, NvueLeakage, + GpuFault, + GpuInventory, } impl Probe { @@ -405,6 +411,10 @@ impl Probe { Self::IntrusionSensorTriggered => "IntrusionSensorTriggered", Self::LeakDetection => "BmcLeakDetection", Self::NvueLeakage => "NvueLeakage", + Self::GpuFault => "GpuFault", + // Reuse the existing shared "SkuValidation" probe id so OOB GPU-count + // alerts dedup with the machine-controller's in-band SKU alerts. + Self::GpuInventory => "SkuValidation", } } } @@ -419,6 +429,7 @@ pub enum Classification { PreventAllocations, Leak, LeakDetector, + GpuFault, } impl Classification { @@ -432,6 +443,7 @@ impl Classification { Self::PreventAllocations => "PreventAllocations", Self::Leak => "Leak", Self::LeakDetector => "LeakDetector", + Self::GpuFault => "GpuFault", } } }