feat(hw-health): out-of-band GPU count & SEL fault alerting

crates/api-core/src/auth/internal_rbac_rules.rs

@@ -608,7 +608,10 @@ impl InternalRBACRules {
         x.perm("VerifySkuForMachine", vec![ForgeAdminCLI]);
         x.perm("RemoveSkuAssociation", vec![ForgeAdminCLI]);
         x.perm("GetAllSkuIds", vec![ForgeAdminCLI, SiteAgent, Flow]);
-        x.perm("FindSkusByIds", vec![ForgeAdminCLI, SiteAgent, Flow]);
+        x.perm(
+            "FindSkusByIds",
+            vec![ForgeAdminCLI, Health, SiteAgent, Flow],
+        );
         x.perm("DeleteSku", vec![ForgeAdminCLI]);
         x.perm("UpdateSkuMetadata", vec![ForgeAdminCLI]);
         x.perm("UpdateMachineHardwareInfo", vec![ForgeAdminCLI]);

crates/health/Cargo.toml

@@ -99,6 +99,7 @@ tonic-prost-build = { workspace = true }
 bench-hooks = []
 
 [dev-dependencies]
+bmc-mock = { path = "../bmc-mock" }
 carbide-instrument = { path = "../instrument", features = ["test-support"] }
 carbide-test-support = { path = "../test-support" }
 criterion = { workspace = true }

crates/health/example/config.example.toml

@@ -174,6 +174,13 @@ firmware_refresh_interval = "30m"
 poll_interval = "1m"
 state_refresh_interval = "30m"
 
+# GPU inventory: out-of-band GPU count vs the machine's assigned SKU (issue #301).
+# Disabled by default. Requires [endpoint_sources.carbide_api] (to resolve the
+# expected count from the SKU) and [sinks.health_report] (to deliver alerts) to be
+# enabled, or config validation fails. Uncomment to enable:
+# [collectors.gpu_inventory]
+# interval = "5m"
+
 [collectors.logs]
 #  "auto": (default)
 #       - spawn SSE per endpoint

crates/health/src/api_client.rs

@@ -150,6 +150,42 @@ impl ApiClientWrapper {
 
         Ok(())
     }
+    /// Fetch SKU manifests by id — the expected-hardware source of truth used
+    /// to validate out-of-band GPU count against the assigned SKU.
+    pub async fn find_skus_by_ids(
+        &self,
+        ids: Vec<String>,
+    ) -> Result<Vec<rpc::forge::Sku>, HealthError> {
+        let request = rpc::forge::SkusByIdsRequest { ids };
+
+        let response = self
+            .client
+            .find_skus_by_ids(request)
+            .await
+            .map_err(HealthError::ApiInvocationError)?;
+
+        Ok(response.skus)
+    }
+
+    /// Fetch a machine's currently-assigned SKU id, re-read live each call so SKU
+    /// assignments/changes after a collector starts are picked up (no caching).
+    pub async fn machine_hw_sku(
+        &self,
+        machine_id: carbide_uuid::machine::MachineId,
+    ) -> Result<Option<String>, HealthError> {
+        let request = rpc::forge::MachinesByIdsRequest {
+            machine_ids: vec![machine_id],
+            ..Default::default()
+        };
+
+        let response = self
+            .client
+            .find_machines_by_ids(request)
+            .await
+            .map_err(HealthError::ApiInvocationError)?;
+
+        Ok(response.machines.into_iter().next().and_then(|m| m.hw_sku))
+    }
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]