diff --git a/crates/health/src/config.rs b/crates/health/src/config.rs index 70a36fac03..875b95e664 100644 --- a/crates/health/src/config.rs +++ b/crates/health/src/config.rs @@ -17,7 +17,7 @@ use std::fmt::Debug; use std::net::{IpAddr, SocketAddr}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::time::Duration; use figment::Figment; @@ -84,6 +84,9 @@ pub struct EndpointSourcesConfig { /// Static BMC endpoints pub static_bmc_endpoints: Vec, + + /// Cluster inventory file source (file or cluster manager JSON RPC) + pub cluster: Configurable, } impl Default for EndpointSourcesConfig { @@ -91,6 +94,62 @@ impl Default for EndpointSourcesConfig { Self { carbide_api: Configurable::Enabled(CarbideApiConnectionConfig::default()), static_bmc_endpoints: Vec::new(), + cluster: Configurable::Disabled, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ClusterEndpointSourceConfig { + /// Path to a JSON inventory file containing BMC endpoints and credentials for the cluster. + /// Used when `cluster_manager_url` is absent. + #[serde(default)] + pub inventory_path: PathBuf, + + /// Cluster manager head-node URL (e.g. https://10.x.x.x:8081). + /// When set, inventory and credentials are fetched live via cluster manager JSON RPC + /// instead of reading `inventory_path`. + #[serde(default)] + pub cluster_manager_url: Option, + + /// Cluster manager partition to read bmcsettings from (default: "base"). + /// The cluster manager stores BMC username/password at partition level; this selects which partition. + #[serde(default = "default_cluster_manager_partition")] + pub cluster_manager_partition: String, + + /// Fallback BMC username if cluster manager JSON RPC does not return one. + /// Cluster manager default is "bright" (set during head-node installation). + #[serde(default = "default_cluster_manager_username")] + pub default_username: String, + + /// Fallback BMC password if cluster manager JSON RPC does not return one. + /// Must be set explicitly — no code-level default. + #[serde(default)] + pub default_password: Option, + + /// Optional BMC port override. None uses the BmcClient default (443/HTTPS). + #[serde(default)] + pub port: Option, +} + +fn default_cluster_manager_partition() -> String { + "base".to_string() +} + +fn default_cluster_manager_username() -> String { + "bright".to_string() +} + +impl Default for ClusterEndpointSourceConfig { + fn default() -> Self { + Self { + inventory_path: PathBuf::default(), + cluster_manager_url: None, + cluster_manager_partition: default_cluster_manager_partition(), + default_username: default_cluster_manager_username(), + default_password: None, + port: None, } } } diff --git a/crates/health/src/endpoint/cluster.rs b/crates/health/src/endpoint/cluster.rs new file mode 100644 index 0000000000..850f414aac --- /dev/null +++ b/crates/health/src/endpoint/cluster.rs @@ -0,0 +1,413 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use std::net::IpAddr; +use std::sync::Arc; + +use carbide_uuid::rack::RackId; +use mac_address::MacAddress; +use nv_redfish::bmc_http::reqwest::Client as ReqwestClient; +use reqwest::Client as HttpClient; +use serde::Deserialize; +use serde_json::json; +use url::Url; + +use crate::HealthError; +use crate::bmc::{BmcClient, FixedCredentialProvider}; +use crate::config::ClusterEndpointSourceConfig; +use crate::endpoint::{BmcAddr, BmcCredentials, BmcEndpoint, BoxFuture, EndpointSource}; + +// ── Inventory file shape ────────────────────────────────────────────────────── + +#[derive(Debug, Deserialize)] +struct FileInventory { + default_credentials: FileCredentials, + nodes: Vec, +} + +#[derive(Debug, Deserialize)] +struct FileCredentials { + username: String, + password: Option, +} + +#[derive(Debug, Deserialize)] +struct FileNode { + hostname: String, + bmc_ip: IpAddr, + rack: String, +} + +// ── Canonical internal node shape (both paths produce this) ────────────────── + +struct ClusterNode { + hostname: String, + bmc_ip: IpAddr, + rack: String, + username: String, + password: Option, +} + +// ── Cluster Manager JSON RPC ──────────────────────────────────────────────── +// +// The cluster manager exposes a JSON RPC API at /json/ for inventory and credential queries. +// Two calls are made: +// +// 1. cmdevice.getDevices — inventory: one entry per PhysicalNode with hostname +// and BMC interface IP. +// +// 2. cmpart.getPartition("") — credentials: the bmcsettings sub-object +// holds the cluster-wide BMC username and password (stored by the cluster manager daemon). +// Default partition is "base". +// +// Exact call names and response field paths need verification against the live +// head node: +// GET https://:8081/api — lists available services + calls +// POST https://:8081/json/ — JSON RPC endpoint +// +// When Joab confirms the real call names, update MANAGER_DEVICE_CALL and +// MANAGER_PARTITION_CALL below, and the field extractors. + +const MANAGER_DEVICE_CALL: &str = "getDevices"; +const MANAGER_PARTITION_CALL: &str = "getPartition"; + +fn build_http() -> Result { + HttpClient::builder() + .danger_accept_invalid_certs(true) + .build() + .map_err(|e| HealthError::GenericError(format!("Cluster manager HTTP client build failed: {e}"))) +} + +async fn manager_rpc( + http: &HttpClient, + json_rpc_url: &Url, + service: &str, + call: &str, + arg: serde_json::Value, +) -> Result { + let body = json!({ "service": service, "call": call, "arg": arg }); + + tracing::debug!(service, call, url = %json_rpc_url, "cluster manager JSON RPC call"); + + let response = http + .post(json_rpc_url.as_str()) + .json(&body) + .send() + .await + .map_err(|e| { + HealthError::GenericError(format!("cluster manager RPC {service}.{call} failed: {e}")) + })?; + + if !response.status().is_success() { + return Err(HealthError::GenericError(format!( + "cluster manager RPC {service}.{call} returned HTTP {}", + response.status() + ))); + } + + response + .json() + .await + .map_err(|e| HealthError::GenericError(format!("cluster manager RPC {service}.{call} parse failed: {e}"))) +} + +async fn fetch_manager_credentials( + http: &HttpClient, + json_rpc_url: &Url, + cfg: &ClusterEndpointSourceConfig, +) -> (String, Option) { + // cmsh equivalent: partition use → bmcsettings → get username / get password + // Password stored by the cluster manager daemon at partition level. + let result = manager_rpc( + http, + json_rpc_url, + "cmpart", + MANAGER_PARTITION_CALL, + json!(cfg.cluster_manager_partition), + ) + .await; + + let raw = match result { + Ok(v) => v, + Err(e) => { + tracing::warn!( + error = %e, + partition = %cfg.cluster_manager_partition, + "Failed to fetch cluster manager partition credentials; using config fallback" + ); + return (cfg.default_username.clone(), cfg.default_password.clone()); + } + }; + + tracing::debug!(response = ?raw, "Raw cluster manager partition response"); + + // Navigate to bmcsettings — try likely paths. + // Update once live /api confirms the exact field layout. + let bmc_settings = raw + .pointer("/bmcSettings") + .or_else(|| raw.pointer("/result/bmcSettings")) + .or_else(|| raw.pointer("/data/bmcSettings")); + + let username = bmc_settings + .and_then(|s| s.get("username").or_else(|| s.get("userName"))) + .and_then(|v| v.as_str()) + .map(str::to_string) + .unwrap_or_else(|| { + tracing::debug!( + "Cluster manager partition response missing bmcSettings.username; using config fallback. \ + Probe /api on head node to find correct field path." + ); + cfg.default_username.clone() + }); + + let password = bmc_settings + .and_then(|s| s.get("password").or_else(|| s.get("bmcPassword"))) + .and_then(|v| v.as_str()) + .map(str::to_string) + .or_else(|| { + tracing::debug!( + "Cluster manager partition response missing bmcSettings.password; using config fallback." + ); + cfg.default_password.clone() + }); + + (username, password) +} + +fn extract_manager_devices( + raw: &serde_json::Value, + username: String, + password: Option, +) -> Vec { + // The device list may be a top-level array or wrapped in result/data/items. + // Update once live /api confirms the exact response shape. + let items: Vec<&serde_json::Value> = if let Some(arr) = raw.as_array() { + arr.iter().collect() + } else { + ["result", "data", "items", "devices"] + .iter() + .find_map(|key| raw.get(key).and_then(|v| v.as_array())) + .map(|arr| arr.iter().collect()) + .unwrap_or_else(|| { + tracing::warn!( + response = ?raw, + "Cluster manager getDevices response has no recognized shape; \ + probe /api on head node and update extract_manager_devices in cluster.rs" + ); + vec![] + }) + }; + + let mut nodes = Vec::new(); + for item in items { + let hostname = item + .get("hostname") + .or_else(|| item.get("name")) + .and_then(|v| v.as_str()) + .map(str::to_string); + + // BMC IP lives in the BMC interface, not at the top level of the device. + // cmsh: device → interfaces → use ipmi0/ilo0/rf0 → get ip + // Try common field paths; update once confirmed. + let bmc_ip_str = item + .pointer("/interfaces/ipmi0/ip") + .or_else(|| item.pointer("/interfaces/bmc/ip")) + .or_else(|| item.pointer("/interfaces/ilo0/ip")) + .or_else(|| item.pointer("/interfaces/rf0/ip")) + .or_else(|| item.get("bmcAddress")) + .or_else(|| item.get("bmcIp")) + .or_else(|| item.get("ipmiAddress")) + .and_then(|v| v.as_str()); + + // Cluster manager category maps to our rack identifier. + let rack = item + .get("category") + .or_else(|| item.get("partition")) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + + let (Some(hostname), Some(bmc_ip_str)) = (hostname, bmc_ip_str) else { + tracing::debug!( + item = ?item, + "Cluster manager device entry missing hostname or BMC IP; skipping. \ + Update field paths in extract_manager_devices once head node is probed." + ); + continue; + }; + + let bmc_ip: IpAddr = match bmc_ip_str.parse() { + Ok(ip) => ip, + Err(e) => { + tracing::warn!(hostname, bmc_ip = bmc_ip_str, error = %e, "Invalid BMC IP; skipping"); + continue; + } + }; + + nodes.push(ClusterNode { + hostname, + bmc_ip, + rack, + username: username.clone(), + password: password.clone(), + }); + } + nodes +} + +// ── Source implementations ──────────────────────────────────────────────────── + +pub struct ClusterEndpointSource { + endpoints: Vec>, +} + +impl ClusterEndpointSource { + pub async fn from_config( + cfg: &ClusterEndpointSourceConfig, + reqwest: &ReqwestClient, + proxy_url: Option<&Url>, + cache_size: usize, + ) -> Result { + let nodes = if let Some(ref cluster_manager_url) = cfg.cluster_manager_url { + fetch_from_manager(cfg, cluster_manager_url).await? + } else { + read_from_file(cfg)? + }; + + let endpoints = build_endpoints(nodes, cfg.port, reqwest, proxy_url, cache_size); + tracing::info!(count = endpoints.len(), "Loaded cluster endpoints"); + Ok(Self { endpoints }) + } +} + +async fn fetch_from_manager( + cfg: &ClusterEndpointSourceConfig, + cluster_manager_url: &Url, +) -> Result, HealthError> { + let http = build_http()?; + let json_rpc_url = cluster_manager_url + .join("/json/") + .map_err(|e| HealthError::GenericError(format!("Invalid cluster manager URL: {e}")))?; + + tracing::info!(url = %json_rpc_url, partition = %cfg.cluster_manager_partition, "Fetching cluster inventory from cluster manager"); + + // Call 1: partition-level BMC credentials + let (username, password) = fetch_manager_credentials(&http, &json_rpc_url, cfg).await; + + // Call 2: device inventory + let raw = manager_rpc( + &http, + &json_rpc_url, + "cmdevice", + MANAGER_DEVICE_CALL, + json!({"type": "PhysicalNode"}), + ) + .await?; + + tracing::debug!(response = ?raw, "Raw cluster manager device response"); + + let nodes = extract_manager_devices(&raw, username, password); + tracing::info!(loaded = nodes.len(), "Cluster manager device fetch complete"); + Ok(nodes) +} + +fn read_from_file(cfg: &ClusterEndpointSourceConfig) -> Result, HealthError> { + let contents = std::fs::read_to_string(&cfg.inventory_path).map_err(|e| { + HealthError::GenericError(format!( + "Failed to read cluster inventory {}: {e}", + cfg.inventory_path.display() + )) + })?; + let inventory: FileInventory = serde_json::from_str(&contents)?; + let username = inventory.default_credentials.username; + let password = inventory.default_credentials.password; + Ok(inventory + .nodes + .into_iter() + .map(|n| ClusterNode { + hostname: n.hostname, + bmc_ip: n.bmc_ip, + rack: n.rack, + username: username.clone(), + password: password.clone(), + }) + .collect()) +} + +fn build_endpoints( + nodes: Vec, + port: Option, + reqwest: &ReqwestClient, + proxy_url: Option<&Url>, + cache_size: usize, +) -> Vec> { + let mut endpoints = Vec::with_capacity(nodes.len()); + for node in nodes { + let IpAddr::V4(v4) = node.bmc_ip else { + tracing::warn!( + hostname = %node.hostname, + bmc_ip = %node.bmc_ip, + "cluster endpoint has non-IPv4 BMC address; skipping" + ); + continue; + }; + + // Deterministic locally-administered MAC: 02:00::::. + // MAC is an internal cache key only; connectivity is IP-based. + let [o1, o2, o3, o4] = v4.octets(); + let mac = MacAddress::new([0x02, 0x00, o1, o2, o3, o4]); + + let addr = BmcAddr { ip: node.bmc_ip, port, mac }; + let credentials = BmcCredentials::UsernamePassword { + username: node.username, + password: node.password, + }; + let provider = Arc::new(FixedCredentialProvider::new(credentials)); + let bmc = match BmcClient::new( + reqwest.clone(), + addr.clone(), + provider, + proxy_url.cloned(), + cache_size, + ) { + Ok(c) => Arc::new(c), + Err(e) => { + tracing::warn!( + error = ?e, + hostname = %node.hostname, + "Failed to construct BmcClient for cluster endpoint; skipping" + ); + continue; + } + }; + endpoints.push(Arc::new(BmcEndpoint { + addr, + metadata: None, + rack_id: Some(RackId::new(&node.rack)), + bmc, + })); + } + endpoints +} + +impl EndpointSource for ClusterEndpointSource { + fn fetch_bmc_hosts<'a>( + &'a self, + ) -> BoxFuture<'a, Result>, HealthError>> { + Box::pin(async move { Ok(self.endpoints.clone()) }) + } +} diff --git a/crates/health/src/endpoint/mod.rs b/crates/health/src/endpoint/mod.rs index e5e4a9e33b..ec01cf3682 100644 --- a/crates/health/src/endpoint/mod.rs +++ b/crates/health/src/endpoint/mod.rs @@ -17,12 +17,14 @@ mod model; mod sources; +mod cluster; pub use model::{ BmcAddr, BmcCredentials, BmcEndpoint, EndpointMetadata, EndpointSource, MachineData, PowerShelfData, SwitchData, SwitchEndpointRole, }; pub use sources::{CompositeEndpointSource, StaticEndpointSource}; +pub use cluster::ClusterEndpointSource; pub use crate::bmc::{BoxFuture, CredentialProvider}; diff --git a/crates/health/src/lib.rs b/crates/health/src/lib.rs index f46311fc50..04f0513f82 100644 --- a/crates/health/src/lib.rs +++ b/crates/health/src/lib.rs @@ -40,7 +40,7 @@ pub use discovery::{DiscoveryIterationStats, DiscoveryLoopContext}; use crate::api_client::{ApiClientWrapper, ApiEndpointSource}; use crate::config::Configurable; -use crate::endpoint::{CompositeEndpointSource, EndpointSource, StaticEndpointSource}; +use crate::endpoint::{CompositeEndpointSource, EndpointSource, StaticEndpointSource, ClusterEndpointSource}; use crate::limiter::{BucketLimiter, NoopLimiter, RateLimiter}; use crate::metrics::{MetricsManager, run_metrics_server}; use crate::processor::{ @@ -119,7 +119,7 @@ struct EndpointWiring { source: Arc, } -fn build_endpoint_wiring(config: &Config) -> Result { +async fn build_endpoint_wiring(config: &Config) -> Result { let reqwest = ReqwestClient::with_params(ReqwestClientParams::new().accept_invalid_certs(true)) .map_err(BmcError::ReqwestError)?; let mut sources: Vec> = Vec::new(); @@ -143,13 +143,24 @@ fn build_endpoint_wiring(config: &Config) -> Result )); let endpoint_source = Arc::new(ApiEndpointSource::new( api_client, - reqwest, + reqwest.clone(), config.bmc_proxy_url.clone(), config.cache_size, )); sources.push(endpoint_source as Arc); } + if let Configurable::Enabled(ref source_cfg) = config.endpoint_sources.cluster { + let cluster_source = ClusterEndpointSource::from_config( + source_cfg, + &reqwest, + config.bmc_proxy_url.as_ref(), + config.cache_size, + ) + .await?; + sources.push(Arc::new(cluster_source)); + } + let composite_source = CompositeEndpointSource::new(sources); if composite_source.is_empty() { @@ -289,7 +300,7 @@ pub async fn run_service(config: Config) -> Result<(), HealthError> { let EndpointWiring { source: endpoint_source, - } = build_endpoint_wiring(&config)?; + } = build_endpoint_wiring(&config).await?; let data_sink = build_data_sink(&config, metrics_manager.clone())?;