diff --git a/crates/health/example/config.bmc-mock-periodic.toml b/crates/health/example/config.bmc-mock-periodic.toml new file mode 100644 index 0000000000..92120d23a5 --- /dev/null +++ b/crates/health/example/config.bmc-mock-periodic.toml @@ -0,0 +1,68 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# BMC Mock should be running on localhost:1266 for this config to work. You can run it with `cargo run -p bmc-mock`. +# +# Variant of config.bmc-mock.toml that additionally enables PERIODIC log +# collection and demonstrates the collectors.logs.periodic.exclude_services +# option (see the [collectors.logs.periodic] section at the bottom). + +[[endpoint_sources.static_bmc_endpoints]] +ip = "127.0.0.1" +port = 1266 +mac = "aa:bb:cc:dd:ee:ff" +username = "admin" +password = "secret" +rack_id = "RACK_1" +machine = { id = "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0", serial = "MN-001", slot_number = 15, tray_index = 5, nvlink_domain_uuid = "00000000-0000-0000-0000-000000000000" } + +[endpoint_sources.carbide_api] +enabled = false + +[processors.leak_detection] +enabled = true +minimum_alerts_per_report = 2 + +[processors.rack_leak] +leaking_tray_threshold = 1 + +[sinks.tracing] +enabled = true + +[sinks.prometheus] +enabled = true + +[sinks.health_report] +enabled = false + +[sinks.rack_health_report] +enabled = false + + +# Periodic log collection (only mode where exclude_services applies; SSE never +# enumerates LogServices). exclude_services skips any Redfish LogService whose +# odata id contains one of the listed substrings — e.g. the bmcweb "Journal" +# HTTP-access log, which is high-volume noise and self-referential (polling the +# BMC generates Journal entries that the next poll re-collects). Empty (default) +# collects from every discovered LogService. +[collectors.logs] +mode = "periodic" + +[collectors.logs.periodic] +logs_collection_interval = "1m" +state_refresh_interval = "30m" +logs_state_file = "/tmp/logs_collector_{machine_id}.json" +exclude_services = ["Journal"] diff --git a/crates/health/example/config.example.toml b/crates/health/example/config.example.toml index 058dae1c9c..0095834c7a 100644 --- a/crates/health/example/config.example.toml +++ b/crates/health/example/config.example.toml @@ -207,6 +207,11 @@ max_backoff = "30s" logs_collection_interval = "5m" state_refresh_interval = "30m" logs_state_file = "/tmp/logs_collector_{machine_id}.json" +# Skip any Redfish LogService whose odata id contains one of these substrings. +# Typically used to drop the bmcweb "Journal" HTTP-access log (high-volume noise, +# and self-referential since polling generates Journal entries). Empty (default) +# collects from every discovered LogService. Periodic mode only. +exclude_services = [] # ============================================================================== # Switch Host Collectors: What data to collect from NVLink Switch Hosts diff --git a/crates/health/src/collectors/logs/periodic.rs b/crates/health/src/collectors/logs/periodic.rs index 2f184592f0..ae105646fa 100644 --- a/crates/health/src/collectors/logs/periodic.rs +++ b/crates/health/src/collectors/logs/periodic.rs @@ -42,6 +42,10 @@ pub struct LogsCollectorConfig { /// Attach Redfish diagnostic payloads to emitted log records. pub include_diagnostics: bool, + + /// Substrings; a discovered LogService whose odata id contains any of these + /// is skipped. Empty collects from every service. + pub exclude_services: Vec, } #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -70,6 +74,7 @@ pub struct LogsCollector { service_refresh_interval: Duration, data_sink: Option>, include_diagnostics: bool, + exclude_services: Vec, } impl PeriodicCollector for LogsCollector { @@ -90,6 +95,7 @@ impl PeriodicCollector for LogsCollector { service_refresh_interval: config.service_refresh_interval, data_sink: config.data_sink, include_diagnostics: config.include_diagnostics, + exclude_services: config.exclude_services, }) } @@ -140,19 +146,36 @@ impl LogsCollector { Ok(()) } + /// True if this service's odata id matches any configured exclude substring. + fn is_excluded(&self, service_id: &str) -> bool { + service_is_excluded(&self.exclude_services, service_id) + } + async fn discover_log_services(&self) -> Result>, HealthError> { let service_root = ServiceRoot::new(self.bmc.clone()).await?; let mut services = Vec::new(); let mut seen_ids = HashSet::new(); + let mut excluded_count = 0usize; + + let consider = |service: LogService, + services: &mut Vec>, + seen_ids: &mut HashSet, + excluded_count: &mut usize| { + let service_id = service.odata_id().to_string(); + if self.is_excluded(&service_id) { + *excluded_count += 1; + return; + } + if seen_ids.insert(service_id) { + services.push(service); + } + }; if let Ok(Some(manager_collection)) = service_root.managers().await { for manager in manager_collection.members().await.iter().flatten() { if let Ok(Some(log_services)) = manager.log_services().await { for service in log_services { - let service_id = service.odata_id().to_string(); - if seen_ids.insert(service_id) { - services.push(service); - } + consider(service, &mut services, &mut seen_ids, &mut excluded_count); } } } @@ -162,10 +185,7 @@ impl LogsCollector { for chassis in chassis_collection.members().await.iter().flatten() { if let Ok(Some(log_services)) = chassis.log_services().await { for service in log_services { - let service_id = service.odata_id().to_string(); - if seen_ids.insert(service_id) { - services.push(service); - } + consider(service, &mut services, &mut seen_ids, &mut excluded_count); } } } @@ -175,10 +195,7 @@ impl LogsCollector { for system in system_collection.members().await.iter().flatten() { if let Ok(Some(log_services)) = system.log_services().await { for service in log_services { - let service_id = service.odata_id().to_string(); - if seen_ids.insert(service_id) { - services.push(service); - } + consider(service, &mut services, &mut seen_ids, &mut excluded_count); } } } @@ -186,6 +203,7 @@ impl LogsCollector { tracing::info!( total_services = services.len(), + excluded_services = excluded_count, "Discovered distinct log services" ); @@ -396,3 +414,90 @@ impl LogsCollector { Ok((total_log_count, fetch_failures)) } } + +/// True if `service_id` contains any of the configured exclude substrings. +/// An empty `exclude_services` never excludes anything. Matching is a plain +/// (case-sensitive) substring test against the Redfish LogService odata id. +fn service_is_excluded(exclude_services: &[String], service_id: &str) -> bool { + exclude_services + .iter() + .any(|pat| !pat.is_empty() && service_id.contains(pat.as_str())) +} + +#[cfg(test)] +mod tests { + use carbide_test_support::{Check, check_values}; + + use super::service_is_excluded; + + const JOURNAL_BMC: &str = "/redfish/v1/Managers/BMC_0/LogServices/Journal"; + const JOURNAL_HGX: &str = "/redfish/v1/Managers/HGX_BMC_0/LogServices/Journal"; + const EVENTLOG: &str = "/redfish/v1/Systems/System_0/LogServices/EventLog"; + const XID: &str = "/redfish/v1/Chassis/HGX_GPU_0/LogServices/XID"; + const SEL: &str = "/redfish/v1/Systems/System_0/LogServices/SEL"; + + #[test] + fn service_exclusion_filter() { + check_values( + [ + Check { + scenario: "empty exclude list keeps all services", + input: (vec![], JOURNAL_BMC), + expect: false, + }, + Check { + scenario: "empty string pattern never excludes", + input: (vec!["".to_string()], JOURNAL_BMC), + expect: false, + }, + Check { + scenario: "substring match excludes BMC journal", + input: (vec!["Journal".to_string()], JOURNAL_BMC), + expect: true, + }, + Check { + scenario: "substring match excludes HGX journal", + input: (vec!["Journal".to_string()], JOURNAL_HGX), + expect: true, + }, + Check { + scenario: "non-matching service is kept", + input: (vec!["Journal".to_string()], EVENTLOG), + expect: false, + }, + Check { + scenario: "any of multiple patterns excludes", + input: (vec!["Journal".to_string(), "Dump".to_string()], JOURNAL_BMC), + expect: true, + }, + Check { + scenario: "second pattern in list matches", + input: ( + vec!["Journal".to_string(), "Dump".to_string()], + "/redfish/v1/Managers/BMC_0/LogServices/Dump", + ), + expect: true, + }, + Check { + scenario: "no pattern matches non-excluded services", + input: (vec!["Journal".to_string(), "Dump".to_string()], XID), + expect: false, + }, + Check { + scenario: "matching is case-sensitive", + input: ( + vec!["Journal".to_string()], + "/redfish/v1/Managers/BMC_0/LogServices/journal", + ), + expect: false, + }, + Check { + scenario: "SEL service is not excluded by Journal pattern", + input: (vec!["Journal".to_string()], SEL), + expect: false, + }, + ], + |(patterns, service_id)| service_is_excluded(&patterns, service_id), + ); + } +} diff --git a/crates/health/src/config.rs b/crates/health/src/config.rs index 70a36fac03..491ce18906 100644 --- a/crates/health/src/config.rs +++ b/crates/health/src/config.rs @@ -814,6 +814,13 @@ pub struct PeriodicLogConfig { /// Path to logs collector state file (supports {machine_id} placeholder). pub logs_state_file: String, + + /// Substrings matched against each Redfish LogService odata id; any service + /// whose id contains one of these is skipped during discovery. Use this to + /// drop noisy services such as the bmcweb "Journal" HTTP-access log. Empty + /// (default) collects from every discovered LogService. + #[serde(default)] + pub exclude_services: Vec, } impl Default for PeriodicLogConfig { @@ -822,6 +829,7 @@ impl Default for PeriodicLogConfig { logs_collection_interval: Duration::from_secs(300), state_refresh_interval: Duration::from_secs(1800), logs_state_file: "/tmp/logs_collector_{machine_id}.json".to_string(), + exclude_services: Vec::new(), } } } diff --git a/crates/health/src/discovery/spawn.rs b/crates/health/src/discovery/spawn.rs index 4b28e84532..0abe394dba 100644 --- a/crates/health/src/discovery/spawn.rs +++ b/crates/health/src/discovery/spawn.rs @@ -238,6 +238,7 @@ fn spawn_generic_redfish_collectors( service_refresh_interval: pcfg.state_refresh_interval, data_sink, include_diagnostics: ctx.logs_include_diagnostics, + exclude_services: pcfg.exclude_services.clone(), }, CollectorStartContext { limiter: ctx.limiter.clone(),