Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 68 additions & 0 deletions crates/health/example/config.bmc-mock-periodic.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# BMC Mock should be running on localhost:1266 for this config to work. You can run it with `cargo run -p bmc-mock`.
#
# Variant of config.bmc-mock.toml that additionally enables PERIODIC log
# collection and demonstrates the collectors.logs.periodic.exclude_services
# option (see the [collectors.logs.periodic] section at the bottom).

[[endpoint_sources.static_bmc_endpoints]]
ip = "127.0.0.1"
port = 1266
mac = "aa:bb:cc:dd:ee:ff"
username = "admin"
password = "secret"
rack_id = "RACK_1"
machine = { id = "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0", serial = "MN-001", slot_number = 15, tray_index = 5, nvlink_domain_uuid = "00000000-0000-0000-0000-000000000000" }

[endpoint_sources.carbide_api]
enabled = false

[processors.leak_detection]
enabled = true
minimum_alerts_per_report = 2

[processors.rack_leak]
leaking_tray_threshold = 1

[sinks.tracing]
enabled = true

[sinks.prometheus]
enabled = true

[sinks.health_report]
enabled = false

[sinks.rack_health_report]
enabled = false


# Periodic log collection (only mode where exclude_services applies; SSE never
# enumerates LogServices). exclude_services skips any Redfish LogService whose
# odata id contains one of the listed substrings — e.g. the bmcweb "Journal"
# HTTP-access log, which is high-volume noise and self-referential (polling the
# BMC generates Journal entries that the next poll re-collects). Empty (default)
# collects from every discovered LogService.
[collectors.logs]
mode = "periodic"

[collectors.logs.periodic]
logs_collection_interval = "1m"
state_refresh_interval = "30m"
logs_state_file = "/tmp/logs_collector_{machine_id}.json"
exclude_services = ["Journal"]
5 changes: 5 additions & 0 deletions crates/health/example/config.example.toml
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,11 @@ max_backoff = "30s"
logs_collection_interval = "5m"
state_refresh_interval = "30m"
logs_state_file = "/tmp/logs_collector_{machine_id}.json"
# Skip any Redfish LogService whose odata id contains one of these substrings.
# Typically used to drop the bmcweb "Journal" HTTP-access log (high-volume noise,
# and self-referential since polling generates Journal entries). Empty (default)
# collects from every discovered LogService. Periodic mode only.
exclude_services = []

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Need to also define this config attrubute on L198.

Auto mode accepts both sse and periodic config variables for fallback behavior


# ==============================================================================
# Switch Host Collectors: What data to collect from NVLink Switch Hosts
Expand Down
129 changes: 117 additions & 12 deletions crates/health/src/collectors/logs/periodic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@ pub struct LogsCollectorConfig {

/// Attach Redfish diagnostic payloads to emitted log records.
pub include_diagnostics: bool,

/// Substrings; a discovered LogService whose odata id contains any of these
/// is skipped. Empty collects from every service.
pub exclude_services: Vec<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize, Default)]
Expand Down Expand Up @@ -70,6 +74,7 @@ pub struct LogsCollector<B: Bmc> {
service_refresh_interval: Duration,
data_sink: Option<Arc<dyn DataSink>>,
include_diagnostics: bool,
exclude_services: Vec<String>,
}

impl<B: Bmc + 'static> PeriodicCollector<B> for LogsCollector<B> {
Expand All @@ -90,6 +95,7 @@ impl<B: Bmc + 'static> PeriodicCollector<B> for LogsCollector<B> {
service_refresh_interval: config.service_refresh_interval,
data_sink: config.data_sink,
include_diagnostics: config.include_diagnostics,
exclude_services: config.exclude_services,
})
}

Expand Down Expand Up @@ -140,19 +146,36 @@ impl<B: Bmc + 'static> LogsCollector<B> {
Ok(())
}

/// True if this service's odata id matches any configured exclude substring.
fn is_excluded(&self, service_id: &str) -> bool {
service_is_excluded(&self.exclude_services, service_id)
}

async fn discover_log_services(&self) -> Result<Vec<LogService<B>>, HealthError> {
let service_root = ServiceRoot::new(self.bmc.clone()).await?;
let mut services = Vec::new();
let mut seen_ids = HashSet::new();
let mut excluded_count = 0usize;

let consider = |service: LogService<B>,
services: &mut Vec<LogService<B>>,
seen_ids: &mut HashSet<String>,
excluded_count: &mut usize| {
let service_id = service.odata_id().to_string();
if self.is_excluded(&service_id) {
*excluded_count += 1;
return;
}
if seen_ids.insert(service_id) {
services.push(service);
}
};

if let Ok(Some(manager_collection)) = service_root.managers().await {
for manager in manager_collection.members().await.iter().flatten() {
if let Ok(Some(log_services)) = manager.log_services().await {
for service in log_services {
let service_id = service.odata_id().to_string();
if seen_ids.insert(service_id) {
services.push(service);
}
consider(service, &mut services, &mut seen_ids, &mut excluded_count);
}
}
}
Expand All @@ -162,10 +185,7 @@ impl<B: Bmc + 'static> LogsCollector<B> {
for chassis in chassis_collection.members().await.iter().flatten() {
if let Ok(Some(log_services)) = chassis.log_services().await {
for service in log_services {
let service_id = service.odata_id().to_string();
if seen_ids.insert(service_id) {
services.push(service);
}
consider(service, &mut services, &mut seen_ids, &mut excluded_count);
}
}
}
Expand All @@ -175,17 +195,15 @@ impl<B: Bmc + 'static> LogsCollector<B> {
for system in system_collection.members().await.iter().flatten() {
if let Ok(Some(log_services)) = system.log_services().await {
for service in log_services {
let service_id = service.odata_id().to_string();
if seen_ids.insert(service_id) {
services.push(service);
}
consider(service, &mut services, &mut seen_ids, &mut excluded_count);
}
}
}
}

tracing::info!(
total_services = services.len(),
excluded_services = excluded_count,
"Discovered distinct log services"
);

Expand Down Expand Up @@ -396,3 +414,90 @@ impl<B: Bmc + 'static> LogsCollector<B> {
Ok((total_log_count, fetch_failures))
}
}

/// True if `service_id` contains any of the configured exclude substrings.
/// An empty `exclude_services` never excludes anything. Matching is a plain
/// (case-sensitive) substring test against the Redfish LogService odata id.
fn service_is_excluded(exclude_services: &[String], service_id: &str) -> bool {
exclude_services
.iter()
.any(|pat| !pat.is_empty() && service_id.contains(pat.as_str()))
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

#[cfg(test)]
mod tests {
use carbide_test_support::{Check, check_values};

use super::service_is_excluded;

const JOURNAL_BMC: &str = "/redfish/v1/Managers/BMC_0/LogServices/Journal";
const JOURNAL_HGX: &str = "/redfish/v1/Managers/HGX_BMC_0/LogServices/Journal";
const EVENTLOG: &str = "/redfish/v1/Systems/System_0/LogServices/EventLog";
const XID: &str = "/redfish/v1/Chassis/HGX_GPU_0/LogServices/XID";
const SEL: &str = "/redfish/v1/Systems/System_0/LogServices/SEL";

#[test]
fn service_exclusion_filter() {
check_values(
[
Check {
scenario: "empty exclude list keeps all services",
input: (vec![], JOURNAL_BMC),
expect: false,
},
Check {
scenario: "empty string pattern never excludes",
input: (vec!["".to_string()], JOURNAL_BMC),
expect: false,
},
Check {
scenario: "substring match excludes BMC journal",
input: (vec!["Journal".to_string()], JOURNAL_BMC),
expect: true,
},
Check {
scenario: "substring match excludes HGX journal",
input: (vec!["Journal".to_string()], JOURNAL_HGX),
expect: true,
},
Check {
scenario: "non-matching service is kept",
input: (vec!["Journal".to_string()], EVENTLOG),
expect: false,
},
Check {
scenario: "any of multiple patterns excludes",
input: (vec!["Journal".to_string(), "Dump".to_string()], JOURNAL_BMC),
expect: true,
},
Check {
scenario: "second pattern in list matches",
input: (
vec!["Journal".to_string(), "Dump".to_string()],
"/redfish/v1/Managers/BMC_0/LogServices/Dump",
),
expect: true,
},
Check {
scenario: "no pattern matches non-excluded services",
input: (vec!["Journal".to_string(), "Dump".to_string()], XID),
expect: false,
},
Check {
scenario: "matching is case-sensitive",
input: (
vec!["Journal".to_string()],
"/redfish/v1/Managers/BMC_0/LogServices/journal",
),
expect: false,
},
Check {
scenario: "SEL service is not excluded by Journal pattern",
input: (vec!["Journal".to_string()], SEL),
expect: false,
},
],
|(patterns, service_id)| service_is_excluded(&patterns, service_id),
);
}
}
8 changes: 8 additions & 0 deletions crates/health/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -814,6 +814,13 @@ pub struct PeriodicLogConfig {

/// Path to logs collector state file (supports {machine_id} placeholder).
pub logs_state_file: String,

/// Substrings matched against each Redfish LogService odata id; any service
/// whose id contains one of these is skipped during discovery. Use this to
/// drop noisy services such as the bmcweb "Journal" HTTP-access log. Empty
/// (default) collects from every discovered LogService.
#[serde(default)]
pub exclude_services: Vec<String>,
}

impl Default for PeriodicLogConfig {
Expand All @@ -822,6 +829,7 @@ impl Default for PeriodicLogConfig {
logs_collection_interval: Duration::from_secs(300),
state_refresh_interval: Duration::from_secs(1800),
logs_state_file: "/tmp/logs_collector_{machine_id}.json".to_string(),
exclude_services: Vec::new(),
}
}
}
Expand Down
1 change: 1 addition & 0 deletions crates/health/src/discovery/spawn.rs
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,7 @@ fn spawn_generic_redfish_collectors(
service_refresh_interval: pcfg.state_refresh_interval,
data_sink,
include_diagnostics: ctx.logs_include_diagnostics,
exclude_services: pcfg.exclude_services.clone(),
},
CollectorStartContext {
limiter: ctx.limiter.clone(),
Expand Down
Loading