From 1a4ad223244b937fa1d87ff36442399096c6593d Mon Sep 17 00:00:00 2001 From: Chet Nichols III Date: Thu, 9 Jul 2026 09:41:20 -0700 Subject: [PATCH] chore: standardize count metric descriptions on "Number of" Every count-style metric description now opens with `Number of` -- the preferred prefix @polarweasel called out in the #3266 review -- instead of the previous mix of "The amount of", "The number of", "The total number of", "Count of", etc. Primary callouts are: - The vault client counters in `forge_vault.rs` (`carbide-api.vault.requests_*`) claimed to count "tls connections" -- a copy-paste from the bmc-proxy metrics. They describe Vault requests now. - The `state-controller` templates lose their mangled grammar: "The amount of types that object handling tasks that have been ..." becomes "Number of object handling tasks that have been ...", and the state entered/exited counters now read "Number of times objects of type X have entered a certain state". - Small grammar fixes ride along inside touched strings only -- "machine validation runs", "pairs that have been identified", "machines that report software components", and lowercase "ips"/"tls"/"tcp" become "IPs"/"TLS"/"TCP". - The golden `/metrics` fixtures assert exposition output verbatim, so their `# HELP` lines get the same rewrite -- 67 lines across 11 fixture files. - Time and duration descriptions ("The amount of time ...", "Total time ...") are not counts and stay untouched, as does the multi-kind `carbide_site_explorer_update_explored_endpoints_count` row. Tests updated! Verified by running `test_integration` locally -- the regenerated `core_metrics.md` comes out identical to what's committed here. This supports https://github.com/NVIDIA/infra-controller/issues/3307 Signed-off-by: Chet Nichols III --- crates/agent/src/instrumentation.rs | 2 +- crates/api-core/src/handlers/machine_scout.rs | 2 +- crates/api-core/src/listener.rs | 2 +- .../src/logging/service_health_metrics.rs | 8 +- .../dpu_nic_firmware_metrics.rs | 6 +- .../machine_update_manager/host_firmware.rs | 6 +- .../src/machine_update_manager/metrics.rs | 8 +- .../fixtures/test_metrics_collector.txt | 8 +- .../src/machine_validation/metrics.rs | 8 +- .../metrics_collector/metrics.rs | 22 +- .../test_data/test_metrics_collector.txt | 16 +- crates/api-core/src/secrets/metrics.rs | 2 +- .../test_dpu_nic_firmware_metrics.txt | 6 +- .../test_host_firmware_metrics.txt | 4 +- .../test_machine_update_manager_metrics.txt | 4 +- .../test_network_segment_metrics_admin.txt | 28 +-- .../test_network_segment_metrics_tenant.txt | 22 +- .../test_network_segment_metrics_tor.txt | 28 +-- .../test_service_health_metrics.txt | 8 +- crates/bmc-proxy/src/bmc_proxy.rs | 6 +- crates/dns/src/lib.rs | 6 +- crates/dpa-manager/src/metrics.rs | 8 +- crates/dsx-exchange-consumer/README.md | 14 +- crates/dsx-exchange-consumer/src/metrics.rs | 14 +- crates/fmds/src/http_request_metrics.rs | 2 +- crates/health-metrics/src/lib.rs | 8 +- crates/health/src/collectors/runtime.rs | 4 +- crates/health/src/lib.rs | 2 +- crates/health/src/metrics.rs | 2 +- crates/ib-fabric/src/metrics.rs | 28 ++- crates/machine-controller/src/metrics.rs | 41 ++-- crates/mqtt-common/src/metrics.rs | 2 +- .../network-segment-controller/src/metrics.rs | 6 +- crates/nvlink-manager/src/metrics.rs | 2 +- .../src/fixtures/test_metrics_collector.txt | 6 +- crates/preingestion-manager/src/metrics.rs | 6 +- crates/secrets/src/forge_vault.rs | 6 +- crates/site-explorer/src/metrics.rs | 72 +++--- crates/sqlx-query-tracing/src/lib.rs | 2 +- crates/ssh-console/src/bmc/client_pool.rs | 8 +- crates/ssh-console/src/ssh_server.rs | 4 +- .../src/controller/periodic_enqueuer.rs | 2 +- .../src/controller/processor.rs | 6 +- crates/state-controller/src/metrics.rs | 22 +- docs/observability/core_metrics.md | 218 +++++++++--------- 45 files changed, 335 insertions(+), 352 deletions(-) diff --git a/crates/agent/src/instrumentation.rs b/crates/agent/src/instrumentation.rs index 42fb0b185d..47d04059b0 100644 --- a/crates/agent/src/instrumentation.rs +++ b/crates/agent/src/instrumentation.rs @@ -73,7 +73,7 @@ impl AgentMetricsState { pub fn create_metrics(meter: Meter) -> Arc { let http_counter = meter .u64_counter("http_requests") - .with_description("Total number of HTTP requests made.") + .with_description("Number of HTTP requests made.") .build(); let http_req_latency_histogram: Histogram = meter .f64_histogram("request_latency") diff --git a/crates/api-core/src/handlers/machine_scout.rs b/crates/api-core/src/handlers/machine_scout.rs index 004a8e1b29..e8d5b45149 100644 --- a/crates/api-core/src/handlers/machine_scout.rs +++ b/crates/api-core/src/handlers/machine_scout.rs @@ -441,7 +441,7 @@ fn record_reboot_duration_metric( log = warn, metric = counter, message = "Failed to wake up state handler for machine", - describe = "The amount of times a machine's state handler could not be woken after a \ + describe = "Number of times a machine's state handler could not be woken after a \ scout-reported event" )] struct StateHandlerWakeupFailed { diff --git a/crates/api-core/src/listener.rs b/crates/api-core/src/listener.rs index 557741981e..2844b981fb 100644 --- a/crates/api-core/src/listener.rs +++ b/crates/api-core/src/listener.rs @@ -306,7 +306,7 @@ pub async fn start( .build(); let connection_failed_counter = meter .u64_counter("carbide-api.tls.connection_fail") - .with_description("The amount of tcp connections that were failures") + .with_description("Number of inbound connections that failed") .build(); let mut tls_acceptor_created = Instant::now(); diff --git a/crates/api-core/src/logging/service_health_metrics.rs b/crates/api-core/src/logging/service_health_metrics.rs index 5c985202a5..bbcad8a384 100644 --- a/crates/api-core/src/logging/service_health_metrics.rs +++ b/crates/api-core/src/logging/service_health_metrics.rs @@ -71,7 +71,7 @@ pub fn start_export_service_health_metrics(health_context: ServiceHealthContext) health_context .meter .u64_observable_gauge("carbide_db_pool_idle_conns") - .with_description("The amount of idle connections in the carbide database pool") + .with_description("Number of idle connections in the carbide database pool") .with_callback(move |observer| { observer.observe(database_pool.num_idle() as u64, &[]); }) @@ -84,7 +84,7 @@ pub fn start_export_service_health_metrics(health_context: ServiceHealthContext) .meter .u64_observable_gauge("carbide_db_pool_total_conns") .with_description( - "The amount of total (active + idle) connections in the carbide database pool", + "Number of total (active + idle) connections in the carbide database pool", ) .with_callback(move |observer| { observer.observe(database_pool.size() as u64, &[]); @@ -97,7 +97,7 @@ pub fn start_export_service_health_metrics(health_context: ServiceHealthContext) health_context .meter .u64_observable_gauge("carbide_resourcepool_used_count") - .with_description("Count of values in the pool currently allocated") + .with_description("Number of values in the pool currently allocated") .with_callback(move |observer| { for (name, stats) in rp_stats.lock().unwrap().iter() { observer.observe( @@ -114,7 +114,7 @@ pub fn start_export_service_health_metrics(health_context: ServiceHealthContext) health_context .meter .u64_observable_gauge("carbide_resourcepool_free_count") - .with_description("Count of values in the pool currently available for allocation") + .with_description("Number of values in the pool currently available for allocation") .with_callback(move |observer| { for (name, stats) in rp_stats.lock().unwrap().iter() { let name_attr = KeyValue::new("pool", name.to_string()); diff --git a/crates/api-core/src/machine_update_manager/dpu_nic_firmware_metrics.rs b/crates/api-core/src/machine_update_manager/dpu_nic_firmware_metrics.rs index 41ed1562b3..df8de56612 100644 --- a/crates/api-core/src/machine_update_manager/dpu_nic_firmware_metrics.rs +++ b/crates/api-core/src/machine_update_manager/dpu_nic_firmware_metrics.rs @@ -47,7 +47,7 @@ impl DpuNicFirmwareUpdateMetrics { let running_dpu_updates = self.running_dpu_updates.clone(); meter .u64_observable_gauge("carbide_pending_dpu_nic_firmware_update_count") - .with_description("The number of machines in the system that need a firmware update.") + .with_description("Number of machines in the system that need a firmware update.") .with_callback(move |observer| { observer.observe(pending_firmware_updates.load(Relaxed), &[]); }) @@ -56,7 +56,7 @@ impl DpuNicFirmwareUpdateMetrics { meter .u64_observable_gauge("carbide_unavailable_dpu_nic_firmware_update_count") .with_description( - "The number of machines in the system that need a firmware update but are unavailable for update.", + "Number of machines in the system that need a firmware update but are unavailable for update.", ) .with_callback(move |observer| { observer.observe(unavailable_dpu_updates.load(Relaxed), &[]); @@ -66,7 +66,7 @@ impl DpuNicFirmwareUpdateMetrics { meter .u64_observable_gauge("carbide_running_dpu_updates_count") .with_description( - "The number of machines in the system that are running a firmware update.", + "Number of machines in the system that are running a firmware update.", ) .with_callback(move |observer| { observer.observe(running_dpu_updates.load(Relaxed), &[]); diff --git a/crates/api-core/src/machine_update_manager/host_firmware.rs b/crates/api-core/src/machine_update_manager/host_firmware.rs index d811ff7e5d..0c634e04b5 100644 --- a/crates/api-core/src/machine_update_manager/host_firmware.rs +++ b/crates/api-core/src/machine_update_manager/host_firmware.rs @@ -258,9 +258,7 @@ impl HostFirmwareUpdateMetrics { let active_firmware_updates = self.active_firmware_updates.clone(); meter .u64_observable_gauge("carbide_pending_host_firmware_update_count") - .with_description( - "The number of host machines in the system that need a firmware update.", - ) + .with_description("Number of host machines in the system that need a firmware update.") .with_callback(move |observer| { observer.observe(pending_firmware_updates.load(Ordering::Relaxed), &[]) }) @@ -268,7 +266,7 @@ impl HostFirmwareUpdateMetrics { meter .u64_observable_gauge("carbide_active_host_firmware_update_count") .with_description( - "The number of host machines in the system currently working on updating their firmware.", + "Number of host machines in the system currently working on updating their firmware.", ) .with_callback(move |observer| observer.observe(active_firmware_updates.load(Ordering::Relaxed), &[])) diff --git a/crates/api-core/src/machine_update_manager/metrics.rs b/crates/api-core/src/machine_update_manager/metrics.rs index 2775e5ff78..0ff5ea33d7 100644 --- a/crates/api-core/src/machine_update_manager/metrics.rs +++ b/crates/api-core/src/machine_update_manager/metrics.rs @@ -41,7 +41,7 @@ impl MachineUpdateManagerMetrics { self.concurrent_machine_updates_available.clone(); meter .u64_observable_gauge("carbide_machines_in_maintenance_count") - .with_description("The total number of machines in the system that are in maintenance.") + .with_description("Number of machines in the system that are in maintenance.") .with_callback(move |observer| { observer.observe(machines_in_maintenance.load(Ordering::Relaxed), &[]) }) @@ -49,7 +49,7 @@ impl MachineUpdateManagerMetrics { meter .u64_observable_gauge("carbide_machine_updates_started_count") .with_description( - "The number of machines in the system that are in the process of updating.", + "Number of machines in the system that are in the process of updating.", ) .with_callback(move |observer| { observer.observe(machine_updates_started.load(Ordering::Relaxed), &[]) @@ -57,9 +57,7 @@ impl MachineUpdateManagerMetrics { .build(); meter .u64_observable_gauge("carbide_concurrent_machine_updates_available") - .with_description( - "The number of machines in the system that we will update concurrently.", - ) + .with_description("Number of machines in the system that we will update concurrently.") .with_callback(move |observer| { observer.observe( concurrent_machine_updates_available.load(Ordering::Relaxed), diff --git a/crates/api-core/src/machine_validation/fixtures/test_metrics_collector.txt b/crates/api-core/src/machine_validation/fixtures/test_metrics_collector.txt index 3d97b905c3..7e311f1ca7 100644 --- a/crates/api-core/src/machine_validation/fixtures/test_metrics_collector.txt +++ b/crates/api-core/src/machine_validation/fixtures/test_metrics_collector.txt @@ -1,16 +1,16 @@ -# HELP carbide_machine_validation_completed Count of machine validation that have completed successfully +# HELP carbide_machine_validation_completed Number of machine validation runs that have completed successfully # TYPE carbide_machine_validation_completed gauge carbide_machine_validation_completed 10 -# HELP carbide_machine_validation_failed Count of machine validation that have failed +# HELP carbide_machine_validation_failed Number of machine validation runs that have failed # TYPE carbide_machine_validation_failed gauge carbide_machine_validation_failed 15 -# HELP carbide_machine_validation_in_progress Count of machine validation that are in progress +# HELP carbide_machine_validation_in_progress Number of machine validation runs that are in progress # TYPE carbide_machine_validation_in_progress gauge carbide_machine_validation_in_progress 20 # HELP carbide_machine_validation_oldest_active_age_seconds Age in seconds of the oldest active machine validation run # TYPE carbide_machine_validation_oldest_active_age_seconds gauge carbide_machine_validation_oldest_active_age_seconds 0 -# HELP carbide_machine_validation_stale_runs_count Count of active machine validation runs considered stale +# HELP carbide_machine_validation_stale_runs_count Number of active machine validation runs considered stale # TYPE carbide_machine_validation_stale_runs_count gauge carbide_machine_validation_stale_runs_count 0 # HELP carbide_machine_validation_tests The details of machine validation tests diff --git a/crates/api-core/src/machine_validation/metrics.rs b/crates/api-core/src/machine_validation/metrics.rs index 216340fe0b..dcd8c02727 100644 --- a/crates/api-core/src/machine_validation/metrics.rs +++ b/crates/api-core/src/machine_validation/metrics.rs @@ -47,7 +47,7 @@ fn hydrate_meter(meter: Meter, shared_metrics: SharedMetricsHolder for Rcode { component = "carbide-dns", log = off, metric = counter, - describe = "The number of DNS queries received, by query type" + describe = "Number of DNS queries received, by query type" )] struct DnsQueryReceived { #[label] @@ -246,7 +246,7 @@ struct DnsQueryReceived { component = "carbide-dns", log = off, metric = counter, - describe = "The number of DNS responses sent, by response code" + describe = "Number of DNS responses sent, by response code" )] struct DnsResponseSent { #[label] diff --git a/crates/dpa-manager/src/metrics.rs b/crates/dpa-manager/src/metrics.rs index 3bf451637b..c1e8e32134 100644 --- a/crates/dpa-manager/src/metrics.rs +++ b/crates/dpa-manager/src/metrics.rs @@ -116,20 +116,20 @@ impl DpaMonitorInstruments { .build(); let heartbeats_sent = meter .u64_counter("carbide_dpa_monitor_heartbeats_sent") - .with_description("The number of heartbeats sent to DPA interfaces") + .with_description("Number of heartbeats sent to DPA interfaces") .build(); let creates = meter .u64_counter("carbide_dpa_monitor_creates") - .with_description("The number of DPA interfaces created") + .with_description("Number of DPA interfaces created") .build(); let deletes = meter .u64_counter("carbide_dpa_monitor_deletes") - .with_description("The number of DPA interfaces deleted") + .with_description("Number of DPA interfaces deleted") .build(); meter .u64_observable_gauge("carbide_dpa_monitor_interfaces_scanned_count") - .with_description("The number of DPA interfaces scanned in the last monitor iteration") + .with_description("Number of DPA interfaces scanned in the last monitor iteration") .with_callback(move |o| { shared_metrics.if_available(|metrics, attrs| { o.observe(metrics.num_dpa_interfaces_scanned as u64, attrs); diff --git a/crates/dsx-exchange-consumer/README.md b/crates/dsx-exchange-consumer/README.md index 4e6640515e..481c43dc43 100644 --- a/crates/dsx-exchange-consumer/README.md +++ b/crates/dsx-exchange-consumer/README.md @@ -57,13 +57,13 @@ Exposed on the configured metrics endpoint (default `:9009`): | Metric | Type | Description | |--------|------|-------------| -| `nico_dsx_exchange_consumer_messages_received_total` | Counter | Total MQTT messages received | -| `nico_dsx_exchange_consumer_messages_processed_total` | Counter | Messages successfully processed | -| `nico_dsx_exchange_consumer_messages_dropped_total` | Counter | Messages dropped (queue overflow) | -| `nico_dsx_exchange_consumer_alerts_detected_total` | Counter | Leak alerts detected (by point_type) | -| `nico_dsx_exchange_consumer_dedup_skipped_total` | Counter | Messages skipped (deduplication) | -| `nico_dsx_exchange_consumer_metadata_cache_size` | Gauge | Metadata cache entry count | -| `nico_dsx_exchange_consumer_value_state_cache_size` | Gauge | Value state cache entry count | +| `carbide_dsx_exchange_consumer_messages_received_total` | Counter | Number of MQTT messages received | +| `carbide_dsx_exchange_consumer_messages_processed_total` | Counter | Number of messages successfully processed | +| `carbide_dsx_exchange_consumer_messages_dropped_total` | Counter | Number of messages dropped due to queue overflow | +| `carbide_dsx_exchange_consumer_alerts_detected_total` | Counter | Number of leak alerts detected | +| `carbide_dsx_exchange_consumer_dedup_skipped_total` | Counter | Number of messages skipped due to deduplication | +| `carbide_dsx_exchange_consumer_metadata_cache_size` | Gauge | Number of entries in the metadata cache | +| `carbide_dsx_exchange_consumer_value_state_cache_size` | Gauge | Number of entries in the value state cache | ## Running diff --git a/crates/dsx-exchange-consumer/src/metrics.rs b/crates/dsx-exchange-consumer/src/metrics.rs index f22792ca5f..f6b3bc6bbc 100644 --- a/crates/dsx-exchange-consumer/src/metrics.rs +++ b/crates/dsx-exchange-consumer/src/metrics.rs @@ -36,7 +36,7 @@ where let cache = cache.clone(); meter .u64_observable_gauge(format!("{METRICS_PREFIX}_metadata_cache_size")) - .with_description("Current number of entries in the metadata cache") + .with_description("Number of entries in the metadata cache") .with_callback(move |observer| { observer.observe(cache.entry_count(), &[]); }) @@ -54,7 +54,7 @@ where let cache = cache.clone(); meter .u64_observable_gauge(format!("{METRICS_PREFIX}_value_state_cache_size")) - .with_description("Current number of entries in the value state cache") + .with_description("Number of entries in the value state cache") .with_callback(move |observer| { observer.observe(cache.entry_count(), &[]); }) @@ -79,23 +79,23 @@ impl ConsumerMetrics { Self { messages_received: meter .u64_counter(format!("{METRICS_PREFIX}_messages_received_total")) - .with_description("Total number of MQTT messages received") + .with_description("Number of MQTT messages received") .build(), messages_processed: meter .u64_counter(format!("{METRICS_PREFIX}_messages_processed_total")) - .with_description("Total number of messages successfully processed") + .with_description("Number of messages successfully processed") .build(), messages_dropped: meter .u64_counter(format!("{METRICS_PREFIX}_messages_dropped_total")) - .with_description("Total number of messages dropped due to queue overflow") + .with_description("Number of messages dropped due to queue overflow") .build(), alerts_detected: meter .u64_counter(format!("{METRICS_PREFIX}_alerts_detected_total")) - .with_description("Total number of leak alerts detected") + .with_description("Number of leak alerts detected") .build(), dedup_skipped: meter .u64_counter(format!("{METRICS_PREFIX}_dedup_skipped_total")) - .with_description("Total number of messages skipped due to deduplication") + .with_description("Number of messages skipped due to deduplication") .build(), } } diff --git a/crates/fmds/src/http_request_metrics.rs b/crates/fmds/src/http_request_metrics.rs index 5a749ece9d..a8ab566f07 100644 --- a/crates/fmds/src/http_request_metrics.rs +++ b/crates/fmds/src/http_request_metrics.rs @@ -47,7 +47,7 @@ impl HttpRequestMetrics { fn new(meter: &Meter) -> Self { let http_counter = meter .u64_counter("http_requests") - .with_description("Total number of HTTP requests made.") + .with_description("Number of HTTP requests made.") .build(); let http_req_latency_histogram = meter .f64_histogram("request_latency") diff --git a/crates/health-metrics/src/lib.rs b/crates/health-metrics/src/lib.rs index 53bd9b0c07..97c75a79aa 100644 --- a/crates/health-metrics/src/lib.rs +++ b/crates/health-metrics/src/lib.rs @@ -151,7 +151,7 @@ pub fn register_health_gauges( meter .u64_observable_gauge(format!("{metric_prefix}_health_status_count")) .with_description( - format!("The total number of {display_name_plural} in the system that have reported either a healthy or not healthy status - based on the presence of health probe alerts"), + format!("Number of {display_name_plural} in the system that have reported either a healthy or not healthy status - based on the presence of health probe alerts"), ) .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { @@ -179,7 +179,7 @@ pub fn register_health_gauges( let project = project.clone(); meter .u64_observable_gauge(format!("{metric_prefix}_health_overrides_count")) - .with_description("The amount of health overrides that are configured in the site") + .with_description("Number of health overrides that are configured in the site") .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { let health = project(metrics); @@ -207,7 +207,7 @@ pub fn register_health_gauges( let project = project.clone(); meter .u64_observable_gauge(format!("{metric_prefix}_unhealthy_by_probe_id_count")) - .with_description("The amount of objects which reported a certain Health Probe Alert") + .with_description("Number of objects which reported a certain Health Probe Alert") .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { let health = project(metrics); @@ -231,7 +231,7 @@ pub fn register_health_gauges( meter .u64_observable_gauge(format!("{metric_prefix}_unhealthy_by_classification_count")) .with_description( - "The amount of objects which are marked with a certain classification due to being unhealthy", + "Number of objects which are marked with a certain classification due to being unhealthy", ) .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { diff --git a/crates/health/src/collectors/runtime.rs b/crates/health/src/collectors/runtime.rs index 3c11d1f21f..d9d0dd4c80 100644 --- a/crates/health/src/collectors/runtime.rs +++ b/crates/health/src/collectors/runtime.rs @@ -325,7 +325,7 @@ impl Collector { let refresh_counter = Counter::with_opts( Opts::new( format!("{}_collector_refresh_total", collector_registry.prefix()), - "Count of collector refreshes", + "Number of collector refreshes", ) .const_labels(const_labels.clone()), )?; @@ -346,7 +346,7 @@ impl Collector { "{}_collector_fetch_failures_total", collector_registry.prefix() ), - "Count of partial collector fetch failures", + "Number of partial collector fetch failures", ) .const_labels(const_labels), )?; diff --git a/crates/health/src/lib.rs b/crates/health/src/lib.rs index f46311fc50..4b1fcf134a 100644 --- a/crates/health/src/lib.rs +++ b/crates/health/src/lib.rs @@ -271,7 +271,7 @@ pub async fn run_service(config: Config) -> Result<(), HealthError> { "{metrics_prefix}_active_endpoints", metrics_prefix = &config.metrics.prefix ), - "Current number of active endpoints", + "Number of active endpoints", )?; registry.register(Box::new(active_endpoints_gauge.clone()))?; diff --git a/crates/health/src/metrics.rs b/crates/health/src/metrics.rs index 07725a0e5e..7a74faad54 100644 --- a/crates/health/src/metrics.rs +++ b/crates/health/src/metrics.rs @@ -75,7 +75,7 @@ impl ComponentMetrics { let failures_total = IntCounterVec::new( prometheus::Opts::new( format!("{prefix}_component_failures_total"), - "Count of component operation failures", + "Number of component operation failures", ), &["component_kind", "component_name"], )?; diff --git a/crates/ib-fabric/src/metrics.rs b/crates/ib-fabric/src/metrics.rs index 4457788898..cd89454a4e 100644 --- a/crates/ib-fabric/src/metrics.rs +++ b/crates/ib-fabric/src/metrics.rs @@ -132,7 +132,7 @@ impl IbFabricMonitorInstruments { let metrics = shared_metrics.clone(); meter .u64_observable_gauge("carbide_ib_monitor_fabrics_count") - .with_description("The amount of InfiniBand fabrics that are monitored") + .with_description("Number of InfiniBand fabrics that are monitored") .with_callback(move |o| { metrics.if_available(|metrics, attrs| { o.observe(metrics.num_fabrics as u64, attrs); @@ -146,7 +146,7 @@ impl IbFabricMonitorInstruments { meter .u64_observable_gauge("carbide_ib_monitor_machine_ib_status_updates_count") .with_description( - "The amount of Machines where the infiniband_status_observation got updated", + "Number of machines whose InfiniBand status observation was updated", ) .with_callback(move |o| { metrics.if_available(|metrics, attrs| { @@ -158,7 +158,7 @@ impl IbFabricMonitorInstruments { let ufm_changes_applied = meter .u64_counter("carbide_ib_monitor_ufm_changes_applied") - .with_description("The amount of changes that have been performed at UFM") + .with_description("Number of changes that have been performed at UFM") .build(); { @@ -166,7 +166,7 @@ impl IbFabricMonitorInstruments { meter .u64_observable_gauge("carbide_ib_monitor_machines_by_port_state_count") .with_description( - "The amount of Machines where the amount of total and active ports matches the values in attributes", + "Number of machines whose total and active port counts match the attribute values", ) .with_callback(move |o| { metrics.if_available(|metrics, attrs| { @@ -193,7 +193,7 @@ impl IbFabricMonitorInstruments { meter .u64_observable_gauge("carbide_ib_monitor_machines_by_ports_with_partitions_count") .with_description( - "The amount of Machines where a certain amount of ports is associated with at least one partition", + "Number of machines where a certain number of ports is associated with at least one partition", ) .with_callback(move |o| { metrics.if_available(|metrics, attrs| { @@ -219,7 +219,7 @@ impl IbFabricMonitorInstruments { meter .u64_observable_gauge("carbide_ib_monitor_machines_with_missing_pkeys_count") .with_description( - "The amount of machines where at least one port is not assigned to the expected pkey on UFM", + "Number of machines where at least one port is not assigned to the expected pkey on UFM", ) .with_callback(move |o| { metrics.if_available(|metrics, attrs| { @@ -234,7 +234,7 @@ impl IbFabricMonitorInstruments { meter .u64_observable_gauge("carbide_ib_monitor_machines_with_unexpected_pkeys_count") .with_description( - "The amount of machines where at least one port is assigned to an unexpected pkey on UFM", + "Number of machines where at least one port is assigned to an unexpected pkey on UFM", ) .with_callback(move |o| { metrics.if_available(|metrics, attrs| { @@ -249,7 +249,7 @@ impl IbFabricMonitorInstruments { meter .u64_observable_gauge("carbide_ib_monitor_machines_with_unknown_pkeys_count") .with_description( - "The amount of machines where at least one port is assigned to a pkey value that is not associated with any partition ID", + "Number of machines where at least one port is assigned to a pkey value that is not associated with any partition ID", ) .with_callback(move |o| { metrics.if_available(|metrics, attrs| { @@ -263,7 +263,7 @@ impl IbFabricMonitorInstruments { let metrics = shared_metrics.clone(); meter .u64_observable_gauge("carbide_ib_monitor_ufm_version_count") - .with_description("The amount of UFM deployments per version") + .with_description("Number of UFM deployments per version") .with_callback(move |o| { metrics.if_available(|metrics, attrs| { for (fabric, metrics) in metrics.fabrics.iter() { @@ -324,9 +324,7 @@ impl IbFabricMonitorInstruments { let metrics = shared_metrics.clone(); meter .u64_observable_gauge("carbide_ib_monitor_insecure_fabric_configuration_count") - .with_description( - "The amount of InfiniBand fabrics that are not configured securely", - ) + .with_description("Number of InfiniBand fabrics that are not configured securely") .with_callback(move |o| { metrics.if_available(|metrics, attrs| { for (fabric, metrics) in metrics.fabrics.iter() { @@ -351,7 +349,7 @@ impl IbFabricMonitorInstruments { "carbide_ib_monitor_allow_insecure_fabric_configuration_count", ) .with_description( - "The amount of InfiniBand fabrics that are not configured securely", + "Number of InfiniBand fabrics that are allowed to use insecure configuration", ) .with_callback(move |o| { metrics.if_available(|metrics, attrs| { @@ -375,7 +373,7 @@ impl IbFabricMonitorInstruments { meter .u64_observable_gauge("carbide_ib_monitor_ufm_partitions_count") .with_description( - "The amount partitions registered at UFM in total (incl non Forge partitions)", + "Number of partitions registered at UFM in total (including non-Forge partitions)", ) .with_callback(move |o| { metrics.if_available(|metrics, attrs| { @@ -398,7 +396,7 @@ impl IbFabricMonitorInstruments { meter .u64_observable_gauge("carbide_ib_monitor_ufm_ports_by_state_count") .with_description( - "Total number of ports reported by UFM (incl non Forge managed ports)", + "Number of ports reported by UFM in each port state (including non-Forge-managed ports)", ) .with_callback(move |o| { metrics.if_available(|metrics, attrs| { diff --git a/crates/machine-controller/src/metrics.rs b/crates/machine-controller/src/metrics.rs index 54f93c60cd..61faf4509d 100644 --- a/crates/machine-controller/src/metrics.rs +++ b/crates/machine-controller/src/metrics.rs @@ -124,7 +124,7 @@ impl MetricsEmitter for MachineMetricsEmitter { let metrics = shared_metrics.clone(); meter .u64_observable_gauge("carbide_gpus_total_count") - .with_description("The total number of GPUs available in the NICo deployment") + .with_description("Number of GPUs in the NICo deployment") .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { observer.observe(metrics.gpus_total as u64, attrs); @@ -136,7 +136,7 @@ impl MetricsEmitter for MachineMetricsEmitter { let metrics = shared_metrics.clone(); meter .u64_observable_gauge("carbide_hosts_usable_count") - .with_description("The remaining number of hosts in the NICo deployment which are available for immediate instance creation") + .with_description("Number of remaining hosts in the NICo deployment which are available for immediate instance creation") .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { observer.observe( @@ -151,7 +151,7 @@ impl MetricsEmitter for MachineMetricsEmitter { let metrics = shared_metrics.clone(); meter .u64_observable_gauge("carbide_gpus_usable_count") - .with_description("The remaining number of GPUs in the NICo deployment which are available for immediate instance creation") + .with_description("Number of remaining GPUs in the NICo deployment which are available for immediate instance creation") .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { observer.observe( @@ -166,7 +166,7 @@ impl MetricsEmitter for MachineMetricsEmitter { let metrics = shared_metrics.clone(); meter .u64_observable_gauge("carbide_gpus_in_use_count") - .with_description("The total number of GPUs that are actively used by tenants in instances in the NICo deployment") + .with_description("Number of GPUs that are actively used by tenants in instances in the NICo deployment") .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { let total_in_use_gpus = metrics.gpus_in_use_by_tenant.values().copied().reduce(|a,b| a + b).unwrap_or_default(); @@ -182,7 +182,7 @@ impl MetricsEmitter for MachineMetricsEmitter { let metrics = shared_metrics.clone(); meter .u64_observable_gauge("carbide_hosts_in_use_count") - .with_description("The total number of hosts that are actively used by tenants as instances in the NICo deployment") + .with_description("Number of hosts that are actively used by tenants as instances in the NICo deployment") .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { let total_in_use_hosts = metrics.hosts_in_use_by_tenant.values().copied().reduce(|a,b| a + b).unwrap_or_default(); @@ -199,7 +199,7 @@ impl MetricsEmitter for MachineMetricsEmitter { meter .u64_observable_gauge("carbide_gpus_in_use_by_tenant_count") .with_description( - "The number of GPUs that are actively used by tenants as instances - by tenant", + "Number of GPUs that are actively used by tenants as instances - by tenant", ) .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { @@ -219,14 +219,15 @@ impl MetricsEmitter for MachineMetricsEmitter { meter .u64_observable_gauge("carbide_hosts_in_use_by_tenant_count") .with_description( - "The number of hosts that are actively used by tenants as instances - by tenant", + "Number of hosts that are actively used by tenants as instances - by tenant", ) .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { for (org, count) in &metrics.hosts_in_use_by_tenant { observer.observe( *count as u64, - &[attrs, &[KeyValue::new("tenant_org_id", org.to_string())]].concat(), + &[attrs, &[KeyValue::new("tenant_org_id", org.to_string())]] + .concat(), ); } }) @@ -237,7 +238,7 @@ impl MetricsEmitter for MachineMetricsEmitter { let metrics = shared_metrics.clone(); meter .u64_observable_gauge("carbide_dpus_up_count") - .with_description("The total number of DPUs in the system that are up. Up means we have received a health report less than 5 minutes ago.") + .with_description("Number of DPUs in the system that are up. Up means we have received a health report less than 5 minutes ago.") .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { observer.observe( @@ -252,7 +253,7 @@ impl MetricsEmitter for MachineMetricsEmitter { let metrics = shared_metrics.clone(); meter .u64_observable_gauge("carbide_dpus_healthy_count") - .with_description("The total number of DPUs in the system that have reported healthy in the last report. Healthy does not imply up - the report from the DPU might be outdated.") + .with_description("Number of DPUs in the system that have reported healthy in the last report. Healthy does not imply up - the report from the DPU might be outdated.") .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { observer.observe( @@ -288,9 +289,7 @@ impl MetricsEmitter for MachineMetricsEmitter { let metrics = shared_metrics.clone(); meter .u64_observable_gauge("carbide_dpu_health_check_failed_count") - .with_description( - "The total number of DPUs in the system that have failed a health-check.", - ) + .with_description("Number of DPUs in the system that have failed a health-check.") .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { for ((probe, target), count) in &metrics.unhealthy_dpus_by_probe_id { @@ -324,7 +323,7 @@ impl MetricsEmitter for MachineMetricsEmitter { meter .u64_observable_gauge("carbide_hosts_by_sku_count") .with_description( - "The amount of hosts by SKU and device type ('unknown' for hosts without SKU)", + "Number of hosts by SKU and device type ('unknown' for hosts without SKU)", ) .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { @@ -348,7 +347,7 @@ impl MetricsEmitter for MachineMetricsEmitter { let metrics = shared_metrics.clone(); meter .u64_observable_gauge("carbide_dpu_agent_version_count") - .with_description("The amount of DPU agents which have reported a certain version.") + .with_description("Number of DPU agents which have reported a certain version.") .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { for (version, count) in &metrics.agent_versions { @@ -369,9 +368,7 @@ impl MetricsEmitter for MachineMetricsEmitter { let metrics = shared_metrics.clone(); meter .u64_observable_gauge("carbide_dpu_firmware_version_count") - .with_description( - "The amount of DPUs which have reported a certain firmware version.", - ) + .with_description("Number of DPUs which have reported a certain firmware version.") .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { for (version, count) in &metrics.dpu_firmware_versions { @@ -391,7 +388,7 @@ impl MetricsEmitter for MachineMetricsEmitter { meter .u64_observable_gauge("carbide_machine_inventory_component_version_count") .with_description( - "The amount of machines report software components with a certain version.", + "Number of machines that report software components with a certain version.", ) .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { @@ -437,12 +434,12 @@ impl MetricsEmitter for MachineMetricsEmitter { let machine_reboot_attempts_in_booting_with_discovery_image = meter .u64_histogram("carbide_reboot_attempts_in_booting_with_discovery_image") - .with_description("The amount of machines rebooted again in BootingWithDiscoveryImage since there is no response after a certain time from host.") + .with_description("Reboot attempts per machine in BootingWithDiscoveryImage, recorded when a machine is rebooted again after no response from the host.") .build(); let machine_reboot_attempts_in_failed_during_discovery = meter .u64_histogram("carbide_reboot_attempts_in_failed_during_discovery") - .with_description("The amount of machines rebooted again in Failed state due to discovery failure since there is no response after a certain time from host.") + .with_description("Reboot attempts per machine in Failed state due to discovery failure, recorded when a machine is rebooted again after no response from the host.") .build(); { @@ -450,7 +447,7 @@ impl MetricsEmitter for MachineMetricsEmitter { meter .u64_observable_gauge("carbide_hosts_with_bios_password_set") .with_description( - "The total number of Hosts in the system that have their BIOS password set.", + "Number of hosts in the system that have their BIOS password set.", ) .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { diff --git a/crates/mqtt-common/src/metrics.rs b/crates/mqtt-common/src/metrics.rs index 5bc9ca4d64..48e1598962 100644 --- a/crates/mqtt-common/src/metrics.rs +++ b/crates/mqtt-common/src/metrics.rs @@ -76,7 +76,7 @@ impl MqttHookMetrics { fn build_publish_count(meter: &Meter) -> Counter { meter .u64_counter("carbide_dsx_event_bus_publish_count") - .with_description("Total number of MQTT publish attempts") + .with_description("Number of MQTT publish attempts") .build() } diff --git a/crates/network-segment-controller/src/metrics.rs b/crates/network-segment-controller/src/metrics.rs index 88bd3605e8..671f14863c 100644 --- a/crates/network-segment-controller/src/metrics.rs +++ b/crates/network-segment-controller/src/metrics.rs @@ -61,7 +61,7 @@ impl MetricsEmitter for NetworkSegmentMetricsEmitter { let metrics = shared_metrics.clone(); meter .u64_observable_gauge("carbide_available_ips_count") - .with_description("The total number of available ips in the site") + .with_description("Number of available IPs per network segment") .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { for (_seg_id, seg_stats) in metrics.seg_stats.clone() { @@ -87,7 +87,7 @@ impl MetricsEmitter for NetworkSegmentMetricsEmitter { let metrics = shared_metrics.clone(); meter .u64_observable_gauge("carbide_reserved_ips_count") - .with_description("The total number of reserved ips in the site") + .with_description("Number of reserved IPs per network segment") .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { for (_seg_id, seg_stats) in metrics.seg_stats.clone() { @@ -113,7 +113,7 @@ impl MetricsEmitter for NetworkSegmentMetricsEmitter { let metrics = shared_metrics; meter .u64_observable_gauge("carbide_total_ips_count") - .with_description("The total number of ips in the site") + .with_description("Number of IPs per network segment") .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { for (_seg_id, seg_stats) in metrics.seg_stats.clone() { diff --git a/crates/nvlink-manager/src/metrics.rs b/crates/nvlink-manager/src/metrics.rs index 7829f0821b..89779d38cb 100644 --- a/crates/nvlink-manager/src/metrics.rs +++ b/crates/nvlink-manager/src/metrics.rs @@ -203,7 +203,7 @@ impl NvlPartitionMonitorInstruments { .u64_observable_gauge( "carbide_nvlink_partition_monitor_machine_status_updates_count", ) - .with_description("Number of machines nvlink_status_observation got updated") + .with_description("Number of machines whose NVLink status observation was updated") .with_callback(move |o| { metrics.if_available(|metrics, attrs| { o.observe(metrics.num_machine_nvl_status_updates as u64, attrs); diff --git a/crates/preingestion-manager/src/fixtures/test_metrics_collector.txt b/crates/preingestion-manager/src/fixtures/test_metrics_collector.txt index b5a7e63429..6a8d3d5d66 100644 --- a/crates/preingestion-manager/src/fixtures/test_metrics_collector.txt +++ b/crates/preingestion-manager/src/fixtures/test_metrics_collector.txt @@ -1,9 +1,9 @@ -# HELP carbide_preingestion_total The amount of known machines currently being evaluated prior to ingestion +# HELP carbide_preingestion_total Number of known machines currently being evaluated prior to ingestion # TYPE carbide_preingestion_total gauge carbide_preingestion_total 20 -# HELP carbide_preingestion_waiting_download The amount of machines that are waiting for firmware downloads on other machines to complete before doing their own +# HELP carbide_preingestion_waiting_download Number of machines that are waiting for firmware downloads on other machines to complete before doing their own # TYPE carbide_preingestion_waiting_download gauge carbide_preingestion_waiting_download 10 -# HELP carbide_preingestion_waiting_installation The amount of machines which have had firmware uploaded to them and are currently in the process of installing that firmware +# HELP carbide_preingestion_waiting_installation Number of machines which have had firmware uploaded to them and are currently in the process of installing that firmware # TYPE carbide_preingestion_waiting_installation gauge carbide_preingestion_waiting_installation 15 diff --git a/crates/preingestion-manager/src/metrics.rs b/crates/preingestion-manager/src/metrics.rs index eac83f9fc8..f2377ba07a 100644 --- a/crates/preingestion-manager/src/metrics.rs +++ b/crates/preingestion-manager/src/metrics.rs @@ -40,7 +40,7 @@ fn hydrate_meter(meter: Meter, shared_metrics: SharedMetricsHolder MetricsEmitter for CommonMetricsEmitter { let metrics = shared_metrics_holder.clone(); meter .u64_observable_gauge(format!("{object_type}_total")) - .with_description(format!("The total number of {object_type} in the system")) + .with_description(format!("Number of {object_type} in the system")) .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { let num_objects = metrics @@ -316,7 +316,7 @@ impl MetricsEmitter for CommonMetricsEmitter { meter .u64_observable_gauge(format!("{object_type}_per_state")) .with_description(format!( - "The number of {object_type} in the system with a given state" + "Number of {object_type} in the system with a given state" )) .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { @@ -343,7 +343,7 @@ impl MetricsEmitter for CommonMetricsEmitter { meter .u64_observable_gauge(format!("{object_type}_per_state_above_sla")) .with_description(format!( - "The number of {object_type} in the system which had been longer in a state than allowed per SLA" + "Number of {object_type} currently in a state longer than the SLA allows" )) .with_callback(move |observer| { metrics.if_available(|metrics, attrs| { @@ -357,7 +357,8 @@ impl MetricsEmitter for CommonMetricsEmitter { KeyValue::new("substate", full_state.substate.to_string()), ], ] - .concat().as_slice(), + .concat() + .as_slice(), ); } }) @@ -372,10 +373,10 @@ impl MetricsEmitter for CommonMetricsEmitter { "{object_type}_with_state_handling_errors_per_state" )) .with_description(format!( - "The number of {object_type} in the system with a given state that failed state handling" + "Number of state-handling errors for {object_type} in a given state" )) .with_callback(move |observer| { - metrics.if_available(|metrics, attrs| { + metrics.if_available(|metrics, attrs| { for (full_state, state_metrics) in metrics.state_metrics.iter() { let mut total_errs = 0; for (error, &count) in state_metrics.handling_errors_per_type.iter() { @@ -386,7 +387,10 @@ impl MetricsEmitter for CommonMetricsEmitter { attrs, &[ KeyValue::new("state", full_state.state.to_string()), - KeyValue::new("substate", full_state.substate.to_string()), + KeyValue::new( + "substate", + full_state.substate.to_string(), + ), KeyValue::new("error", error.to_string()), ], ] @@ -415,13 +419,13 @@ impl MetricsEmitter for CommonMetricsEmitter { let state_entered_counter = meter .u64_counter(format!("{object_type}_state_entered")) .with_description(format!( - "The amount of types that objects of type {object_type} have entered a certain state" + "Number of times objects of type {object_type} have entered a certain state" )) .build(); let state_exited_counter = meter .u64_counter(format!("{object_type}_state_exited")) .with_description(format!( - "The amount of types that objects of type {object_type} have exited a certain state" + "Number of times objects of type {object_type} have exited a certain state" )) .build(); let time_in_state_histogram = meter diff --git a/docs/observability/core_metrics.md b/docs/observability/core_metrics.md index 778167180a..19522a00c3 100644 --- a/docs/observability/core_metrics.md +++ b/docs/observability/core_metrics.md @@ -4,8 +4,8 @@ This file contains a list of metrics exported by NVIDIA Infra Controller (NICo). - - + + @@ -14,157 +14,157 @@ This file contains a list of metrics exported by NVIDIA Infra Controller (NICo). - - - + + + - + - - - - - - - + + + + + + + - - - - + + + + - - + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - + + - - - - + + + + - + - + - - - - - - - - + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - + + + - - + + - - + + - - - - - + + + + + - - + + - - - - - - - - - + + + + + + + + + - + - - - + + + - - + + - - - - + + + + - - - - - - - - + + + + + + + + - - + +
NameTypeDescription
carbide_active_host_firmware_update_countgaugeThe number of host machines in the system currently working on updating their firmware.
carbide_api_db_queries_totalcounterThe amount of database queries that occurred inside a span
carbide_active_host_firmware_update_countgaugeNumber of host machines in the system currently working on updating their firmware.
carbide_api_db_queries_totalcounterNumber of database queries that occurred inside a span
carbide_api_db_span_query_time_millisecondshistogramTotal time the request spent inside a span on database transactions
carbide_api_grpc_server_duration_millisecondshistogramProcessing time for a request on the carbide API server
carbide_api_readygaugeWhether the NICo API is running
carbide_api_tls_connection_success_totalcounterNumber of successful TLS connections
carbide_api_tracing_spans_opengaugeNumber of open logging/tracing spans
carbide_api_vault_request_duration_millisecondshistogramthe duration of outbound vault requests, in milliseconds
carbide_api_vault_requests_attempted_totalcounterThe amount of tls connections that were attempted
carbide_api_vault_requests_failed_totalcounterThe amount of tcp connections that were failures
carbide_api_vault_requests_succeeded_totalcounterThe amount of tls connections that were successful
carbide_api_vault_requests_attempted_totalcounterNumber of vault requests attempted
carbide_api_vault_requests_failed_totalcounterNumber of vault requests that failed
carbide_api_vault_requests_succeeded_totalcounterNumber of vault requests that succeeded
carbide_api_vault_token_time_until_refresh_secondsgaugeThe amount of time, in seconds, until the vault token is required to be refreshed
carbide_api_versiongaugeVersion (git sha, build date, etc) of this service
carbide_authn_client_cert_rejected_totalcounterNumber of client certificates rejected during authentication
carbide_available_ips_countgaugeThe total number of available ips in the site
carbide_available_ips_countgaugeNumber of available IPs per network segment
carbide_client_tcp_connect_attempts_totalcounterNumber of outbound TCP connect attempts across all HTTP connectors
carbide_client_tcp_connect_errors_totalcounterNumber of failed outbound TCP connect attempts across all HTTP connectors
carbide_client_tcp_connect_successes_totalcounterNumber of successful outbound TCP connects across all HTTP connectors
carbide_concurrent_machine_updates_availablegaugeThe number of machines in the system that we will update concurrently.
carbide_db_pool_idle_connsgaugeThe amount of idle connections in the carbide database pool
carbide_db_pool_total_connsgaugeThe amount of total (active + idle) connections in the carbide database pool
carbide_dpu_agent_version_countgaugeThe amount of DPU agents which have reported a certain version.
carbide_dpu_firmware_version_countgaugeThe amount of DPUs which have reported a certain firmware version.
carbide_dpus_healthy_countgaugeThe total number of DPUs in the system that have reported healthy in the last report. Healthy does not imply up - the report from the DPU might be outdated.
carbide_dpus_up_countgaugeThe total number of DPUs in the system that are up. Up means we have received a health report less than 5 minutes ago.
carbide_concurrent_machine_updates_availablegaugeNumber of machines in the system that we will update concurrently.
carbide_db_pool_idle_connsgaugeNumber of idle connections in the carbide database pool
carbide_db_pool_total_connsgaugeNumber of total (active + idle) connections in the carbide database pool
carbide_dpu_agent_version_countgaugeNumber of DPU agents which have reported a certain version.
carbide_dpu_firmware_version_countgaugeNumber of DPUs which have reported a certain firmware version.
carbide_dpus_healthy_countgaugeNumber of DPUs in the system that have reported healthy in the last report. Healthy does not imply up - the report from the DPU might be outdated.
carbide_dpus_up_countgaugeNumber of DPUs in the system that are up. Up means we have received a health report less than 5 minutes ago.
carbide_endpoint_exploration_duration_millisecondshistogramThe time it took to explore an endpoint
carbide_endpoint_exploration_expected_machines_missing_overall_countgaugeThe total number of machines that were expected but not identified
carbide_endpoint_exploration_expected_power_shelves_missing_overall_countgaugeThe total number of power shelves that were expected but not identified
carbide_endpoint_exploration_identified_managed_hosts_overall_countgaugeThe total number of managed hosts identified by expectation
carbide_endpoint_exploration_machines_explored_overall_countgaugeThe total number of machines explored by machine type
carbide_endpoint_exploration_expected_machines_missing_overall_countgaugeNumber of machines that were expected but not identified
carbide_endpoint_exploration_expected_power_shelves_missing_overall_countgaugeNumber of power shelves that were expected but not identified
carbide_endpoint_exploration_identified_managed_hosts_overall_countgaugeNumber of managed hosts identified by expectation
carbide_endpoint_exploration_machines_explored_overall_countgaugeNumber of machines explored, by expectation and machine type
carbide_endpoint_exploration_step_latency_millisecondshistogramThe time it took to perform one endpoint exploration step
carbide_endpoint_exploration_success_countgaugeThe amount of endpoint explorations that have been successful
carbide_endpoint_explorations_countgaugeThe amount of endpoint explorations that have been attempted
carbide_endpoint_exploration_success_countgaugeNumber of endpoint explorations that have been successful
carbide_endpoint_explorations_countgaugeNumber of endpoint explorations that have been attempted
carbide_external_call_duration_millisecondshistogramDuration of outbound calls by backend, operation, and outcome; the _count series, split by outcome, gives the request and error rates.
carbide_gpus_in_use_countgaugeThe total number of GPUs that are actively used by tenants in instances in the NICo deployment
carbide_gpus_total_countgaugeThe total number of GPUs available in the NICo deployment
carbide_gpus_usable_countgaugeThe remaining number of GPUs in the NICo deployment which are available for immediate instance creation
carbide_hosts_by_sku_countgaugeThe amount of hosts by SKU and device type ('unknown' for hosts without SKU)
carbide_hosts_health_overrides_countgaugeThe amount of health overrides that are configured in the site
carbide_hosts_health_status_countgaugeThe total number of Managed Hosts in the system that have reported either a healthy or not healthy status - based on the presence of health probe alerts
carbide_hosts_in_use_countgaugeThe total number of hosts that are actively used by tenants as instances in the NICo deployment
carbide_hosts_unhealthy_by_classification_countgaugeThe amount of objects which are marked with a certain classification due to being unhealthy
carbide_hosts_unhealthy_by_probe_id_countgaugeThe amount of objects which reported a certain Health Probe Alert
carbide_hosts_usable_countgaugeThe remaining number of hosts in the NICo deployment which are available for immediate instance creation
carbide_hosts_with_bios_password_setgaugeThe total number of Hosts in the system that have their BIOS password set.
carbide_ib_monitor_fabrics_countgaugeThe amount of InfiniBand fabrics that are monitored
carbide_ib_monitor_machine_ib_status_updates_countgaugeThe amount of Machines where the infiniband_status_observation got updated
carbide_ib_monitor_machines_with_missing_pkeys_countgaugeThe amount of machines where at least one port is not assigned to the expected pkey on UFM
carbide_ib_monitor_machines_with_unexpected_pkeys_countgaugeThe amount of machines where at least one port is assigned to an unexpected pkey on UFM
carbide_ib_monitor_machines_with_unknown_pkeys_countgaugeThe amount of machines where at least one port is assigned to a pkey value that is not associated with any partition ID
carbide_gpus_in_use_countgaugeNumber of GPUs that are actively used by tenants in instances in the NICo deployment
carbide_gpus_total_countgaugeNumber of GPUs in the NICo deployment
carbide_gpus_usable_countgaugeNumber of remaining GPUs in the NICo deployment which are available for immediate instance creation
carbide_hosts_by_sku_countgaugeNumber of hosts by SKU and device type ('unknown' for hosts without SKU)
carbide_hosts_health_overrides_countgaugeNumber of health overrides that are configured in the site
carbide_hosts_health_status_countgaugeNumber of Managed Hosts in the system that have reported either a healthy or not healthy status - based on the presence of health probe alerts
carbide_hosts_in_use_countgaugeNumber of hosts that are actively used by tenants as instances in the NICo deployment
carbide_hosts_unhealthy_by_classification_countgaugeNumber of objects which are marked with a certain classification due to being unhealthy
carbide_hosts_unhealthy_by_probe_id_countgaugeNumber of objects which reported a certain Health Probe Alert
carbide_hosts_usable_countgaugeNumber of remaining hosts in the NICo deployment which are available for immediate instance creation
carbide_hosts_with_bios_password_setgaugeNumber of hosts in the system that have their BIOS password set.
carbide_ib_monitor_fabrics_countgaugeNumber of InfiniBand fabrics that are monitored
carbide_ib_monitor_machine_ib_status_updates_countgaugeNumber of machines whose InfiniBand status observation was updated
carbide_ib_monitor_machines_with_missing_pkeys_countgaugeNumber of machines where at least one port is not assigned to the expected pkey on UFM
carbide_ib_monitor_machines_with_unexpected_pkeys_countgaugeNumber of machines where at least one port is assigned to an unexpected pkey on UFM
carbide_ib_monitor_machines_with_unknown_pkeys_countgaugeNumber of machines where at least one port is assigned to a pkey value that is not associated with any partition ID
carbide_ib_partitions_enqueuer_iteration_latency_millisecondshistogramThe overall time it took to enqueue state handling tasks for all carbide_ib_partitions in the system
carbide_ib_partitions_iteration_latency_millisecondshistogramThe elapsed time in the last state processor iteration to handle objects of type carbide_ib_partitions
carbide_ib_partitions_object_tasks_enqueued_totalcounterThe amount of types that object handling tasks that have been freshly enqueued for objects of type carbide_ib_partitions
carbide_ib_partitions_totalgaugeThe total number of carbide_ib_partitions in the system
carbide_ib_partitions_object_tasks_enqueued_totalcounterNumber of object handling tasks that have been freshly enqueued for objects of type carbide_ib_partitions
carbide_ib_partitions_totalgaugeNumber of carbide_ib_partitions in the system
carbide_log_events_totalcounterNumber of log events emitted, by level and component. The always-on log-volume and error-rate signal for every binary.
carbide_machine_reboot_duration_secondshistogramTime taken for machine/host to reboot in seconds
carbide_machine_updates_started_countgaugeThe number of machines in the system that are in the process of updating.
carbide_machine_validation_completedgaugeCount of machine validation that have completed successfully
carbide_machine_validation_failedgaugeCount of machine validation that have failed
carbide_machine_validation_in_progressgaugeCount of machine validation that are in progress
carbide_machine_updates_started_countgaugeNumber of machines in the system that are in the process of updating.
carbide_machine_validation_completedgaugeNumber of machine validation runs that have completed successfully
carbide_machine_validation_failedgaugeNumber of machine validation runs that have failed
carbide_machine_validation_in_progressgaugeNumber of machine validation runs that are in progress
carbide_machine_validation_oldest_active_age_secondsgaugeAge in seconds of the oldest active machine validation run
carbide_machine_validation_stale_runs_countgaugeCount of active machine validation runs considered stale
carbide_machine_validation_stale_runs_countgaugeNumber of active machine validation runs considered stale
carbide_machine_validation_testsgaugeThe details of machine validation tests
carbide_machines_enqueuer_iteration_latency_millisecondshistogramThe overall time it took to enqueue state handling tasks for all carbide_machines in the system
carbide_machines_handler_latency_in_state_millisecondshistogramThe amount of time it took to invoke the state handler for objects of type carbide_machines in a certain state
carbide_machines_in_maintenance_countgaugeThe total number of machines in the system that are in maintenance.
carbide_machines_in_maintenance_countgaugeNumber of machines in the system that are in maintenance.
carbide_machines_iteration_latency_millisecondshistogramThe elapsed time in the last state processor iteration to handle objects of type carbide_machines
carbide_machines_object_tasks_completed_totalcounterThe amount of object handling tasks that have been completed for objects of type carbide_machines
carbide_machines_object_tasks_dispatched_totalcounterThe amount of types that object handling tasks that have been dequeued and dispatched for processing for objects of type carbide_machines
carbide_machines_object_tasks_enqueued_totalcounterThe amount of types that object handling tasks that have been freshly enqueued for objects of type carbide_machines
carbide_machines_object_tasks_requeued_totalcounterThe amount of object handling tasks that have been requeued for objects of type carbide_machines
carbide_machines_per_stategaugeThe number of carbide_machines in the system with a given state
carbide_machines_per_state_above_slagaugeThe number of carbide_machines in the system which had been longer in a state than allowed per SLA
carbide_machines_state_entered_totalcounterThe amount of types that objects of type carbide_machines have entered a certain state
carbide_machines_state_exited_totalcounterThe amount of types that objects of type carbide_machines have exited a certain state
carbide_machines_object_tasks_completed_totalcounterNumber of object handling tasks that have been completed for objects of type carbide_machines
carbide_machines_object_tasks_dispatched_totalcounterNumber of object handling tasks that have been dequeued and dispatched for processing for objects of type carbide_machines
carbide_machines_object_tasks_enqueued_totalcounterNumber of object handling tasks that have been freshly enqueued for objects of type carbide_machines
carbide_machines_object_tasks_requeued_totalcounterNumber of object handling tasks that have been requeued for objects of type carbide_machines
carbide_machines_per_stategaugeNumber of carbide_machines in the system with a given state
carbide_machines_per_state_above_slagaugeNumber of carbide_machines currently in a state longer than the SLA allows
carbide_machines_state_entered_totalcounterNumber of times objects of type carbide_machines have entered a certain state
carbide_machines_state_exited_totalcounterNumber of times objects of type carbide_machines have exited a certain state
carbide_machines_time_in_state_secondshistogramThe amount of time objects of type carbide_machines have spent in a certain state
carbide_machines_totalgaugeThe total number of carbide_machines in the system
carbide_machines_with_state_handling_errors_per_stategaugeThe number of carbide_machines in the system with a given state that failed state handling
carbide_measured_boot_bundles_totalgaugeThe total number of measured boot bundles.
carbide_measured_boot_machines_per_bundle_state_totalgaugeThe total number of machines per a given measured boot bundle state.
carbide_measured_boot_machines_per_machine_state_totalgaugeThe total number of machines per a given measured boot machine state.
carbide_measured_boot_machines_totalgaugeThe total number of machines reporting measurements.
carbide_measured_boot_profiles_totalgaugeThe total number of measured boot profiles.
carbide_machines_totalgaugeNumber of carbide_machines in the system
carbide_machines_with_state_handling_errors_per_stategaugeNumber of state-handling errors for carbide_machines in a given state
carbide_measured_boot_bundles_totalgaugeNumber of measured boot bundles.
carbide_measured_boot_machines_per_bundle_state_totalgaugeNumber of machines in each measured boot bundle state.
carbide_measured_boot_machines_per_machine_state_totalgaugeNumber of machines in each measured boot machine state.
carbide_measured_boot_machines_totalgaugeNumber of machines reporting measurements.
carbide_measured_boot_profiles_totalgaugeNumber of measured boot profiles.
carbide_network_segments_enqueuer_iteration_latency_millisecondshistogramThe overall time it took to enqueue state handling tasks for all carbide_network_segments in the system
carbide_network_segments_handler_latency_in_state_millisecondshistogramThe amount of time it took to invoke the state handler for objects of type carbide_network_segments in a certain state
carbide_network_segments_iteration_latency_millisecondshistogramThe elapsed time in the last state processor iteration to handle objects of type carbide_network_segments
carbide_network_segments_object_tasks_completed_totalcounterThe amount of object handling tasks that have been completed for objects of type carbide_network_segments
carbide_network_segments_object_tasks_dispatched_totalcounterThe amount of types that object handling tasks that have been dequeued and dispatched for processing for objects of type carbide_network_segments
carbide_network_segments_object_tasks_enqueued_totalcounterThe amount of types that object handling tasks that have been freshly enqueued for objects of type carbide_network_segments
carbide_network_segments_object_tasks_requeued_totalcounterThe amount of object handling tasks that have been requeued for objects of type carbide_network_segments
carbide_network_segments_per_stategaugeThe number of carbide_network_segments in the system with a given state
carbide_network_segments_per_state_above_slagaugeThe number of carbide_network_segments in the system which had been longer in a state than allowed per SLA
carbide_network_segments_state_entered_totalcounterThe amount of types that objects of type carbide_network_segments have entered a certain state
carbide_network_segments_state_exited_totalcounterThe amount of types that objects of type carbide_network_segments have exited a certain state
carbide_network_segments_object_tasks_completed_totalcounterNumber of object handling tasks that have been completed for objects of type carbide_network_segments
carbide_network_segments_object_tasks_dispatched_totalcounterNumber of object handling tasks that have been dequeued and dispatched for processing for objects of type carbide_network_segments
carbide_network_segments_object_tasks_enqueued_totalcounterNumber of object handling tasks that have been freshly enqueued for objects of type carbide_network_segments
carbide_network_segments_object_tasks_requeued_totalcounterNumber of object handling tasks that have been requeued for objects of type carbide_network_segments
carbide_network_segments_per_stategaugeNumber of carbide_network_segments in the system with a given state
carbide_network_segments_per_state_above_slagaugeNumber of carbide_network_segments currently in a state longer than the SLA allows
carbide_network_segments_state_entered_totalcounterNumber of times objects of type carbide_network_segments have entered a certain state
carbide_network_segments_state_exited_totalcounterNumber of times objects of type carbide_network_segments have exited a certain state
carbide_network_segments_time_in_state_secondshistogramThe amount of time objects of type carbide_network_segments have spent in a certain state
carbide_network_segments_totalgaugeThe total number of carbide_network_segments in the system
carbide_network_segments_with_state_handling_errors_per_stategaugeThe number of carbide_network_segments in the system with a given state that failed state handling
carbide_nvlink_partition_monitor_machine_status_updates_countgaugeNumber of machines nvlink_status_observation got updated
carbide_network_segments_totalgaugeNumber of carbide_network_segments in the system
carbide_network_segments_with_state_handling_errors_per_stategaugeNumber of state-handling errors for carbide_network_segments in a given state
carbide_nvlink_partition_monitor_machine_status_updates_countgaugeNumber of machines whose NVLink status observation was updated
carbide_nvlink_partition_monitor_nmxc_changes_applied_totalcounterNumber of changes requested to NMX-C
carbide_nvlink_partition_monitor_num_logical_partitionsgaugeNumber of logical partitions that were monitored
carbide_nvlink_partition_monitor_num_physical_partitionsgaugeNumber of physical partitions that were monitored
carbide_nvlink_partition_monitor_nvlink_info_mismatchesgaugeNumber of NVLink GPU partition ID mismatches between DB and NMX-C
carbide_nvlink_partition_monitor_stale_partitions_deletedgaugeNumber of stale partitions deleted from DB (not found in NMX-C)
carbide_pending_dpu_nic_firmware_update_countgaugeThe number of machines in the system that need a firmware update.
carbide_pending_host_firmware_update_countgaugeThe number of host machines in the system that need a firmware update.
carbide_pending_dpu_nic_firmware_update_countgaugeNumber of machines in the system that need a firmware update.
carbide_pending_host_firmware_update_countgaugeNumber of host machines in the system that need a firmware update.
carbide_power_shelves_enqueuer_iteration_latency_millisecondshistogramThe overall time it took to enqueue state handling tasks for all carbide_power_shelves in the system
carbide_power_shelves_health_overrides_countgaugeThe amount of health overrides that are configured in the site
carbide_power_shelves_health_status_countgaugeThe total number of Power Shelves in the system that have reported either a healthy or not healthy status - based on the presence of health probe alerts
carbide_power_shelves_health_overrides_countgaugeNumber of health overrides that are configured in the site
carbide_power_shelves_health_status_countgaugeNumber of Power Shelves in the system that have reported either a healthy or not healthy status - based on the presence of health probe alerts
carbide_power_shelves_iteration_latency_millisecondshistogramThe elapsed time in the last state processor iteration to handle objects of type carbide_power_shelves
carbide_power_shelves_object_tasks_enqueued_totalcounterThe amount of types that object handling tasks that have been freshly enqueued for objects of type carbide_power_shelves
carbide_power_shelves_totalgaugeThe total number of carbide_power_shelves in the system
carbide_preingestion_totalgaugeThe amount of known machines currently being evaluated prior to ingestion
carbide_preingestion_waiting_downloadgaugeThe amount of machines that are waiting for firmware downloads on other machines to complete before doing their own
carbide_preingestion_waiting_installationgaugeThe amount of machines which have had firmware uploaded to them and are currently in the process of installing that firmware
carbide_power_shelves_object_tasks_enqueued_totalcounterNumber of object handling tasks that have been freshly enqueued for objects of type carbide_power_shelves
carbide_power_shelves_totalgaugeNumber of carbide_power_shelves in the system
carbide_preingestion_totalgaugeNumber of known machines currently being evaluated prior to ingestion
carbide_preingestion_waiting_downloadgaugeNumber of machines that are waiting for firmware downloads on other machines to complete before doing their own
carbide_preingestion_waiting_installationgaugeNumber of machines which have had firmware uploaded to them and are currently in the process of installing that firmware
carbide_racks_enqueuer_iteration_latency_millisecondshistogramThe overall time it took to enqueue state handling tasks for all carbide_racks in the system
carbide_racks_health_overrides_countgaugeThe amount of health overrides that are configured in the site
carbide_racks_health_status_countgaugeThe total number of Racks in the system that have reported either a healthy or not healthy status - based on the presence of health probe alerts
carbide_racks_health_overrides_countgaugeNumber of health overrides that are configured in the site
carbide_racks_health_status_countgaugeNumber of Racks in the system that have reported either a healthy or not healthy status - based on the presence of health probe alerts
carbide_racks_iteration_latency_millisecondshistogramThe elapsed time in the last state processor iteration to handle objects of type carbide_racks
carbide_racks_object_tasks_enqueued_totalcounterThe amount of types that object handling tasks that have been freshly enqueued for objects of type carbide_racks
carbide_racks_totalgaugeThe total number of carbide_racks in the system
carbide_reboot_attempts_in_booting_with_discovery_imagehistogramThe amount of machines rebooted again in BootingWithDiscoveryImage since there is no response after a certain time from host.
carbide_reserved_ips_countgaugeThe total number of reserved ips in the site
carbide_resourcepool_free_countgaugeCount of values in the pool currently available for allocation
carbide_resourcepool_used_countgaugeCount of values in the pool currently allocated
carbide_running_dpu_updates_countgaugeThe number of machines in the system that are running a firmware update.
carbide_site_exploration_expected_machines_sku_countgaugeThe total count of expected machines by SKU ID and device type
carbide_site_exploration_identified_managed_hosts_countgaugeThe amount of Host+DPU pairs that has been identified in the last SiteExplorer run
carbide_racks_object_tasks_enqueued_totalcounterNumber of object handling tasks that have been freshly enqueued for objects of type carbide_racks
carbide_racks_totalgaugeNumber of carbide_racks in the system
carbide_reboot_attempts_in_booting_with_discovery_imagehistogramReboot attempts per machine in BootingWithDiscoveryImage, recorded when a machine is rebooted again after no response from the host.
carbide_reserved_ips_countgaugeNumber of reserved IPs per network segment
carbide_resourcepool_free_countgaugeNumber of values in the pool currently available for allocation
carbide_resourcepool_used_countgaugeNumber of values in the pool currently allocated
carbide_running_dpu_updates_countgaugeNumber of machines in the system that are running a firmware update.
carbide_site_exploration_expected_machines_sku_countgaugeNumber of expected machines by SKU ID and device type
carbide_site_exploration_identified_managed_hosts_countgaugeNumber of Host+DPU pairs that have been identified in the last SiteExplorer run
carbide_site_explorer_bmc_password_rotations_totalcounterNumber of BMC root password rotations onto the site-wide credential, by outcome
carbide_site_explorer_bmc_reset_countgaugeThe amount of BMC resets initiated in the last SiteExplorer run
carbide_site_explorer_bmc_reset_countgaugeNumber of BMC resets initiated in the last SiteExplorer run
carbide_site_explorer_create_machinesgaugeWhether site-explorer machine creation is enabled (1) or disabled (0)
carbide_site_explorer_create_machines_latency_millisecondshistogramThe time it took to perform create_machines inside site-explorer
carbide_site_explorer_created_machines_countgaugeThe amount of Machine pairs that had been created by Site Explorer after being identified
carbide_site_explorer_created_power_shelves_countgaugeThe amount of Power Shelves that had been created by Site Explorer after being identified
carbide_site_explorer_dpu_migration_signals_countgaugeCount of DPU NIC-mode migration signals by kind -- mode-mismatch found, set_nic_mode issued, reset requested, and zero-DPU registered for a NicMode host.
carbide_site_explorer_created_machines_countgaugeNumber of machine pairs created by Site Explorer after identification
carbide_site_explorer_created_power_shelves_countgaugeNumber of power shelves created by Site Explorer after identification
carbide_site_explorer_dpu_migration_signals_countgaugeNumber of DPU NIC-mode migration signals by signal type -- mode-mismatch found, set_nic_mode issued, reset requested, and zero-DPU registered for a NicMode host.
carbide_site_explorer_enabledgaugeWhether site-explorer is enabled (1) or paused (0)
carbide_site_explorer_iteration_latency_millisecondshistogramThe time it took to perform one site explorer iteration
carbide_site_explorer_last_run_statusgaugeThe status of the latest Site Explorer run
carbide_site_explorer_phase_latency_millisecondshistogramThe time it took to perform one site explorer iteration phase
carbide_site_explorer_update_explored_endpoints_countgaugeCounts from the last update_explored_endpoints phase by kind
carbide_switches_enqueuer_iteration_latency_millisecondshistogramThe overall time it took to enqueue state handling tasks for all carbide_switches in the system
carbide_switches_health_overrides_countgaugeThe amount of health overrides that are configured in the site
carbide_switches_health_status_countgaugeThe total number of Switches in the system that have reported either a healthy or not healthy status - based on the presence of health probe alerts
carbide_switches_health_overrides_countgaugeNumber of health overrides that are configured in the site
carbide_switches_health_status_countgaugeNumber of Switches in the system that have reported either a healthy or not healthy status - based on the presence of health probe alerts
carbide_switches_iteration_latency_millisecondshistogramThe elapsed time in the last state processor iteration to handle objects of type carbide_switches
carbide_switches_object_tasks_enqueued_totalcounterThe amount of types that object handling tasks that have been freshly enqueued for objects of type carbide_switches
carbide_switches_totalgaugeThe total number of carbide_switches in the system
carbide_total_ips_countgaugeThe total number of ips in the site
carbide_unavailable_dpu_nic_firmware_update_countgaugeThe number of machines in the system that need a firmware update but are unavailable for update.
carbide_switches_object_tasks_enqueued_totalcounterNumber of object handling tasks that have been freshly enqueued for objects of type carbide_switches
carbide_switches_totalgaugeNumber of carbide_switches in the system
carbide_total_ips_countgaugeNumber of IPs per network segment
carbide_unavailable_dpu_nic_firmware_update_countgaugeNumber of machines in the system that need a firmware update but are unavailable for update.
carbide_vpc_prefixes_enqueuer_iteration_latency_millisecondshistogramThe overall time it took to enqueue state handling tasks for all carbide_vpc_prefixes in the system
carbide_vpc_prefixes_handler_latency_in_state_millisecondshistogramThe amount of time it took to invoke the state handler for objects of type carbide_vpc_prefixes in a certain state
carbide_vpc_prefixes_iteration_latency_millisecondshistogramThe elapsed time in the last state processor iteration to handle objects of type carbide_vpc_prefixes
carbide_vpc_prefixes_object_tasks_completed_totalcounterThe amount of object handling tasks that have been completed for objects of type carbide_vpc_prefixes
carbide_vpc_prefixes_object_tasks_dispatched_totalcounterThe amount of types that object handling tasks that have been dequeued and dispatched for processing for objects of type carbide_vpc_prefixes
carbide_vpc_prefixes_object_tasks_enqueued_totalcounterThe amount of types that object handling tasks that have been freshly enqueued for objects of type carbide_vpc_prefixes
carbide_vpc_prefixes_object_tasks_requeued_totalcounterThe amount of object handling tasks that have been requeued for objects of type carbide_vpc_prefixes
carbide_vpc_prefixes_per_stategaugeThe number of carbide_vpc_prefixes in the system with a given state
carbide_vpc_prefixes_per_state_above_slagaugeThe number of carbide_vpc_prefixes in the system which had been longer in a state than allowed per SLA
carbide_vpc_prefixes_state_entered_totalcounterThe amount of types that objects of type carbide_vpc_prefixes have entered a certain state
carbide_vpc_prefixes_state_exited_totalcounterThe amount of types that objects of type carbide_vpc_prefixes have exited a certain state
carbide_vpc_prefixes_object_tasks_completed_totalcounterNumber of object handling tasks that have been completed for objects of type carbide_vpc_prefixes
carbide_vpc_prefixes_object_tasks_dispatched_totalcounterNumber of object handling tasks that have been dequeued and dispatched for processing for objects of type carbide_vpc_prefixes
carbide_vpc_prefixes_object_tasks_enqueued_totalcounterNumber of object handling tasks that have been freshly enqueued for objects of type carbide_vpc_prefixes
carbide_vpc_prefixes_object_tasks_requeued_totalcounterNumber of object handling tasks that have been requeued for objects of type carbide_vpc_prefixes
carbide_vpc_prefixes_per_stategaugeNumber of carbide_vpc_prefixes in the system with a given state
carbide_vpc_prefixes_per_state_above_slagaugeNumber of carbide_vpc_prefixes currently in a state longer than the SLA allows
carbide_vpc_prefixes_state_entered_totalcounterNumber of times objects of type carbide_vpc_prefixes have entered a certain state
carbide_vpc_prefixes_state_exited_totalcounterNumber of times objects of type carbide_vpc_prefixes have exited a certain state
carbide_vpc_prefixes_time_in_state_secondshistogramThe amount of time objects of type carbide_vpc_prefixes have spent in a certain state
carbide_vpc_prefixes_totalgaugeThe total number of carbide_vpc_prefixes in the system
carbide_vpc_prefixes_with_state_handling_errors_per_stategaugeThe number of carbide_vpc_prefixes in the system with a given state that failed state handling
carbide_vpc_prefixes_totalgaugeNumber of carbide_vpc_prefixes in the system
carbide_vpc_prefixes_with_state_handling_errors_per_stategaugeNumber of state-handling errors for carbide_vpc_prefixes in a given state