From 83a24df6612fa9258764a61bdc64c5c4cdef7639 Mon Sep 17 00:00:00 2001 From: mike-realuptime Date: Wed, 2 Sep 2026 04:56:36 +0000 Subject: [PATCH 1/4] Add OpenTelemetry-native host and process monitoring --- .github/workflows/skywalking.yaml | 3 + .../otel-rules/process-hostmetrics-linux.yaml | 81 +++ .../process-hostmetrics-windows.yaml | 81 +++ .../src/main/resources/otel-rules/vm.yaml | 250 +++++++-- .../main/resources/otel-rules/windows.yaml | 227 ++++++-- .../vm/otel-hostmetrics/Dockerfile.otelcol | 27 + .../vm/otel-hostmetrics/docker-compose.yml | 53 ++ .../e2e-v2/cases/vm/otel-hostmetrics/e2e.yaml | 66 +++ .../cases/vm/otel-hostmetrics/entrypoint.sh | 25 + .../expected/process-count-3.yml | 30 ++ .../otel-collector-config.yaml | 483 ++++++++++++++++++ .../otel-rules/process-hostmetrics-linux.yaml | 81 +++ .../process-hostmetrics-windows.yaml | 81 +++ .../vm/otel-hostmetrics/otel-rules/vm.yaml | 257 ++++++++++ .../otel-hostmetrics/otel-rules/windows.yaml | 225 ++++++++ .../otel-rules/vm.yaml | 257 ++++++++-- 16 files changed, 2097 insertions(+), 130 deletions(-) create mode 100644 oap-server/server-starter/src/main/resources/otel-rules/process-hostmetrics-linux.yaml create mode 100644 oap-server/server-starter/src/main/resources/otel-rules/process-hostmetrics-windows.yaml create mode 100644 test/e2e-v2/cases/vm/otel-hostmetrics/Dockerfile.otelcol create mode 100644 test/e2e-v2/cases/vm/otel-hostmetrics/docker-compose.yml create mode 100644 test/e2e-v2/cases/vm/otel-hostmetrics/e2e.yaml create mode 100644 test/e2e-v2/cases/vm/otel-hostmetrics/entrypoint.sh create mode 100644 test/e2e-v2/cases/vm/otel-hostmetrics/expected/process-count-3.yml create mode 100644 test/e2e-v2/cases/vm/otel-hostmetrics/otel-collector-config.yaml create mode 100644 test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/process-hostmetrics-linux.yaml create mode 100644 test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/process-hostmetrics-windows.yaml create mode 100644 test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/vm.yaml create mode 100644 test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/windows.yaml diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index c177cc086e3d..23ba06fd879b 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -664,6 +664,9 @@ jobs: config: test/e2e-v2/cases/vm/zabbix/e2e.yaml - name: VM Prometheus config: test/e2e-v2/cases/vm/prometheus-node-exporter/e2e.yaml + - name: VM OpenTelemetry Hostmetrics + config: test/e2e-v2/cases/vm/otel-hostmetrics/e2e.yaml + env: OTEL_COLLECTOR_CONTRIB_VERSION=0.158.0 - name: VM Telegraf config: test/e2e-v2/cases/vm/telegraf/e2e.yaml - name: So11y diff --git a/oap-server/server-starter/src/main/resources/otel-rules/process-hostmetrics-linux.yaml b/oap-server/server-starter/src/main/resources/otel-rules/process-hostmetrics-linux.yaml new file mode 100644 index 000000000000..61912ca9807f --- /dev/null +++ b/oap-server/server-starter/src/main/resources/otel-rules/process-hostmetrics-linux.yaml @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Process metrics are pre-aggregated by normalized process_name in the +# Collector to bound PID cardinality. MAL performs the final SkyWalking +# service-instance mapping and metric calculation. +filter: "{ tags -> tags.job_name == 'hostmetrics-process-monitoring-linux' }" + +expSuffix: |- + service(['node_identifier_host_name'], '', Layer.OS_LINUX) + .instance( + ['node_identifier_host_name'], + '', + ['process_name'], + '', + Layer.OS_LINUX, + { + tags -> [ + 'process_name': tags.process_name, + 'host': tags.node_identifier_host_name + ] + } + ) + +metricPrefix: mp_process_linux + +metricsRules: + - name: num_procs + exp: "process_count.sum(['node_identifier_host_name','process_name'])" + + - name: num_threads + exp: "process_threads.sum(['node_identifier_host_name','process_name'])" + + - name: cpu_total_percent + exp: > + process_cpu_utilization + .tagNotEqual('state','wait') + .sum(['node_identifier_host_name','process_name']) + * 100 + + - name: cpu_user_percent + exp: > + process_cpu_utilization + .tagEqual('state','user') + .sum(['node_identifier_host_name','process_name']) + * 100 + + - name: cpu_system_percent + exp: > + process_cpu_utilization + .tagEqual('state','system') + .sum(['node_identifier_host_name','process_name']) + * 100 + + - name: memory_resident_bytes + exp: "process_memory_usage.sum(['node_identifier_host_name','process_name'])" + + # OTel process.memory.utilization uses unit 1 (fraction), so expose % here. + - name: memory_resident_percent + exp: > + process_memory_utilization + .sum(['node_identifier_host_name','process_name']) + * 100 + + - name: open_handles + exp: "process_open_handles.sum(['node_identifier_host_name','process_name'])" + + - name: oldest_process_uptime_seconds + exp: "process_uptime.max(['node_identifier_host_name','process_name'])" diff --git a/oap-server/server-starter/src/main/resources/otel-rules/process-hostmetrics-windows.yaml b/oap-server/server-starter/src/main/resources/otel-rules/process-hostmetrics-windows.yaml new file mode 100644 index 000000000000..c2f299f779d6 --- /dev/null +++ b/oap-server/server-starter/src/main/resources/otel-rules/process-hostmetrics-windows.yaml @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Process metrics are pre-aggregated by normalized process_name in the +# Collector to bound PID cardinality. MAL performs the final SkyWalking +# service-instance mapping and metric calculation. +filter: "{ tags -> tags.job_name == 'hostmetrics-process-monitoring-windows' }" + +expSuffix: |- + service(['node_identifier_host_name'], '', Layer.OS_WINDOWS) + .instance( + ['node_identifier_host_name'], + '', + ['process_name'], + '', + Layer.OS_WINDOWS, + { + tags -> [ + 'process_name': tags.process_name, + 'host': tags.node_identifier_host_name + ] + } + ) + +metricPrefix: mp_process_windows + +metricsRules: + - name: num_procs + exp: "process_count.sum(['node_identifier_host_name','process_name'])" + + - name: num_threads + exp: "process_threads.sum(['node_identifier_host_name','process_name'])" + + - name: cpu_total_percent + exp: > + process_cpu_utilization + .tagNotEqual('state','wait') + .sum(['node_identifier_host_name','process_name']) + * 100 + + - name: cpu_user_percent + exp: > + process_cpu_utilization + .tagEqual('state','user') + .sum(['node_identifier_host_name','process_name']) + * 100 + + - name: cpu_system_percent + exp: > + process_cpu_utilization + .tagEqual('state','system') + .sum(['node_identifier_host_name','process_name']) + * 100 + + - name: memory_resident_bytes + exp: "process_memory_usage.sum(['node_identifier_host_name','process_name'])" + + # OTel process.memory.utilization uses unit 1 (fraction), so expose % here. + - name: memory_resident_percent + exp: > + process_memory_utilization + .sum(['node_identifier_host_name','process_name']) + * 100 + + - name: open_handles + exp: "process_open_handles.sum(['node_identifier_host_name','process_name'])" + + - name: oldest_process_uptime_seconds + exp: "process_uptime.max(['node_identifier_host_name','process_name'])" diff --git a/oap-server/server-starter/src/main/resources/otel-rules/vm.yaml b/oap-server/server-starter/src/main/resources/otel-rules/vm.yaml index 4937251af5f5..0397e01d4c1f 100644 --- a/oap-server/server-starter/src/main/resources/otel-rules/vm.yaml +++ b/oap-server/server-starter/src/main/resources/otel-rules/vm.yaml @@ -1,4 +1,3 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 @@ -13,85 +12,246 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This will parse a textual representation of a duration. The formats -# accepted are based on the ISO-8601 duration format {@code PnDTnHnMn.nS} -# with days considered to be exactly 24 hours. -#

-# Examples: -#

-#    "PT20.345S" -- parses as "20.345 seconds"
-#    "PT15M"     -- parses as "15 minutes" (where a minute is 60 seconds)
-#    "PT10H"     -- parses as "10 hours" (where an hour is 3600 seconds)
-#    "P2D"       -- parses as "2 days" (where a day is 24 hours or 86400 seconds)
-#    "P2DT3H4M"  -- parses as "2 days, 3 hours and 4 minutes"
-#    "P-6H3M"    -- parses as "-6 hours and +3 minutes"
-#    "-P6H3M"    -- parses as "-6 hours and -3 minutes"
-#    "-P-6H+3M"  -- parses as "+6 hours and -3 minutes"
-# 
-filter: "{ tags -> tags.job_name == 'vm-monitoring' }" # The OpenTelemetry job name -expSuffix: service(['node_identifier_host_name'] , Layer.OS_LINUX) +# Linux infrastructure MAL. +# +# This rule intentionally accepts TWO source metric families under the same +# existing job_name "vm-monitoring": +# 1. node-exporter metrics (existing SkyWalking integration) +# 2. OpenTelemetry hostmetrics metrics (new alternative source) +# +# Both sources are mapped by ONE MAL file to the canonical meter_vm_* targets. +# This avoids cross-file metric ownership collisions in OAP 11 runtime rules. +# +# MAL SampleFamily '+' treats an EMPTY family as identity. Because a host sends +# one source family or the other, each expression can safely add the equivalent +# node-exporter and hostmetrics branches. The output metric is registered once. +filter: "{ tags -> tags.job_name == 'vm-monitoring' }" +expSuffix: service(['node_identifier_host_name'], Layer.OS_LINUX) metricPrefix: meter_vm + metricsRules: + # --------------------------------------------------------------------------- + # Canonical metrics: same target names and semantics for both sources. + # --------------------------------------------------------------------------- - #node cpu + # CPU - name: cpu_total_percentage - exp: (node_cpu_seconds_total * 100).tagNotEqual('mode' , 'idle').sum(['node_identifier_host_name']).rate('PT1M') + exp: > + (node_cpu_seconds_total * 100) + .tagNotEqual('mode','idle') + .sum(['node_identifier_host_name']) + .rate('PT1M') + + + (system_cpu_utilization * 100) + .tagNotEqual('mode','idle') + .sum(['node_identifier_host_name']) + - name: cpu_average_used - exp: (node_cpu_seconds_total * 100).sum(['node_identifier_host_name' , 'mode']).rate('PT1M') + exp: > + (node_cpu_seconds_total * 100) + .sum(['node_identifier_host_name','mode']) + .rate('PT1M') + + + (system_cpu_utilization * 100) + .sum(['node_identifier_host_name','mode']) + - name: cpu_load1 - exp: node_load1 * 100 + exp: "(node_load1 * 100) + (system_cpu_load_average_1m * 100)" + - name: cpu_load5 - exp: node_load5 * 100 + exp: "(node_load5 * 100) + (system_cpu_load_average_5m * 100)" + - name: cpu_load15 - exp: node_load15 * 100 + exp: "(node_load15 * 100) + (system_cpu_load_average_15m * 100)" - #node Memory + # Physical memory - name: memory_total - exp: node_memory_MemTotal_bytes + exp: "node_memory_MemTotal_bytes + system_memory_limit" + - name: memory_available - exp: node_memory_MemAvailable_bytes + exp: "node_memory_MemAvailable_bytes + system_linux_memory_available" + - name: memory_used - exp: node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes + exp: > + (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) + + + (system_memory_limit - system_linux_memory_available) + - name: memory_buff_cache - exp: node_memory_Buffers_bytes + node_memory_Cached_bytes + exp: > + (node_memory_Buffers_bytes + node_memory_Cached_bytes) + + + ( + system_memory_usage.tagEqual('state','buffered').sum(['node_identifier_host_name']) + + + system_memory_usage.tagEqual('state','cached').sum(['node_identifier_host_name']) + ) + + # Swap - name: memory_swap_free - exp: node_memory_SwapFree_bytes + exp: > + node_memory_SwapFree_bytes + + + system_paging_usage + .tagEqual('state','free') + .sum(['node_identifier_host_name']) + - name: memory_swap_total - exp: node_memory_SwapTotal_bytes + exp: > + node_memory_SwapTotal_bytes + + + ( + system_paging_usage.tagEqual('state','used').sum(['node_identifier_host_name']) + + + system_paging_usage.tagEqual('state','free').sum(['node_identifier_host_name']) + ) + - name: memory_swap_percentage - exp: 100 - ((node_memory_SwapFree_bytes * 100) / node_memory_SwapTotal_bytes) + exp: > + ( + 100 - ((node_memory_SwapFree_bytes * 100) / node_memory_SwapTotal_bytes) + ) + + + ( + ( + system_paging_usage.tagEqual('state','used').sum(['node_identifier_host_name']) + * 100 + ) + / + ( + system_paging_usage.tagEqual('state','used').sum(['node_identifier_host_name']) + + + system_paging_usage.tagEqual('state','free').sum(['node_identifier_host_name']) + ) + ) - #node filesystem + # Filesystem. hostmetrics free+reserved+used is the full capacity. - name: filesystem_percentage - exp: 100 - ((node_filesystem_avail_bytes * 100).sum(['node_identifier_host_name' , 'mountpoint']) / node_filesystem_size_bytes.sum(['node_identifier_host_name' , 'mountpoint'])) + exp: > + ( + 100 - ( + (node_filesystem_avail_bytes * 100) + .sum(['node_identifier_host_name','mountpoint']) + / + node_filesystem_size_bytes + .sum(['node_identifier_host_name','mountpoint']) + ) + ) + + + ( + 100 - ( + (system_filesystem_usage.tagEqual('state','free') * 100) + .sum(['node_identifier_host_name','mountpoint']) + / + ( + system_filesystem_usage.tagEqual('state','free') + .sum(['node_identifier_host_name','mountpoint']) + + + system_filesystem_usage.tagEqual('state','reserved') + .sum(['node_identifier_host_name','mountpoint']) + + + system_filesystem_usage.tagEqual('state','used') + .sum(['node_identifier_host_name','mountpoint']) + ) + ) + ) - #node disk + # Disk throughput - name: disk_read - exp: node_disk_read_bytes_total.sum(['node_identifier_host_name']).rate('PT1M') + exp: > + node_disk_read_bytes_total + .sum(['node_identifier_host_name']) + .rate('PT1M') + + + system_disk_io + .tagEqual('direction','read') + .sum(['node_identifier_host_name']) + .rate('PT1M') + - name: disk_written - exp: node_disk_written_bytes_total.sum(['node_identifier_host_name']).rate('PT1M') + exp: > + node_disk_written_bytes_total + .sum(['node_identifier_host_name']) + .rate('PT1M') + + + system_disk_io + .tagEqual('direction','write') + .sum(['node_identifier_host_name']) + .rate('PT1M') - #node network + # Network throughput - name: network_receive - exp: node_network_receive_bytes_total.sum(['node_identifier_host_name']).irate() + exp: > + node_network_receive_bytes_total + .sum(['node_identifier_host_name']) + .irate() + + + system_network_io + .tagEqual('direction','receive') + .sum(['node_identifier_host_name']) + .irate() + - name: network_transmit - exp: node_network_transmit_bytes_total.sum(['node_identifier_host_name']).irate() + exp: > + node_network_transmit_bytes_total + .sum(['node_identifier_host_name']) + .irate() + + + system_network_io + .tagEqual('direction','transmit') + .sum(['node_identifier_host_name']) + .irate() - #node netstat + # TCP connection metrics with true hostmetrics equivalents. + # Collector normalizes system.network.connections state values to lowercase. - name: tcp_curr_estab - exp: node_netstat_Tcp_CurrEstab + exp: > + node_netstat_Tcp_CurrEstab + + + system_network_connections + .tagEqual('protocol','tcp') + .tagEqual('state','established') + .sum(['node_identifier_host_name']) + - name: tcp_tw - exp: node_sockstat_TCP_tw + exp: > + node_sockstat_TCP_tw + + + system_network_connections + .tagEqual('protocol','tcp') + .tagEqual('state','time_wait') + .sum(['node_identifier_host_name']) + + # --------------------------------------------------------------------------- + # Existing node-exporter-only canonical metrics. + # hostmetrics has no source with equivalent semantics, so these remain node + # exporter only rather than being populated with a misleading approximation. + # --------------------------------------------------------------------------- - name: tcp_alloc exp: node_sockstat_TCP_alloc + - name: sockets_used exp: node_sockstat_sockets_used + - name: udp_inuse exp: node_sockstat_UDP_inuse - #node filefd - name: filefd_allocated exp: node_filefd_allocated + # --------------------------------------------------------------------------- + # New hostmetrics-only capabilities. New names are used whenever semantics + # do not exactly match an existing node-exporter target metric. + # --------------------------------------------------------------------------- + - name: cpu_cores_num + exp: system_cpu_logical_count.sum(['node_identifier_host_name']) - + - name: cpu_norm_percentage + exp: > + ( + (system_cpu_utilization * 100) + .tagNotEqual('mode','idle') + .sum(['node_identifier_host_name']) + ) + / + system_cpu_logical_count.sum(['node_identifier_host_name']) diff --git a/oap-server/server-starter/src/main/resources/otel-rules/windows.yaml b/oap-server/server-starter/src/main/resources/otel-rules/windows.yaml index de2a651111d5..2e872e86659a 100644 --- a/oap-server/server-starter/src/main/resources/otel-rules/windows.yaml +++ b/oap-server/server-starter/src/main/resources/otel-rules/windows.yaml @@ -13,60 +13,213 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This will parse a textual representation of a duration. The formats -# accepted are based on the ISO-8601 duration format {@code PnDTnHnMn.nS} -# with days considered to be exactly 24 hours. -#

-# Examples: -#

-#    "PT20.345S" -- parses as "20.345 seconds"
-#    "PT15M"     -- parses as "15 minutes" (where a minute is 60 seconds)
-#    "PT10H"     -- parses as "10 hours" (where an hour is 3600 seconds)
-#    "P2D"       -- parses as "2 days" (where a day is 24 hours or 86400 seconds)
-#    "P2DT3H4M"  -- parses as "2 days, 3 hours and 4 minutes"
-#    "P-6H3M"    -- parses as "-6 hours and +3 minutes"
-#    "-P6H3M"    -- parses as "-6 hours and -3 minutes"
-#    "-P-6H+3M"  -- parses as "+6 hours and -3 minutes"
-# 
-filter: "{ tags -> tags.job_name == 'windows-monitoring' }" # The OpenTelemetry job name -expSuffix: service(['node_identifier_host_name'] , Layer.OS_WINDOWS) +# Windows infrastructure MAL. +# +# The existing windows-exporter source and the OpenTelemetry hostmetrics source +# intentionally use the same existing job_name "windows-monitoring". One MAL +# file maps either input family to the canonical meter_win_* targets, avoiding +# cross-file metric ownership collisions in OAP 11. +filter: "{ tags -> tags.job_name == 'windows-monitoring' }" +expSuffix: service(['node_identifier_host_name'], Layer.OS_WINDOWS) metricPrefix: meter_win + metricsRules: - #cpu windows don't expose cpu load metrics + # --------------------------------------------------------------------------- + # Canonical windows.yaml metrics, now accepting either source. + # --------------------------------------------------------------------------- + + # CPU - name: cpu_total_percentage - exp: (windows_cpu_time_total * 100).tagNotEqual('mode' , 'idle').sum(['node_identifier_host_name']).rate('PT1M') + exp: > + (windows_cpu_time_total * 100) + .tagNotEqual('mode','idle') + .sum(['node_identifier_host_name']) + .rate('PT1M') + + + (system_cpu_time * 100) + .tagNotEqual('mode','idle') + .sum(['node_identifier_host_name']) + .rate('PT1M') + - name: cpu_average_used - exp: (windows_cpu_time_total * 100).sum(['node_identifier_host_name' , 'mode']).rate('PT1M') + exp: > + (windows_cpu_time_total * 100) + .sum(['node_identifier_host_name','mode']) + .rate('PT1M') + + + (system_cpu_time * 100) + .sum(['node_identifier_host_name','mode']) + .rate('PT1M') - #memory + # Physical memory - name: memory_total - exp: windows_cs_physical_memory_bytes + exp: > + windows_cs_physical_memory_bytes + + + system_memory_limit.sum(['node_identifier_host_name']) + - name: memory_available - exp: windows_os_physical_memory_free_bytes + exp: > + windows_os_physical_memory_free_bytes + + + system_memory_usage + .tagEqual('state','free') + .sum(['node_identifier_host_name']) + - name: memory_used - exp: windows_cs_physical_memory_bytes - windows_os_physical_memory_free_bytes + exp: > + (windows_cs_physical_memory_bytes - windows_os_physical_memory_free_bytes) + + + ( + system_memory_limit.sum(['node_identifier_host_name']) + - + system_memory_usage.tagEqual('state','free').sum(['node_identifier_host_name']) + ) + + # Canonical virtual memory semantics. + # The hostmetrics branch uses Windows Performance Counters: + # Memory\\Commit Limit + # Memory\\Committed Bytes - name: memory_virtual_memory_free - exp: windows_os_virtual_memory_free_bytes + exp: > + windows_os_virtual_memory_free_bytes + + + ( + system_virtual_memory_commit_limit.sum(['node_identifier_host_name']) + - + system_virtual_memory_committed.sum(['node_identifier_host_name']) + ) + - name: memory_virtual_memory_total - exp: windows_os_virtual_memory_bytes + exp: > + windows_os_virtual_memory_bytes + + + system_virtual_memory_commit_limit.sum(['node_identifier_host_name']) + - name: memory_virtual_memory_percentage - exp: 100 - ((windows_os_virtual_memory_free_bytes * 100) / windows_os_virtual_memory_bytes) + exp: > + ( + 100 - ((windows_os_virtual_memory_free_bytes * 100) / windows_os_virtual_memory_bytes) + ) + + + ( + system_virtual_memory_committed.sum(['node_identifier_host_name']) * 100 + / + system_virtual_memory_commit_limit.sum(['node_identifier_host_name']) + ) - #disk + # Disk throughput - name: disk_read - exp: windows_logical_disk_read_bytes_total.sum(['node_identifier_host_name']).rate('PT1M') + exp: > + windows_logical_disk_read_bytes_total + .sum(['node_identifier_host_name']) + .rate('PT1M') + + + system_disk_io + .tagEqual('direction','read') + .sum(['node_identifier_host_name']) + .rate('PT1M') + - name: disk_written - exp: windows_logical_disk_write_bytes_total.sum(['node_identifier_host_name']).rate('PT1M') + exp: > + windows_logical_disk_write_bytes_total + .sum(['node_identifier_host_name']) + .rate('PT1M') + + + system_disk_io + .tagEqual('direction','write') + .sum(['node_identifier_host_name']) + .rate('PT1M') - #network + # Network throughput - name: network_receive - exp: windows_net_bytes_received_total.sum(['node_identifier_host_name']).irate() + exp: > + windows_net_bytes_received_total + .sum(['node_identifier_host_name']) + .irate() + + + system_network_io + .tagEqual('direction','receive') + .sum(['node_identifier_host_name']) + .irate() + - name: network_transmit - exp: windows_net_bytes_sent_total.sum(['node_identifier_host_name']).irate() - - - - + exp: > + windows_net_bytes_sent_total + .sum(['node_identifier_host_name']) + .irate() + + + system_network_io + .tagEqual('direction','transmit') + .sum(['node_identifier_host_name']) + .irate() + + # --------------------------------------------------------------------------- + # Additional hostmetrics capabilities. These are additive and do not change + # the existing windows-exporter metric contract. + # --------------------------------------------------------------------------- + - name: cpu_cores_num + exp: system_cpu_logical_count.sum(['node_identifier_host_name']) + + - name: cpu_norm_percentage + exp: > + ( + (system_cpu_time * 100) + .tagNotEqual('mode','idle') + .sum(['node_identifier_host_name']) + .rate('PT1M') + ) + / + system_cpu_logical_count.sum(['node_identifier_host_name']) + + - name: cpu_load1 + exp: system_cpu_load_average_1m.sum(['node_identifier_host_name']) * 100 + + - name: cpu_load5 + exp: system_cpu_load_average_5m.sum(['node_identifier_host_name']) * 100 + + - name: cpu_load15 + exp: system_cpu_load_average_15m.sum(['node_identifier_host_name']) * 100 + + - name: filesystem_percentage + exp: > + 100 - ( + (system_filesystem_usage.tagEqual('state','free') * 100) + .sum(['node_identifier_host_name','mountpoint']) + / + ( + system_filesystem_usage.tagEqual('state','free') + .sum(['node_identifier_host_name','mountpoint']) + + + system_filesystem_usage.tagEqual('state','reserved') + .sum(['node_identifier_host_name','mountpoint']) + + + system_filesystem_usage.tagEqual('state','used') + .sum(['node_identifier_host_name','mountpoint']) + ) + ) + + - name: filehandles_allocated + exp: system_filehandles_allocated.sum(['node_identifier_host_name']) + # system.paging.usage is the Windows pagefile, not the windows.yaml virtual + # memory metric. Keep it under explicit new names. + - name: memory_pagefile_free + exp: > + system_paging_usage + .tagEqual('state','free') + .sum(['node_identifier_host_name']) + - name: memory_pagefile_total + exp: system_paging_usage.sum(['node_identifier_host_name']) + - name: memory_pagefile_percentage + exp: > + ( + system_paging_usage + .tagEqual('state','used') + .sum(['node_identifier_host_name']) + * 100 + ) + / + system_paging_usage.sum(['node_identifier_host_name']) diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/Dockerfile.otelcol b/test/e2e-v2/cases/vm/otel-hostmetrics/Dockerfile.otelcol new file mode 100644 index 000000000000..e13d04d7a708 --- /dev/null +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/Dockerfile.otelcol @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG OTEL_COLLECTOR_CONTRIB_VERSION + +FROM otel/opentelemetry-collector-contrib:${OTEL_COLLECTOR_CONTRIB_VERSION} AS otel + +FROM alpine:3.22 + +COPY --from=otel /otelcol-contrib /otelcol-contrib +COPY entrypoint.sh /entrypoint.sh + +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/docker-compose.yml b/test/e2e-v2/cases/vm/otel-hostmetrics/docker-compose.yml new file mode 100644 index 000000000000..693a4f444242 --- /dev/null +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/docker-compose.yml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + oap: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: oap + environment: + SW_OTEL_RECEIVER: default + SW_OTEL_RECEIVER_ENABLED_OTEL_METRICS_RULES: vm,process-hostmetrics-linux + volumes: + - ./otel-rules/vm.yaml:/skywalking/config/otel-rules/vm.yaml + - ./otel-rules/process-hostmetrics-linux.yaml:/skywalking/config/otel-rules/process-hostmetrics-linux.yaml + ports: + - 12800 + + banyandb: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: banyandb + ports: + - 17912 + + vm-service: + hostname: vm-service + build: + context: . + dockerfile: Dockerfile.otelcol + args: + OTEL_COLLECTOR_CONTRIB_VERSION: ${OTEL_COLLECTOR_CONTRIB_VERSION} + networks: + - e2e + volumes: + - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml + depends_on: + oap: + condition: service_healthy + +networks: + e2e: diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/e2e.yaml b/test/e2e-v2/cases/vm/otel-hostmetrics/e2e.yaml new file mode 100644 index 000000000000..fba61f3fc4f2 --- /dev/null +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/e2e.yaml @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +setup: + env: compose + file: docker-compose.yml + timeout: 20m + init-system-environment: ../../../script/env + steps: + - name: set PATH + command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + +verify: + retry: + count: 30 + interval: 3s + + cases: + # Existing shared VM assertions: + # service exists and canonical memory metrics have values. + - includes: + - ../vm-cases.yaml + + # Canonical metrics produced from native OTel hostmetrics. + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_vm_cpu_total_percentage --service-name=vm-service + expected: ../expected/metrics-has-value.yml + + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_vm_cpu_cores_num --service-name=vm-service + expected: ../expected/metrics-has-value.yml + + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_vm_cpu_norm_percentage --service-name=vm-service + expected: ../expected/metrics-has-value.yml + + # The normalized process name must be usable as a SkyWalking instance. + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=mp_process_linux_num_procs --service-name=vm-service --instance-name=sleep + expected: expected/process-count-3.yml + + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=mp_process_linux_num_threads --service-name=vm-service --instance-name=sleep + expected: ../expected/metrics-has-value.yml + + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=mp_process_linux_memory_resident_bytes --service-name=vm-service --instance-name=sleep + expected: ../expected/metrics-has-value.yml + + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=mp_process_linux_open_handles --service-name=vm-service --instance-name=sleep + expected: ../expected/metrics-has-value.yml + + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=mp_process_linux_oldest_process_uptime_seconds --service-name=vm-service --instance-name=sleep + expected: ../expected/metrics-has-value.yml diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/entrypoint.sh b/test/e2e-v2/cases/vm/otel-hostmetrics/entrypoint.sh new file mode 100644 index 000000000000..004c642e9b68 --- /dev/null +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +sleep 3600 & +sleep 3600 & +sleep 3600 & + +exec /otelcol-contrib \ + --config=/etc/otelcol-contrib/config.yaml diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/expected/process-count-3.yml b/test/e2e-v2/cases/vm/otel-hostmetrics/expected/process-count-3.yml new file mode 100644 index 000000000000..738637d60290 --- /dev/null +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/expected/process-count-3.yml @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +debuggingtrace: null +type: TIME_SERIES_VALUES +results: + {{- contains .results }} + - metric: + labels: [] + values: + {{- contains .values }} + - id: {{ notEmpty .id }} + value: 3 + owner: null + traceid: null + {{- end}} + {{- end}} +error: null diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/otel-collector-config.yaml b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-collector-config.yaml new file mode 100644 index 000000000000..79487395ab83 --- /dev/null +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-collector-config.yaml @@ -0,0 +1,483 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# E2E configuration for OpenTelemetry-native Linux host/process monitoring. +# +# Differences from the production/reference configuration: +# - collection_interval is 5s instead of 30s +# - process allowlist contains only "^sleep$" +# - OTLP endpoint is the Docker Compose service "oap:11800" +# + +receivers: + + # --------------------------------------------------------------------------- + # Host metrics + # --------------------------------------------------------------------------- + hostmetrics/system: + collection_interval: 5s + + scrapers: + + cpu: + metrics: + system.cpu.utilization: + enabled: true + + system.cpu.logical.count: + enabled: true + + memory: + metrics: + system.linux.memory.available: + enabled: true + + system.memory.limit: + enabled: true + + disk: {} + + network: {} + + filesystem: {} + + paging: {} + + load: {} + + + # --------------------------------------------------------------------------- + # Process metrics + # + # The E2E workload starts exactly three "sleep" processes. + # Only those processes are collected. + # --------------------------------------------------------------------------- + hostmetrics/processes: + collection_interval: 5s + + scrapers: + + process: + + include: + match_type: regexp + names: + - "^sleep$" + + mute_process_all_errors: true + + metrics: + + process.cpu.time: + enabled: false + + process.cpu.utilization: + enabled: true + + process.memory.usage: + enabled: true + + process.memory.virtual: + enabled: false + + process.memory.utilization: + enabled: true + + process.threads: + enabled: true + + process.uptime: + enabled: true + + process.open_file_descriptors: + enabled: true + + process.context_switches: + enabled: false + + process.paging.faults: + enabled: false + + process.signals_pending: + enabled: false + + process.disk.io: + enabled: false + + process.disk.operations: + enabled: false + + resource_attributes: + + # This is the only process identity attribute required. + process.executable.name: + enabled: true + + process.executable.path: + enabled: false + + process.command: + enabled: false + + process.command_line: + enabled: false + + process.owner: + enabled: false + + process.pid: + enabled: false + + process.parent_pid: + enabled: false + + +processors: + + # --------------------------------------------------------------------------- + # Detect host.name. + # + # Because docker-compose.yml sets: + # + # hostname: vm-service + # + # this should produce: + # + # host.name = vm-service + # + # --------------------------------------------------------------------------- + resource_detection/system: + detectors: + - system + + override: true + + system: + hostname_sources: + - os + + + # --------------------------------------------------------------------------- + # Host identity / routing + # + # Makes hostmetrics use the same canonical SkyWalking VM model as the + # existing node-exporter integration: + # + # node_identifier_host_name = host.name + # job_name = vm-monitoring + # + # CPU state is renamed to "mode" to preserve the canonical vm.yaml shape. + # --------------------------------------------------------------------------- + transform/host-linux: + error_mode: ignore + + metric_statements: + + - context: resource + statements: + + - set( + resource.attributes["node_identifier_host_name"], + resource.attributes["host.name"] + ) where resource.attributes["host.name"] != nil + + - set( + resource.attributes["job_name"], + "vm-monitoring" + ) + + - context: datapoint + statements: + + - set( + datapoint.attributes["mode"], + datapoint.attributes["state"] + ) where metric.name == "system.cpu.utilization" and datapoint.attributes["state"] != nil + + - delete_key( + datapoint.attributes, + "state" + ) where metric.name == "system.cpu.utilization" + + - set( + datapoint.attributes["protocol"], + ConvertCase(datapoint.attributes["protocol"], "lower") + ) where metric.name == "system.network.connections" and datapoint.attributes["protocol"] != nil + + - set( + datapoint.attributes["state"], + ConvertCase(datapoint.attributes["state"], "lower") + ) where metric.name == "system.network.connections" and datapoint.attributes["state"] != nil + + + # --------------------------------------------------------------------------- + # Process identity + # + # Converts: + # + # process.executable.name = sleep + # + # into: + # + # node_identifier_host_name = vm-service + # job_name = hostmetrics-process-monitoring-linux + # process_name = sleep + # + # PID is deliberately NOT part of the identity. + # --------------------------------------------------------------------------- + transform/process-identity-linux: + error_mode: ignore + + metric_statements: + + - context: resource + statements: + + - set( + resource.attributes["node_identifier_host_name"], + resource.attributes["host.name"] + ) where resource.attributes["host.name"] != nil + + - set( + resource.attributes["job_name"], + "hostmetrics-process-monitoring-linux" + ) + + - set( + resource.attributes["process_name"], + ConvertCase( + resource.attributes["process.executable.name"], + "lower" + ) + ) where resource.attributes["process.executable.name"] != nil + + # Generic Python normalization retained from the reference config. + - replace_pattern( + resource.attributes["process_name"], + "^python[0-9.]*$", + "python" + ) where resource.attributes["process_name"] != nil + + # process_name is now the canonical process identity. + - delete_key( + resource.attributes, + "process.executable.name" + ) + + + # --------------------------------------------------------------------------- + # Do not send process telemetry without a normalized process name. + # --------------------------------------------------------------------------- + filter/process-without-name-linux: + error_mode: ignore + + metric_conditions: + + - resource.attributes["process_name"] == nil + + - resource.attributes["process_name"] == "" + + + # --------------------------------------------------------------------------- + # Prepare process metrics for aggregation. + # + # Linux FD: + # process.open_file_descriptors -> process.open_handles + # + # Process count: + # create one process.count sample for each process by copying the + # process.threads metric. Later aggregate_on_attributes("count", []) + # counts those samples. + # --------------------------------------------------------------------------- + transform/process-prepare-linux: + error_mode: ignore + + metric_statements: + + - context: metric + statements: + + - set( + metric.name, + "process.open_handles" + ) where metric.name == "process.open_file_descriptors" + + - copy_metric( + name="process.count", + description="Number of processes in the normalized process group" + ) where metric.name == "process.threads" + + + # --------------------------------------------------------------------------- + # Batch host telemetry. + # --------------------------------------------------------------------------- + batch/host-linux: + timeout: 2s + send_batch_size: 8192 + + + # --------------------------------------------------------------------------- + # Batch individual process telemetry before grouping. + # --------------------------------------------------------------------------- + batch/process-linux: + timeout: 2s + send_batch_size: 8192 + + + # --------------------------------------------------------------------------- + # Collapse ResourceMetrics belonging to the same normalized process group. + # + # process_name remains a resource attribute and therefore remains the + # process group identity. + # + # PID is not present. + # --------------------------------------------------------------------------- + groupbyattrs/process-linux: + keys: [] + + + # --------------------------------------------------------------------------- + # Aggregate all PIDs belonging to the same process_name. + # + # For the E2E workload: + # + # sleep PID A + # sleep PID B + # sleep PID C + # + # becomes one logical process group: + # + # process_name = sleep + # + # with: + # + # process.count = 3 + # + # --------------------------------------------------------------------------- + transform/process-aggregate-linux: + error_mode: ignore + + metric_statements: + + - context: metric + statements: + + # Number of actual processes in this normalized group. + - aggregate_on_attributes( + "count", + [] + ) where metric.name == "process.count" + + # Total threads across the group. + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.threads" + + # Preserve CPU state while aggregating all processes. + - aggregate_on_attributes( + "sum", + ["state"] + ) where metric.name == "process.cpu.utilization" + + # Total resident memory. + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.memory.usage" + + # Aggregate process memory utilization. + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.memory.utilization" + + # Total open file descriptors. + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.open_handles" + + # Oldest process in the group. + - aggregate_on_attributes( + "max", + [] + ) where metric.name == "process.uptime" + + + # --------------------------------------------------------------------------- + # Final batching before OTLP export. + # --------------------------------------------------------------------------- + batch/export: + timeout: 2s + send_batch_size: 8192 + + +exporters: + + # --------------------------------------------------------------------------- + # OAP service defined in docker-compose.yml. + # + # No authentication is required inside this isolated E2E network. + # --------------------------------------------------------------------------- + otlp: + endpoint: oap:11800 + + tls: + insecure: true + + +service: + + pipelines: + + # ------------------------------------------------------------------------- + # Host telemetry -> vm.yaml + # ------------------------------------------------------------------------- + metrics/host-linux: + + receivers: + - hostmetrics/system + + processors: + - resource_detection/system + - transform/host-linux + - batch/host-linux + + exporters: + - otlp + + + # ------------------------------------------------------------------------- + # Process telemetry -> process-hostmetrics-linux.yaml + # ------------------------------------------------------------------------- + metrics/processes-linux: + + receivers: + - hostmetrics/processes + + processors: + - resource_detection/system + - transform/process-identity-linux + - filter/process-without-name-linux + - transform/process-prepare-linux + - batch/process-linux + - groupbyattrs/process-linux + - transform/process-aggregate-linux + - batch/export + + exporters: + - otlp diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/process-hostmetrics-linux.yaml b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/process-hostmetrics-linux.yaml new file mode 100644 index 000000000000..61912ca9807f --- /dev/null +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/process-hostmetrics-linux.yaml @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Process metrics are pre-aggregated by normalized process_name in the +# Collector to bound PID cardinality. MAL performs the final SkyWalking +# service-instance mapping and metric calculation. +filter: "{ tags -> tags.job_name == 'hostmetrics-process-monitoring-linux' }" + +expSuffix: |- + service(['node_identifier_host_name'], '', Layer.OS_LINUX) + .instance( + ['node_identifier_host_name'], + '', + ['process_name'], + '', + Layer.OS_LINUX, + { + tags -> [ + 'process_name': tags.process_name, + 'host': tags.node_identifier_host_name + ] + } + ) + +metricPrefix: mp_process_linux + +metricsRules: + - name: num_procs + exp: "process_count.sum(['node_identifier_host_name','process_name'])" + + - name: num_threads + exp: "process_threads.sum(['node_identifier_host_name','process_name'])" + + - name: cpu_total_percent + exp: > + process_cpu_utilization + .tagNotEqual('state','wait') + .sum(['node_identifier_host_name','process_name']) + * 100 + + - name: cpu_user_percent + exp: > + process_cpu_utilization + .tagEqual('state','user') + .sum(['node_identifier_host_name','process_name']) + * 100 + + - name: cpu_system_percent + exp: > + process_cpu_utilization + .tagEqual('state','system') + .sum(['node_identifier_host_name','process_name']) + * 100 + + - name: memory_resident_bytes + exp: "process_memory_usage.sum(['node_identifier_host_name','process_name'])" + + # OTel process.memory.utilization uses unit 1 (fraction), so expose % here. + - name: memory_resident_percent + exp: > + process_memory_utilization + .sum(['node_identifier_host_name','process_name']) + * 100 + + - name: open_handles + exp: "process_open_handles.sum(['node_identifier_host_name','process_name'])" + + - name: oldest_process_uptime_seconds + exp: "process_uptime.max(['node_identifier_host_name','process_name'])" diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/process-hostmetrics-windows.yaml b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/process-hostmetrics-windows.yaml new file mode 100644 index 000000000000..c2f299f779d6 --- /dev/null +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/process-hostmetrics-windows.yaml @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Process metrics are pre-aggregated by normalized process_name in the +# Collector to bound PID cardinality. MAL performs the final SkyWalking +# service-instance mapping and metric calculation. +filter: "{ tags -> tags.job_name == 'hostmetrics-process-monitoring-windows' }" + +expSuffix: |- + service(['node_identifier_host_name'], '', Layer.OS_WINDOWS) + .instance( + ['node_identifier_host_name'], + '', + ['process_name'], + '', + Layer.OS_WINDOWS, + { + tags -> [ + 'process_name': tags.process_name, + 'host': tags.node_identifier_host_name + ] + } + ) + +metricPrefix: mp_process_windows + +metricsRules: + - name: num_procs + exp: "process_count.sum(['node_identifier_host_name','process_name'])" + + - name: num_threads + exp: "process_threads.sum(['node_identifier_host_name','process_name'])" + + - name: cpu_total_percent + exp: > + process_cpu_utilization + .tagNotEqual('state','wait') + .sum(['node_identifier_host_name','process_name']) + * 100 + + - name: cpu_user_percent + exp: > + process_cpu_utilization + .tagEqual('state','user') + .sum(['node_identifier_host_name','process_name']) + * 100 + + - name: cpu_system_percent + exp: > + process_cpu_utilization + .tagEqual('state','system') + .sum(['node_identifier_host_name','process_name']) + * 100 + + - name: memory_resident_bytes + exp: "process_memory_usage.sum(['node_identifier_host_name','process_name'])" + + # OTel process.memory.utilization uses unit 1 (fraction), so expose % here. + - name: memory_resident_percent + exp: > + process_memory_utilization + .sum(['node_identifier_host_name','process_name']) + * 100 + + - name: open_handles + exp: "process_open_handles.sum(['node_identifier_host_name','process_name'])" + + - name: oldest_process_uptime_seconds + exp: "process_uptime.max(['node_identifier_host_name','process_name'])" diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/vm.yaml b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/vm.yaml new file mode 100644 index 000000000000..0397e01d4c1f --- /dev/null +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/vm.yaml @@ -0,0 +1,257 @@ +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Linux infrastructure MAL. +# +# This rule intentionally accepts TWO source metric families under the same +# existing job_name "vm-monitoring": +# 1. node-exporter metrics (existing SkyWalking integration) +# 2. OpenTelemetry hostmetrics metrics (new alternative source) +# +# Both sources are mapped by ONE MAL file to the canonical meter_vm_* targets. +# This avoids cross-file metric ownership collisions in OAP 11 runtime rules. +# +# MAL SampleFamily '+' treats an EMPTY family as identity. Because a host sends +# one source family or the other, each expression can safely add the equivalent +# node-exporter and hostmetrics branches. The output metric is registered once. +filter: "{ tags -> tags.job_name == 'vm-monitoring' }" +expSuffix: service(['node_identifier_host_name'], Layer.OS_LINUX) +metricPrefix: meter_vm + +metricsRules: + # --------------------------------------------------------------------------- + # Canonical metrics: same target names and semantics for both sources. + # --------------------------------------------------------------------------- + + # CPU + - name: cpu_total_percentage + exp: > + (node_cpu_seconds_total * 100) + .tagNotEqual('mode','idle') + .sum(['node_identifier_host_name']) + .rate('PT1M') + + + (system_cpu_utilization * 100) + .tagNotEqual('mode','idle') + .sum(['node_identifier_host_name']) + + - name: cpu_average_used + exp: > + (node_cpu_seconds_total * 100) + .sum(['node_identifier_host_name','mode']) + .rate('PT1M') + + + (system_cpu_utilization * 100) + .sum(['node_identifier_host_name','mode']) + + - name: cpu_load1 + exp: "(node_load1 * 100) + (system_cpu_load_average_1m * 100)" + + - name: cpu_load5 + exp: "(node_load5 * 100) + (system_cpu_load_average_5m * 100)" + + - name: cpu_load15 + exp: "(node_load15 * 100) + (system_cpu_load_average_15m * 100)" + + # Physical memory + - name: memory_total + exp: "node_memory_MemTotal_bytes + system_memory_limit" + + - name: memory_available + exp: "node_memory_MemAvailable_bytes + system_linux_memory_available" + + - name: memory_used + exp: > + (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) + + + (system_memory_limit - system_linux_memory_available) + + - name: memory_buff_cache + exp: > + (node_memory_Buffers_bytes + node_memory_Cached_bytes) + + + ( + system_memory_usage.tagEqual('state','buffered').sum(['node_identifier_host_name']) + + + system_memory_usage.tagEqual('state','cached').sum(['node_identifier_host_name']) + ) + + # Swap + - name: memory_swap_free + exp: > + node_memory_SwapFree_bytes + + + system_paging_usage + .tagEqual('state','free') + .sum(['node_identifier_host_name']) + + - name: memory_swap_total + exp: > + node_memory_SwapTotal_bytes + + + ( + system_paging_usage.tagEqual('state','used').sum(['node_identifier_host_name']) + + + system_paging_usage.tagEqual('state','free').sum(['node_identifier_host_name']) + ) + + - name: memory_swap_percentage + exp: > + ( + 100 - ((node_memory_SwapFree_bytes * 100) / node_memory_SwapTotal_bytes) + ) + + + ( + ( + system_paging_usage.tagEqual('state','used').sum(['node_identifier_host_name']) + * 100 + ) + / + ( + system_paging_usage.tagEqual('state','used').sum(['node_identifier_host_name']) + + + system_paging_usage.tagEqual('state','free').sum(['node_identifier_host_name']) + ) + ) + + # Filesystem. hostmetrics free+reserved+used is the full capacity. + - name: filesystem_percentage + exp: > + ( + 100 - ( + (node_filesystem_avail_bytes * 100) + .sum(['node_identifier_host_name','mountpoint']) + / + node_filesystem_size_bytes + .sum(['node_identifier_host_name','mountpoint']) + ) + ) + + + ( + 100 - ( + (system_filesystem_usage.tagEqual('state','free') * 100) + .sum(['node_identifier_host_name','mountpoint']) + / + ( + system_filesystem_usage.tagEqual('state','free') + .sum(['node_identifier_host_name','mountpoint']) + + + system_filesystem_usage.tagEqual('state','reserved') + .sum(['node_identifier_host_name','mountpoint']) + + + system_filesystem_usage.tagEqual('state','used') + .sum(['node_identifier_host_name','mountpoint']) + ) + ) + ) + + # Disk throughput + - name: disk_read + exp: > + node_disk_read_bytes_total + .sum(['node_identifier_host_name']) + .rate('PT1M') + + + system_disk_io + .tagEqual('direction','read') + .sum(['node_identifier_host_name']) + .rate('PT1M') + + - name: disk_written + exp: > + node_disk_written_bytes_total + .sum(['node_identifier_host_name']) + .rate('PT1M') + + + system_disk_io + .tagEqual('direction','write') + .sum(['node_identifier_host_name']) + .rate('PT1M') + + # Network throughput + - name: network_receive + exp: > + node_network_receive_bytes_total + .sum(['node_identifier_host_name']) + .irate() + + + system_network_io + .tagEqual('direction','receive') + .sum(['node_identifier_host_name']) + .irate() + + - name: network_transmit + exp: > + node_network_transmit_bytes_total + .sum(['node_identifier_host_name']) + .irate() + + + system_network_io + .tagEqual('direction','transmit') + .sum(['node_identifier_host_name']) + .irate() + + # TCP connection metrics with true hostmetrics equivalents. + # Collector normalizes system.network.connections state values to lowercase. + - name: tcp_curr_estab + exp: > + node_netstat_Tcp_CurrEstab + + + system_network_connections + .tagEqual('protocol','tcp') + .tagEqual('state','established') + .sum(['node_identifier_host_name']) + + - name: tcp_tw + exp: > + node_sockstat_TCP_tw + + + system_network_connections + .tagEqual('protocol','tcp') + .tagEqual('state','time_wait') + .sum(['node_identifier_host_name']) + + # --------------------------------------------------------------------------- + # Existing node-exporter-only canonical metrics. + # hostmetrics has no source with equivalent semantics, so these remain node + # exporter only rather than being populated with a misleading approximation. + # --------------------------------------------------------------------------- + - name: tcp_alloc + exp: node_sockstat_TCP_alloc + + - name: sockets_used + exp: node_sockstat_sockets_used + + - name: udp_inuse + exp: node_sockstat_UDP_inuse + + - name: filefd_allocated + exp: node_filefd_allocated + + # --------------------------------------------------------------------------- + # New hostmetrics-only capabilities. New names are used whenever semantics + # do not exactly match an existing node-exporter target metric. + # --------------------------------------------------------------------------- + - name: cpu_cores_num + exp: system_cpu_logical_count.sum(['node_identifier_host_name']) + + - name: cpu_norm_percentage + exp: > + ( + (system_cpu_utilization * 100) + .tagNotEqual('mode','idle') + .sum(['node_identifier_host_name']) + ) + / + system_cpu_logical_count.sum(['node_identifier_host_name']) diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/windows.yaml b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/windows.yaml new file mode 100644 index 000000000000..2e872e86659a --- /dev/null +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/windows.yaml @@ -0,0 +1,225 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Windows infrastructure MAL. +# +# The existing windows-exporter source and the OpenTelemetry hostmetrics source +# intentionally use the same existing job_name "windows-monitoring". One MAL +# file maps either input family to the canonical meter_win_* targets, avoiding +# cross-file metric ownership collisions in OAP 11. +filter: "{ tags -> tags.job_name == 'windows-monitoring' }" +expSuffix: service(['node_identifier_host_name'], Layer.OS_WINDOWS) +metricPrefix: meter_win + +metricsRules: + # --------------------------------------------------------------------------- + # Canonical windows.yaml metrics, now accepting either source. + # --------------------------------------------------------------------------- + + # CPU + - name: cpu_total_percentage + exp: > + (windows_cpu_time_total * 100) + .tagNotEqual('mode','idle') + .sum(['node_identifier_host_name']) + .rate('PT1M') + + + (system_cpu_time * 100) + .tagNotEqual('mode','idle') + .sum(['node_identifier_host_name']) + .rate('PT1M') + + - name: cpu_average_used + exp: > + (windows_cpu_time_total * 100) + .sum(['node_identifier_host_name','mode']) + .rate('PT1M') + + + (system_cpu_time * 100) + .sum(['node_identifier_host_name','mode']) + .rate('PT1M') + + # Physical memory + - name: memory_total + exp: > + windows_cs_physical_memory_bytes + + + system_memory_limit.sum(['node_identifier_host_name']) + + - name: memory_available + exp: > + windows_os_physical_memory_free_bytes + + + system_memory_usage + .tagEqual('state','free') + .sum(['node_identifier_host_name']) + + - name: memory_used + exp: > + (windows_cs_physical_memory_bytes - windows_os_physical_memory_free_bytes) + + + ( + system_memory_limit.sum(['node_identifier_host_name']) + - + system_memory_usage.tagEqual('state','free').sum(['node_identifier_host_name']) + ) + + # Canonical virtual memory semantics. + # The hostmetrics branch uses Windows Performance Counters: + # Memory\\Commit Limit + # Memory\\Committed Bytes + - name: memory_virtual_memory_free + exp: > + windows_os_virtual_memory_free_bytes + + + ( + system_virtual_memory_commit_limit.sum(['node_identifier_host_name']) + - + system_virtual_memory_committed.sum(['node_identifier_host_name']) + ) + + - name: memory_virtual_memory_total + exp: > + windows_os_virtual_memory_bytes + + + system_virtual_memory_commit_limit.sum(['node_identifier_host_name']) + + - name: memory_virtual_memory_percentage + exp: > + ( + 100 - ((windows_os_virtual_memory_free_bytes * 100) / windows_os_virtual_memory_bytes) + ) + + + ( + system_virtual_memory_committed.sum(['node_identifier_host_name']) * 100 + / + system_virtual_memory_commit_limit.sum(['node_identifier_host_name']) + ) + + # Disk throughput + - name: disk_read + exp: > + windows_logical_disk_read_bytes_total + .sum(['node_identifier_host_name']) + .rate('PT1M') + + + system_disk_io + .tagEqual('direction','read') + .sum(['node_identifier_host_name']) + .rate('PT1M') + + - name: disk_written + exp: > + windows_logical_disk_write_bytes_total + .sum(['node_identifier_host_name']) + .rate('PT1M') + + + system_disk_io + .tagEqual('direction','write') + .sum(['node_identifier_host_name']) + .rate('PT1M') + + # Network throughput + - name: network_receive + exp: > + windows_net_bytes_received_total + .sum(['node_identifier_host_name']) + .irate() + + + system_network_io + .tagEqual('direction','receive') + .sum(['node_identifier_host_name']) + .irate() + + - name: network_transmit + exp: > + windows_net_bytes_sent_total + .sum(['node_identifier_host_name']) + .irate() + + + system_network_io + .tagEqual('direction','transmit') + .sum(['node_identifier_host_name']) + .irate() + + # --------------------------------------------------------------------------- + # Additional hostmetrics capabilities. These are additive and do not change + # the existing windows-exporter metric contract. + # --------------------------------------------------------------------------- + - name: cpu_cores_num + exp: system_cpu_logical_count.sum(['node_identifier_host_name']) + + - name: cpu_norm_percentage + exp: > + ( + (system_cpu_time * 100) + .tagNotEqual('mode','idle') + .sum(['node_identifier_host_name']) + .rate('PT1M') + ) + / + system_cpu_logical_count.sum(['node_identifier_host_name']) + + - name: cpu_load1 + exp: system_cpu_load_average_1m.sum(['node_identifier_host_name']) * 100 + + - name: cpu_load5 + exp: system_cpu_load_average_5m.sum(['node_identifier_host_name']) * 100 + + - name: cpu_load15 + exp: system_cpu_load_average_15m.sum(['node_identifier_host_name']) * 100 + + - name: filesystem_percentage + exp: > + 100 - ( + (system_filesystem_usage.tagEqual('state','free') * 100) + .sum(['node_identifier_host_name','mountpoint']) + / + ( + system_filesystem_usage.tagEqual('state','free') + .sum(['node_identifier_host_name','mountpoint']) + + + system_filesystem_usage.tagEqual('state','reserved') + .sum(['node_identifier_host_name','mountpoint']) + + + system_filesystem_usage.tagEqual('state','used') + .sum(['node_identifier_host_name','mountpoint']) + ) + ) + + - name: filehandles_allocated + exp: system_filehandles_allocated.sum(['node_identifier_host_name']) + + # system.paging.usage is the Windows pagefile, not the windows.yaml virtual + # memory metric. Keep it under explicit new names. + - name: memory_pagefile_free + exp: > + system_paging_usage + .tagEqual('state','free') + .sum(['node_identifier_host_name']) + + - name: memory_pagefile_total + exp: system_paging_usage.sum(['node_identifier_host_name']) + + - name: memory_pagefile_percentage + exp: > + ( + system_paging_usage + .tagEqual('state','used') + .sum(['node_identifier_host_name']) + * 100 + ) + / + system_paging_usage.sum(['node_identifier_host_name']) diff --git a/test/e2e-v2/cases/vm/prometheus-node-exporter/otel-rules/vm.yaml b/test/e2e-v2/cases/vm/prometheus-node-exporter/otel-rules/vm.yaml index a0921ad99d91..0397e01d4c1f 100644 --- a/test/e2e-v2/cases/vm/prometheus-node-exporter/otel-rules/vm.yaml +++ b/test/e2e-v2/cases/vm/prometheus-node-exporter/otel-rules/vm.yaml @@ -1,4 +1,3 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 @@ -13,84 +12,246 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This will parse a textual representation of a duration. The formats -# accepted are based on the ISO-8601 duration format {@code PnDTnHnMn.nS} -# with days considered to be exactly 24 hours. -#

-# Examples: -#

-#    "PT20.345S" -- parses as "20.345 seconds"
-#    "PT15M"     -- parses as "15 minutes" (where a minute is 60 seconds)
-#    "PT10H"     -- parses as "10 hours" (where an hour is 3600 seconds)
-#    "P2D"       -- parses as "2 days" (where a day is 24 hours or 86400 seconds)
-#    "P2DT3H4M"  -- parses as "2 days, 3 hours and 4 minutes"
-#    "P-6H3M"    -- parses as "-6 hours and +3 minutes"
-#    "-P6H3M"    -- parses as "-6 hours and -3 minutes"
-#    "-P-6H+3M"  -- parses as "+6 hours and -3 minutes"
-# 
-filter: "{ tags -> tags.job_name == 'vm-monitoring' }" # The OpenTelemetry job name +# Linux infrastructure MAL. +# +# This rule intentionally accepts TWO source metric families under the same +# existing job_name "vm-monitoring": +# 1. node-exporter metrics (existing SkyWalking integration) +# 2. OpenTelemetry hostmetrics metrics (new alternative source) +# +# Both sources are mapped by ONE MAL file to the canonical meter_vm_* targets. +# This avoids cross-file metric ownership collisions in OAP 11 runtime rules. +# +# MAL SampleFamily '+' treats an EMPTY family as identity. Because a host sends +# one source family or the other, each expression can safely add the equivalent +# node-exporter and hostmetrics branches. The output metric is registered once. +filter: "{ tags -> tags.job_name == 'vm-monitoring' }" +expSuffix: service(['node_identifier_host_name'], Layer.OS_LINUX) metricPrefix: meter_vm + metricsRules: + # --------------------------------------------------------------------------- + # Canonical metrics: same target names and semantics for both sources. + # --------------------------------------------------------------------------- - #node cpu + # CPU - name: cpu_total_percentage - exp: (node_cpu_seconds_total * 100).tagNotEqual('mode' , 'idle').sum(['node_identifier_host_name']).rate('PT1M').service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: > + (node_cpu_seconds_total * 100) + .tagNotEqual('mode','idle') + .sum(['node_identifier_host_name']) + .rate('PT1M') + + + (system_cpu_utilization * 100) + .tagNotEqual('mode','idle') + .sum(['node_identifier_host_name']) + - name: cpu_average_used - exp: (node_cpu_seconds_total * 100).sum(['node_identifier_host_name' , 'mode']).rate('PT1M').service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: > + (node_cpu_seconds_total * 100) + .sum(['node_identifier_host_name','mode']) + .rate('PT1M') + + + (system_cpu_utilization * 100) + .sum(['node_identifier_host_name','mode']) + - name: cpu_load1 - exp: (node_load1 * 100).service(['node_identifier_host_name'] , Layer.OS_LINUX).decorate({ me -> me.attr0 = me.layer.name()}) + exp: "(node_load1 * 100) + (system_cpu_load_average_1m * 100)" + - name: cpu_load5 - exp: (node_load5 * 100).service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: "(node_load5 * 100) + (system_cpu_load_average_5m * 100)" + - name: cpu_load15 - exp: (node_load15 * 100).service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: "(node_load15 * 100) + (system_cpu_load_average_15m * 100)" - #node Memory + # Physical memory - name: memory_total - exp: node_memory_MemTotal_bytes.service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: "node_memory_MemTotal_bytes + system_memory_limit" + - name: memory_available - exp: node_memory_MemAvailable_bytes.service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: "node_memory_MemAvailable_bytes + system_linux_memory_available" + - name: memory_used - exp: (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes).service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: > + (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) + + + (system_memory_limit - system_linux_memory_available) + - name: memory_buff_cache - exp: (node_memory_Buffers_bytes + node_memory_Cached_bytes).service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: > + (node_memory_Buffers_bytes + node_memory_Cached_bytes) + + + ( + system_memory_usage.tagEqual('state','buffered').sum(['node_identifier_host_name']) + + + system_memory_usage.tagEqual('state','cached').sum(['node_identifier_host_name']) + ) + + # Swap - name: memory_swap_free - exp: node_memory_SwapFree_bytes.service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: > + node_memory_SwapFree_bytes + + + system_paging_usage + .tagEqual('state','free') + .sum(['node_identifier_host_name']) + - name: memory_swap_total - exp: node_memory_SwapTotal_bytes.service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: > + node_memory_SwapTotal_bytes + + + ( + system_paging_usage.tagEqual('state','used').sum(['node_identifier_host_name']) + + + system_paging_usage.tagEqual('state','free').sum(['node_identifier_host_name']) + ) + - name: memory_swap_percentage - exp: (100 - ((node_memory_SwapFree_bytes * 100) / node_memory_SwapTotal_bytes)).service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: > + ( + 100 - ((node_memory_SwapFree_bytes * 100) / node_memory_SwapTotal_bytes) + ) + + + ( + ( + system_paging_usage.tagEqual('state','used').sum(['node_identifier_host_name']) + * 100 + ) + / + ( + system_paging_usage.tagEqual('state','used').sum(['node_identifier_host_name']) + + + system_paging_usage.tagEqual('state','free').sum(['node_identifier_host_name']) + ) + ) - #node filesystem + # Filesystem. hostmetrics free+reserved+used is the full capacity. - name: filesystem_percentage - exp: (100 - ((node_filesystem_avail_bytes * 100).sum(['node_identifier_host_name' , 'mountpoint']) / node_filesystem_size_bytes.sum(['node_identifier_host_name' , 'mountpoint']))).service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: > + ( + 100 - ( + (node_filesystem_avail_bytes * 100) + .sum(['node_identifier_host_name','mountpoint']) + / + node_filesystem_size_bytes + .sum(['node_identifier_host_name','mountpoint']) + ) + ) + + + ( + 100 - ( + (system_filesystem_usage.tagEqual('state','free') * 100) + .sum(['node_identifier_host_name','mountpoint']) + / + ( + system_filesystem_usage.tagEqual('state','free') + .sum(['node_identifier_host_name','mountpoint']) + + + system_filesystem_usage.tagEqual('state','reserved') + .sum(['node_identifier_host_name','mountpoint']) + + + system_filesystem_usage.tagEqual('state','used') + .sum(['node_identifier_host_name','mountpoint']) + ) + ) + ) - #node disk + # Disk throughput - name: disk_read - exp: (node_disk_read_bytes_total.sum(['node_identifier_host_name']).rate('PT1M')).service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: > + node_disk_read_bytes_total + .sum(['node_identifier_host_name']) + .rate('PT1M') + + + system_disk_io + .tagEqual('direction','read') + .sum(['node_identifier_host_name']) + .rate('PT1M') + - name: disk_written - exp: (node_disk_written_bytes_total.sum(['node_identifier_host_name']).rate('PT1M')).service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: > + node_disk_written_bytes_total + .sum(['node_identifier_host_name']) + .rate('PT1M') + + + system_disk_io + .tagEqual('direction','write') + .sum(['node_identifier_host_name']) + .rate('PT1M') - #node network + # Network throughput - name: network_receive - exp: (node_network_receive_bytes_total.sum(['node_identifier_host_name']).irate()).service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: > + node_network_receive_bytes_total + .sum(['node_identifier_host_name']) + .irate() + + + system_network_io + .tagEqual('direction','receive') + .sum(['node_identifier_host_name']) + .irate() + - name: network_transmit - exp: (node_network_transmit_bytes_total.sum(['node_identifier_host_name']).irate()).service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: > + node_network_transmit_bytes_total + .sum(['node_identifier_host_name']) + .irate() + + + system_network_io + .tagEqual('direction','transmit') + .sum(['node_identifier_host_name']) + .irate() - #node netstat + # TCP connection metrics with true hostmetrics equivalents. + # Collector normalizes system.network.connections state values to lowercase. - name: tcp_curr_estab - exp: node_netstat_Tcp_CurrEstab.service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: > + node_netstat_Tcp_CurrEstab + + + system_network_connections + .tagEqual('protocol','tcp') + .tagEqual('state','established') + .sum(['node_identifier_host_name']) + - name: tcp_tw - exp: node_sockstat_TCP_tw.service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: > + node_sockstat_TCP_tw + + + system_network_connections + .tagEqual('protocol','tcp') + .tagEqual('state','time_wait') + .sum(['node_identifier_host_name']) + + # --------------------------------------------------------------------------- + # Existing node-exporter-only canonical metrics. + # hostmetrics has no source with equivalent semantics, so these remain node + # exporter only rather than being populated with a misleading approximation. + # --------------------------------------------------------------------------- - name: tcp_alloc - exp: node_sockstat_TCP_alloc.service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: node_sockstat_TCP_alloc + - name: sockets_used - exp: node_sockstat_sockets_used.service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: node_sockstat_sockets_used + - name: udp_inuse - exp: node_sockstat_UDP_inuse.service(['node_identifier_host_name'] , Layer.OS_LINUX) + exp: node_sockstat_UDP_inuse - #node filefd - name: filefd_allocated - exp: node_filefd_allocated.service(['node_identifier_host_name'] , Layer.OS_LINUX) - + exp: node_filefd_allocated + # --------------------------------------------------------------------------- + # New hostmetrics-only capabilities. New names are used whenever semantics + # do not exactly match an existing node-exporter target metric. + # --------------------------------------------------------------------------- + - name: cpu_cores_num + exp: system_cpu_logical_count.sum(['node_identifier_host_name']) + - name: cpu_norm_percentage + exp: > + ( + (system_cpu_utilization * 100) + .tagNotEqual('mode','idle') + .sum(['node_identifier_host_name']) + ) + / + system_cpu_logical_count.sum(['node_identifier_host_name']) From 07ab80f4420cfb6ae8c5d7bed6c6729b1f55a606 Mon Sep 17 00:00:00 2001 From: mike-realuptime Date: Wed, 2 Sep 2026 14:58:16 +0000 Subject: [PATCH 2/4] Fix Apache license headers --- .../server-starter/src/main/resources/otel-rules/vm.yaml | 7 ++++--- test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/vm.yaml | 7 ++++--- .../cases/vm/prometheus-node-exporter/otel-rules/vm.yaml | 7 ++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/oap-server/server-starter/src/main/resources/otel-rules/vm.yaml b/oap-server/server-starter/src/main/resources/otel-rules/vm.yaml index 0397e01d4c1f..4e1e1865b1a8 100644 --- a/oap-server/server-starter/src/main/resources/otel-rules/vm.yaml +++ b/oap-server/server-starter/src/main/resources/otel-rules/vm.yaml @@ -1,8 +1,9 @@ -# contributor license agreements. See the NOTICE file distributed with +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # @@ -11,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +# # Linux infrastructure MAL. # # This rule intentionally accepts TWO source metric families under the same diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/vm.yaml b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/vm.yaml index 0397e01d4c1f..4e1e1865b1a8 100644 --- a/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/vm.yaml +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/vm.yaml @@ -1,8 +1,9 @@ -# contributor license agreements. See the NOTICE file distributed with +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # @@ -11,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +# # Linux infrastructure MAL. # # This rule intentionally accepts TWO source metric families under the same diff --git a/test/e2e-v2/cases/vm/prometheus-node-exporter/otel-rules/vm.yaml b/test/e2e-v2/cases/vm/prometheus-node-exporter/otel-rules/vm.yaml index 0397e01d4c1f..4e1e1865b1a8 100644 --- a/test/e2e-v2/cases/vm/prometheus-node-exporter/otel-rules/vm.yaml +++ b/test/e2e-v2/cases/vm/prometheus-node-exporter/otel-rules/vm.yaml @@ -1,8 +1,9 @@ -# contributor license agreements. See the NOTICE file distributed with +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # @@ -11,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +# # Linux infrastructure MAL. # # This rule intentionally accepts TWO source metric families under the same From fe5d5ff47cb3cd1d04e344299cddcb8fb1918b79 Mon Sep 17 00:00:00 2001 From: mike-realuptime Date: Fri, 4 Sep 2026 17:58:00 +0000 Subject: [PATCH 3/4] Address OpenTelemetry host and process monitoring review feedback --- .../en/setup/backend/backend-vm-monitoring.md | 55 ++-- .../setup/backend/backend-win-monitoring.md | 43 ++- .../otel-collector-hostmetrics-linux.yaml | 254 +++++++++++++++ .../otel-collector-hostmetrics-windows.yaml | 274 ++++++++++++++++ .../process-hostmetrics-linux.data.yaml | 121 +++++++ .../process-hostmetrics-windows.data.yaml | 121 +++++++ .../scripts/mal/test-otel-rules/vm.data.yaml | 39 ++- .../mal/test-otel-rules/windows.data.yaml | 149 ++++++++- .../src/main/resources/application.yml | 2 +- .../otel-rules/process-hostmetrics-linux.yaml | 11 +- .../process-hostmetrics-windows.yaml | 11 +- .../src/main/resources/otel-rules/vm.yaml | 68 +++- .../assert-collector-output.sh | 133 ++++++++ .../vm/otel-hostmetrics/docker-compose.yml | 41 ++- .../e2e-v2/cases/vm/otel-hostmetrics/e2e.yaml | 16 + .../expected/collector-contract.yml | 22 ++ .../fixtures/windows-host.json | 1 + .../fixtures/windows-process.json | 1 + .../otel-collector-config.yaml | 307 ++++++++++++------ .../otel-rules/process-hostmetrics-linux.yaml | 11 +- .../process-hostmetrics-windows.yaml | 11 +- .../otel-rules/vm.yaml | 68 +++- 22 files changed, 1573 insertions(+), 186 deletions(-) create mode 100644 docs/en/setup/backend/otel-collector-hostmetrics-linux.yaml create mode 100644 docs/en/setup/backend/otel-collector-hostmetrics-windows.yaml create mode 100644 oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/process-hostmetrics-linux.data.yaml create mode 100644 oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/process-hostmetrics-windows.data.yaml create mode 100755 test/e2e-v2/cases/vm/otel-hostmetrics/assert-collector-output.sh create mode 100644 test/e2e-v2/cases/vm/otel-hostmetrics/expected/collector-contract.yml create mode 100644 test/e2e-v2/cases/vm/otel-hostmetrics/fixtures/windows-host.json create mode 100644 test/e2e-v2/cases/vm/otel-hostmetrics/fixtures/windows-process.json diff --git a/docs/en/setup/backend/backend-vm-monitoring.md b/docs/en/setup/backend/backend-vm-monitoring.md index d84a50861660..dbdc6218306a 100644 --- a/docs/en/setup/backend/backend-vm-monitoring.md +++ b/docs/en/setup/backend/backend-vm-monitoring.md @@ -2,56 +2,71 @@ SkyWalking leverages Prometheus node-exporter to collect metrics data from the VMs and leverages OpenTelemetry Collector to transfer the metrics to [OpenTelemetry receiver](opentelemetry-receiver.md) and into the [Meter System](./../../concepts-and-designs/mal.md). VM entity as a `Service` in OAP and on the `Layer: OS_LINUX`. - SkyWalking also provides InfluxDB Telegraf to receive VMs' metrics data by [Telegraf receiver](./telegraf-receiver.md). The telegraf receiver plugin receiver, process and convert the metrics, then it send converted metrics to [Meter System](./../../concepts-and-designs/mal.md). VM entity as a `Service` in OAP and on the `Layer: OS_LINUX`. - ## Data flow **For OpenTelemetry receiver:** 1. The Prometheus node-exporter collects metrics data from the VMs. 2. The OpenTelemetry Collector fetches metrics from node-exporter via Prometheus Receiver and pushes metrics to the SkyWalking OAP Server via OpenTelemetry gRPC exporter. 3. The SkyWalking OAP Server parses the expression with [MAL](../../concepts-and-designs/mal.md) to filter/calculate/aggregate and store the results. - **For Telegraf receiver:** 1. The InfluxDB Telegraf [input plugins](https://docs.influxdata.com/telegraf/v1.24/plugins/) collects various metrics data from the VMs. 2. The cpu, mem, system, disk and diskio input plugins should be set in telegraf.conf file. 2. The InfluxDB Telegraf send `JSON` format metrics by `HTTP` messages to Telegraf Receiver, then pushes converted metrics to the SkyWalking OAP Server [Meter System](./../../concepts-and-designs/mal.md). 3. The SkyWalking OAP Server parses the expression with [MAL](../../concepts-and-designs/mal.md) to filter/calculate/aggregate ad store the results. 4. The meter_vm_cpu_average_used metrics indicates the average usage of each CPU core for telegraf receiver. - ## Setup **For OpenTelemetry receiver:** 1. Setup [Prometheus node-exporter](https://prometheus.io/docs/guides/node-exporter/). 2. Setup [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/). This is an example for OpenTelemetry Collector configuration [otel-collector-config.yaml](../../../../test/e2e-v2/cases/vm/prometheus-node-exporter/otel-collector-config.yaml). 3. Config SkyWalking [OpenTelemetry receiver](opentelemetry-receiver.md). - **For Telegraf receiver:** 1. Setup InfluxDB Telegraf's `telegraf.conf file` according to [Telegraf office document](https://docs.influxdata.com/telegraf/v1.24/). 2. Setup InfluxDB Telegraf's `telegraf.conf file` specific rules according to [Telegraf receiver document](telegraf-receiver.md). 3. Config SkyWalking [Telegraf receiver](telegraf-receiver.md). +### Native OpenTelemetry hostmetrics (extended alternative) +SkyWalking can also receive Linux host and process metrics directly from the OpenTelemetry Collector `hostmetrics` receiver, without Prometheus node-exporter. + +1. Install OpenTelemetry Collector Contrib `0.158.0` on the monitored host. +2. Use the deployable reference configuration [otel-collector-hostmetrics-linux.yaml](otel-collector-hostmetrics-linux.yaml). +3. Set `SW_OAP_GRPC_ADDRESS` to the SkyWalking OAP OTLP/gRPC endpoint. Optionally set `SW_PROCESS_NAME_REGEX` to restrict the process executable names to collect. +4. Config SkyWalking [OpenTelemetry receiver](opentelemetry-receiver.md). The default OAP rule set enables both `vm` and `process-hostmetrics-linux`. + +The Collector normalizes host and process identity, compacts process resource metrics, aligns process datapoint timestamps to the collection window, and pre-aggregates operating-system processes with the same normalized `process_name` before OTLP export. `process-hostmetrics-linux.yaml` maps the aggregated metrics to the corresponding SkyWalking process instance. + ## Supported Metrics -| Monitoring Panel | Unit | Metric Name | Description | Data Source | -|------------------------------|------|-------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------|-----------------------------------------------------| -| CPU Usage | % | meter_vm_cpu_total_percentage | The total percentage usage of the CPU core. If there are 2 cores, the maximum usage is 200%. | Prometheus node-exporter
Telegraf input plugin | -| Memory RAM Usage | MB | meter_vm_memory_used | The total RAM usage | Prometheus node-exporter
Telegraf input plugin | -| Memory Swap Usage | % | meter_vm_memory_swap_percentage | The percentage usage of swap memory | Prometheus node-exporter
Telegraf input plugin | -| CPU Average Used | % | meter_vm_cpu_average_used | The percentage usage of the CPU core in each mode | Prometheus node-exporter
Telegraf input plugin | -| CPU Load | | meter_vm_cpu_load1
meter_vm_cpu_load5
meter_vm_cpu_load15 | The CPU 1m / 5m / 15m average load | Prometheus node-exporter
Telegraf input plugin | -| Memory RAM | MB | meter_vm_memory_total
meter_vm_memory_available
meter_vm_memory_used
meter_vm_memory_buff_cache | The RAM statistics, including Total / Available / Used / Buff-Cache | Prometheus node-exporter
Telegraf input plugin | -| Memory Swap | MB | meter_vm_memory_swap_free
meter_vm_memory_swap_total | Swap memory statistics, including Free / Total | Prometheus node-exporter
Telegraf input plugin | -| File System Mountpoint Usage | % | meter_vm_filesystem_percentage | The percentage usage of the file system at each mount point | Prometheus node-exporter
Telegraf input plugin | -| Disk R/W | KB/s | meter_vm_disk_read
meter_vm_disk_written | The disk read and written | Prometheus node-exporter
Telegraf input plugin | -| Network Bandwidth Usage | KB/s | meter_vm_network_receive
meter_vm_network_transmit | The network receive and transmit | Prometheus node-exporter
Telegraf input plugin | -| Network Status | | meter_vm_tcp_curr_estab
meter_vm_tcp_tw
meter_vm_tcp_alloc
meter_vm_sockets_used
meter_vm_udp_inuse | The number of TCPs established / TCP time wait / TCPs allocated / sockets in use / UDPs in use | Prometheus node-exporter
Telegraf input plugin | -| Filefd Allocated | | meter_vm_filefd_allocated | The number of file descriptors allocated | Prometheus node-exporter | +The two OpenTelemetry receiver paths do not expose exactly the same Linux kernel metrics. In particular, OpenTelemetry Collector Contrib `0.158.0` `hostmetrics` has no host-level metric equivalent to node-exporter's `node_filefd_allocated`, which is derived from `/proc/sys/fs/file-nr`. The per-process `process.open_file_descriptors` metric has different semantics and is intentionally not used as a substitute. Therefore, `meter_vm_filefd_allocated` remains available only through the node-exporter path. + +Likewise, `meter_vm_tcp_alloc`, `meter_vm_sockets_used`, and `meter_vm_udp_inuse` remain node-exporter-only because `hostmetrics` `system.network.connections` does not expose equivalent Linux sockstat counters. `meter_vm_tcp_curr_estab` and `meter_vm_tcp_tw` do have compatible hostmetrics equivalents. + +| Monitoring Panel | Unit | Metric Name | Description | node-exporter | OTel hostmetrics | Telegraf | +|---|---|---|---|:---:|:---:|:---:| +| CPU Usage | % | `meter_vm_cpu_total_percentage` | Total CPU usage across all logical CPUs | Yes | Yes | Yes | +| CPU Cores | count | `meter_vm_cpu_cores_num` | Number of logical CPUs | — | Yes | — | +| Normalized CPU Usage | % | `meter_vm_cpu_norm_percentage` | CPU usage normalized by the number of logical CPUs | — | Yes | — | +| CPU Average Used | % | `meter_vm_cpu_average_used` | CPU usage by mode/state | Yes | Yes | Yes | +| CPU Load | | `meter_vm_cpu_load1`
`meter_vm_cpu_load5`
`meter_vm_cpu_load15` | CPU 1m / 5m / 15m average load | Yes | Yes | Yes | +| Memory RAM Usage | MB | `meter_vm_memory_used` | Total RAM usage | Yes | Yes | Yes | +| Memory RAM | MB | `meter_vm_memory_total`
`meter_vm_memory_available`
`meter_vm_memory_used`
`meter_vm_memory_buff_cache` | RAM Total / Available / Used / Buff-Cache | Yes | Yes | Yes | +| Memory Swap Usage | % | `meter_vm_memory_swap_percentage` | Percentage of swap memory in use | Yes | Yes | Yes | +| Memory Swap | MB | `meter_vm_memory_swap_free`
`meter_vm_memory_swap_total` | Swap Free / Total | Yes | Yes | Yes | +| File System Mountpoint Usage | % | `meter_vm_filesystem_percentage` | File-system usage at each mount point | Yes | Yes | Yes | +| Disk R/W | KB/s | `meter_vm_disk_read`
`meter_vm_disk_written` | Disk read and write throughput | Yes | Yes | Yes | +| Network Bandwidth Usage | KB/s | `meter_vm_network_receive`
`meter_vm_network_transmit` | Network receive and transmit throughput | Yes | Yes | Yes | +| TCP Established / Close-Wait | count | `meter_vm_tcp_curr_estab` | TCP connections in ESTABLISHED or CLOSE-WAIT state | Yes | Yes | Yes | +| TCP Time Wait | count | `meter_vm_tcp_tw` | TCP connections in TIME-WAIT state | Yes | Yes | Yes | +| TCP Allocated | count | `meter_vm_tcp_alloc` | Allocated TCP sockets | Yes | No | Yes | +| Sockets Used | count | `meter_vm_sockets_used` | Kernel sockets currently in use | Yes | No | Yes | +| UDP In Use | count | `meter_vm_udp_inuse` | UDP sockets currently in use | Yes | No | Yes | +| Filefd Allocated | count | `meter_vm_filefd_allocated` | Host-level allocated file descriptors from Linux `/proc/sys/fs/file-nr` | Yes | No | — | +| Network Connections | count | `meter_vm_network_connections` | TCP connections grouped by protocol and state | — | Yes | — | ## Customizing You can customize your own metrics/expression/dashboard panel. The metrics definition and expression rules are found in `/config/otel-rules/vm.yaml` and `/config/telegraf-rules/vm.yaml`. The dashboard panel confirmations ship from the SkyWalking Horizon UI bundle (apache/skywalking-horizon-ui); the OAP backend no longer hosts UI dashboard JSONs. - ## Blog For more details, see the blog article [SkyWalking 8.4 provides infrastructure monitoring](https://skywalking.apache.org/blog/2021-02-07-infrastructure-monitoring/). diff --git a/docs/en/setup/backend/backend-win-monitoring.md b/docs/en/setup/backend/backend-win-monitoring.md index 498afd54fb5d..f9366780cb68 100644 --- a/docs/en/setup/backend/backend-win-monitoring.md +++ b/docs/en/setup/backend/backend-win-monitoring.md @@ -2,31 +2,50 @@ SkyWalking leverages Prometheus windows_exporter to collect metrics data from the Windows and leverages OpenTelemetry Collector to transfer the metrics to [OpenTelemetry receiver](opentelemetry-receiver.md) and into the [Meter System](./../../concepts-and-designs/mal.md). Windows entity as a `Service` in OAP and on the `Layer: OS_WINDOWS`. - ## Data flow **For OpenTelemetry receiver:** 1. The Prometheus windows_exporter collects metrics data from the VMs. 2. The OpenTelemetry Collector fetches metrics from windows_exporter via Prometheus Receiver and pushes metrics to the SkyWalking OAP Server via OpenTelemetry gRPC exporter. 3. The SkyWalking OAP Server parses the expression with [MAL](../../concepts-and-designs/mal.md) to filter/calculate/aggregate and store the results. - ## Setup **For OpenTelemetry receiver:** 1. Setup [Prometheus windows_exporter](https://github.com/prometheus-community/windows_exporter). 2. Setup [OpenTelemetry Collector ](https://opentelemetry.io/docs/collector/). This is an example for OpenTelemetry Collector configuration [otel-collector-config.yaml](../../../../test/e2e-v2/cases/win/prometheus-windows_exporter/otel-collector-config.yaml). 3. Config SkyWalking [OpenTelemetry receiver](opentelemetry-receiver.md). +### Native OpenTelemetry hostmetrics (expanded alternative) +SkyWalking can also receive Windows host and process metrics directly from OpenTelemetry Collector Contrib, without Prometheus windows_exporter. + +1. Install OpenTelemetry Collector Contrib `0.158.0` on the monitored Windows host. +2. Use the deployable reference configuration [otel-collector-hostmetrics-windows.yaml](otel-collector-hostmetrics-windows.yaml). It includes the Windows Performance Counters required by the existing `windows.yaml` virtual-memory and system-handle metrics. +3. Set `SW_OAP_GRPC_ADDRESS` to the SkyWalking OAP OTLP/gRPC endpoint. Optionally set `SW_PROCESS_NAME_REGEX` to restrict the process executable names to collect. +4. Config SkyWalking [OpenTelemetry receiver](opentelemetry-receiver.md). The default OAP rule set enables both `windows` and `process-hostmetrics-windows`. + +The configuration normalizes the `system.cpu.time` `state` attribute to the existing `mode` contract and maps native `process.handles` to `process.open_handles`. The process pipeline normalizes process identity and pre-aggregates operating-system processes with the same normalized `process_name` before OTLP export. + ## Supported Metrics -| Monitoring Panel | Unit | Metric Name | Description | Data Source | -|------------------------------|------|-------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------|-----------------------------------------------------| -| CPU Usage | % | meter_win_cpu_total_percentage | The total percentage usage of the CPU core. If there are 2 cores, the maximum usage is 200%. | Prometheus windows_exporter | -| Memory RAM Usage | MB | meter_win_memory_used | The total RAM usage | Prometheus windows_exporter | -| Virtual-Memory Usage | % | meter_win_memory_virtual_memory_percentage | The percentage usage of virtual memeory memory | Prometheus windows_exporter | -| CPU Average Used | % | meter_win_cpu_average_used | The percentage usage of the CPU core in each mode | Prometheus windows_exporter | -| Memory RAM | MB | meter_win_memory_total
meter_win_memory_available
meter_win_memory_used | The RAM statistics, including Total / Available / Used | Prometheus windows_exporter | -| Virtual-Memroy | MB | meter_win_memory_virtual_memory_free
meter_win_memory_virtual_memory_total | Virtual memory statistics, including Free / Total | Prometheus windows_exporter | | The percentage usage of the file system at each mount point | Prometheus windows_exporter | -| Disk R/W | KB/s | meter_win_disk_read,meter_win_disk_written | The disk read and written | Prometheus windows_exporter | -| Network Bandwidth Usage | KB/s | meter_win_network_receive
meter_win_network_transmit | The network receive and transmit | Prometheus windows_exporter | | The number of file descriptors allocated | Prometheus windows_exporter | +The `OTel hostmetrics` column below refers to the complete OpenTelemetry Collector Contrib reference configuration documented above. Most values come from the `hostmetrics` receiver. Metrics marked with `*` use Windows Performance Counters in the same Collector configuration where `hostmetrics` does not expose the required equivalent semantics. + +| Monitoring Panel | Unit | Metric Name | Description | windows_exporter | OTel hostmetrics | +|---|---|---|---|:---:|:---:| +| CPU Usage | % | `meter_win_cpu_total_percentage` | Total CPU usage across all logical CPUs | Yes | Yes | +| CPU Average Used | % | `meter_win_cpu_average_used` | CPU usage by mode/state | Yes | Yes | +| CPU Cores | count | `meter_win_cpu_cores_num` | Number of logical CPUs | — | Yes | +| Normalized CPU Usage | % | `meter_win_cpu_norm_percentage` | CPU usage normalized by the number of logical CPUs | — | Yes | +| CPU Load | | `meter_win_cpu_load1`
`meter_win_cpu_load5`
`meter_win_cpu_load15` | CPU load metrics exposed by the Collector hostmetrics load scraper | — | Yes | +| Memory RAM Usage | MB | `meter_win_memory_used` | Total RAM usage | Yes | Yes | +| Memory RAM | MB | `meter_win_memory_total`
`meter_win_memory_available`
`meter_win_memory_used` | RAM Total / Available / Used | Yes | Yes | +| Virtual Memory Usage | % | `meter_win_memory_virtual_memory_percentage` | Percentage of committed virtual memory in use | Yes | Yes* | +| Virtual Memory | MB | `meter_win_memory_virtual_memory_free`
`meter_win_memory_virtual_memory_total` | Virtual memory Free / Total | Yes | Yes* | +| File System Mountpoint Usage | % | `meter_win_filesystem_percentage` | File-system usage at each volume or mount point | — | Yes | +| Disk R/W | KB/s | `meter_win_disk_read`
`meter_win_disk_written` | Disk read and write throughput | Yes | Yes | +| Network Bandwidth Usage | KB/s | `meter_win_network_receive`
`meter_win_network_transmit` | Network receive and transmit throughput | Yes | Yes | +| Allocated Handles | count | `meter_win_filehandles_allocated` | Current system-wide Windows handle count | — | Yes* | +| Pagefile Usage | % | `meter_win_memory_pagefile_percentage` | Percentage of Windows pagefile in use | — | Yes | +| Pagefile | MB | `meter_win_memory_pagefile_free`
`meter_win_memory_pagefile_total` | Windows pagefile Free / Total | — | Yes | + +`*` The OpenTelemetry path obtains these values through the `windowsperfcounters` receiver included in `otel-collector-hostmetrics-windows.yaml`: `Memory\\Commit Limit`, `Memory\\Committed Bytes`, and `System\\Handle Count`. They are transported and processed through the same OpenTelemetry Collector pipeline but are not native `hostmetrics` scraper metrics. ## Customizing You can customize your own metrics/expression/dashboard panel. diff --git a/docs/en/setup/backend/otel-collector-hostmetrics-linux.yaml b/docs/en/setup/backend/otel-collector-hostmetrics-linux.yaml new file mode 100644 index 000000000000..37308a9e1d10 --- /dev/null +++ b/docs/en/setup/backend/otel-collector-hostmetrics-linux.yaml @@ -0,0 +1,254 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0. +# +# Reference configuration for OpenTelemetry Collector Contrib 0.158.0. +# Native Linux host + process monitoring for SkyWalking OAP. +# +# Environment variables: +# SW_OAP_GRPC_ADDRESS OAP OTLP/gRPC endpoint (default 127.0.0.1:11800) +# SW_PROCESS_NAME_REGEX process executable-name regexp (default .*) +# +# The Collector normalizes process identity and pre-aggregates +# operating-system processes with the same normalized process_name before +# OTLP export. + +receivers: + hostmetrics/system: + collection_interval: 30s + scrapers: + cpu: + metrics: + system.cpu.utilization: + enabled: true + system.cpu.logical.count: + enabled: true + memory: + metrics: + system.linux.memory.available: + enabled: true + system.memory.limit: + enabled: true + disk: {} + network: {} + filesystem: {} + paging: {} + load: {} + + hostmetrics/processes: + collection_interval: 30s + scrapers: + process: + include: + match_type: regexp + names: + - "${env:SW_PROCESS_NAME_REGEX:-.*}" + mute_process_all_errors: true + metrics: + process.cpu.time: + enabled: false + process.cpu.utilization: + enabled: true + process.memory.usage: + enabled: true + process.memory.virtual: + enabled: false + process.memory.utilization: + enabled: true + process.threads: + enabled: true + process.uptime: + enabled: true + process.open_file_descriptors: + enabled: true + process.context_switches: + enabled: false + process.paging.faults: + enabled: false + process.signals_pending: + enabled: false + process.disk.io: + enabled: false + process.disk.operations: + enabled: false + resource_attributes: + process.executable.name: + enabled: true + process.executable.path: + enabled: false + process.command: + enabled: false + process.command_line: + enabled: false + process.owner: + enabled: false + process.pid: + enabled: false + process.parent_pid: + enabled: false + +processors: + resource_detection/system: + detectors: [system] + override: true + system: + hostname_sources: [os] + + transform/host-linux: + error_mode: ignore + metric_statements: + - context: resource + statements: + - set(resource.attributes["node_identifier_host_name"], resource.attributes["host.name"]) + where resource.attributes["host.name"] != nil + - set(resource.attributes["job_name"], "vm-monitoring") + - context: datapoint + statements: + - set(datapoint.attributes["mode"], datapoint.attributes["state"]) + where metric.name == "system.cpu.utilization" and datapoint.attributes["state"] != nil + - set(datapoint.attributes["mode"], "irq") + where metric.name == "system.cpu.utilization" and datapoint.attributes["mode"] == "interrupt" + - set(datapoint.attributes["mode"], "iowait") + where metric.name == "system.cpu.utilization" and datapoint.attributes["mode"] == "wait" + - delete_key(datapoint.attributes, "state") + where metric.name == "system.cpu.utilization" + - set(datapoint.attributes["protocol"], ConvertCase(datapoint.attributes["protocol"], "lower")) + where metric.name == "system.network.connections" and datapoint.attributes["protocol"] != nil + - set(datapoint.attributes["state"], ConvertCase(datapoint.attributes["state"], "lower")) + where metric.name == "system.network.connections" and datapoint.attributes["state"] != nil + + transform/process-identity-linux: + error_mode: ignore + metric_statements: + - context: resource + statements: + - set(resource.attributes["node_identifier_host_name"], resource.attributes["host.name"]) + where resource.attributes["host.name"] != nil + - set(resource.attributes["job_name"], "hostmetrics-process-monitoring-linux") + - set(resource.attributes["process_name"], ConvertCase(resource.attributes["process.executable.name"], "lower")) + where resource.attributes["process.executable.name"] != nil + - replace_pattern(resource.attributes["process_name"], "^python[0-9.]*$", "python") + where resource.attributes["process_name"] != nil + - delete_key(resource.attributes, "process.executable.name") + + filter/process-without-name-linux: + error_mode: ignore + metric_conditions: + - resource.attributes["process_name"] == nil + - resource.attributes["process_name"] == "" + + transform/process-prepare-linux: + error_mode: ignore + metric_statements: + - context: metric + statements: + - set(metric.name, "process.open_handles") + where metric.name == "process.open_file_descriptors" + - copy_metric( + name="process.count", + description="One sample per operating-system process", + unit="{process}" + ) where metric.name == "process.threads" + - context: datapoint + statements: + - set(value_int, 1) where metric.name == "process.count" + + # hostmetrics/process emits each operating-system process as a separate + # ResourceMetrics. After process identity normalization, compact equivalent + # resources so datapoints for the same logical process share Metric objects. + groupbyattrs/process-linux: + keys: [] + + transform/process-aggregate-linux: + error_mode: propagate + metric_statements: + # The process scraper assigns slightly different timestamps while + # iterating over PIDs. Align them to the 30-second collection window + # before aggregating datapoints. + - context: datapoint + statements: + - set( + time, + TruncateTime(time, Duration("30s")) + ) where IsMatch(metric.name, "^process\\.") + + - context: metric + statements: + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.count" + + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.threads" + + # Preserve CPU state because MAL consumes user/system/wait + # separately. + - aggregate_on_attributes( + "sum", + ["state"] + ) where metric.name == "process.cpu.utilization" + + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.memory.usage" + + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.memory.virtual" + + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.memory.utilization" + + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.open_handles" + + # Logical process uptime represents the oldest process in the group. + - aggregate_on_attributes( + "max", + [] + ) where metric.name == "process.uptime" + + batch/host-linux: + timeout: 2s + send_batch_size: 8192 + + batch/export: + timeout: 2s + send_batch_size: 8192 + +exporters: + otlp: + endpoint: "${env:SW_OAP_GRPC_ADDRESS:-127.0.0.1:11800}" + tls: + insecure: true + +service: + pipelines: + metrics/host-linux: + receivers: [hostmetrics/system] + processors: + - resource_detection/system + - transform/host-linux + - batch/host-linux + exporters: [otlp] + metrics/processes-linux: + receivers: [hostmetrics/processes] + processors: + - resource_detection/system + - transform/process-identity-linux + - filter/process-without-name-linux + - transform/process-prepare-linux + - groupbyattrs/process-linux + - transform/process-aggregate-linux + - batch/export + exporters: [otlp] diff --git a/docs/en/setup/backend/otel-collector-hostmetrics-windows.yaml b/docs/en/setup/backend/otel-collector-hostmetrics-windows.yaml new file mode 100644 index 000000000000..7371772b5aa6 --- /dev/null +++ b/docs/en/setup/backend/otel-collector-hostmetrics-windows.yaml @@ -0,0 +1,274 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0. +# +# Reference configuration for OpenTelemetry Collector Contrib 0.158.0. +# Native Windows host + process monitoring for SkyWalking OAP. +# +# Environment variables: +# SW_OAP_GRPC_ADDRESS OAP OTLP/gRPC endpoint (default 127.0.0.1:11800) +# SW_PROCESS_NAME_REGEX process executable-name regexp (default .*) +# +# system.cpu.time uses OTel's "state" attribute; this config normalizes it to +# the existing SkyWalking/windows_exporter "mode" contract. process.handles is +# normalized to process.open_handles for the process MAL rule. +# +# The Collector also normalizes process identity and pre-aggregates +# operating-system processes with the same normalized process_name before +# OTLP export. + +receivers: + hostmetrics/system: + collection_interval: 30s + scrapers: + cpu: + metrics: + system.cpu.logical.count: + enabled: true + memory: + metrics: + system.memory.limit: + enabled: true + disk: {} + network: {} + filesystem: {} + paging: {} + load: {} + + # Windows Performance Counters needed to preserve the existing windows.yaml + # virtual-memory and system-handle semantics. + windowsperfcounters/skywalking: + collection_interval: 30s + metrics: + system.virtual.memory.commit_limit: + description: Windows virtual memory commit limit + unit: By + gauge: {} + system.virtual.memory.committed: + description: Windows committed virtual memory bytes + unit: By + gauge: {} + system.filehandles.allocated: + description: Current number of system handles + unit: "{count}" + gauge: {} + perfcounters: + - object: Memory + counters: + - name: Commit Limit + metric: system.virtual.memory.commit_limit + - name: Committed Bytes + metric: system.virtual.memory.committed + - object: System + counters: + - name: Handle Count + metric: system.filehandles.allocated + + hostmetrics/processes: + collection_interval: 30s + scrapers: + process: + include: + match_type: regexp + names: + - "${env:SW_PROCESS_NAME_REGEX:-.*}" + mute_process_all_errors: true + metrics: + process.cpu.time: + enabled: false + process.cpu.utilization: + enabled: true + process.memory.usage: + enabled: true + process.memory.virtual: + enabled: false + process.memory.utilization: + enabled: true + process.threads: + enabled: true + process.uptime: + enabled: true + process.handles: + enabled: true + process.open_file_descriptors: + enabled: false + process.context_switches: + enabled: false + process.paging.faults: + enabled: false + process.signals_pending: + enabled: false + process.disk.io: + enabled: false + process.disk.operations: + enabled: false + resource_attributes: + process.executable.name: + enabled: true + process.executable.path: + enabled: false + process.command: + enabled: false + process.command_line: + enabled: false + process.owner: + enabled: false + process.pid: + enabled: false + process.parent_pid: + enabled: false + +processors: + resource_detection/system: + detectors: [system] + override: true + system: + hostname_sources: [os] + + transform/host-windows: + error_mode: ignore + metric_statements: + - context: resource + statements: + - set(resource.attributes["node_identifier_host_name"], resource.attributes["host.name"]) + where resource.attributes["host.name"] != nil + - set(resource.attributes["job_name"], "windows-monitoring") + - context: datapoint + statements: + - set(datapoint.attributes["mode"], datapoint.attributes["state"]) + where metric.name == "system.cpu.time" and datapoint.attributes["state"] != nil + - delete_key(datapoint.attributes, "state") + where metric.name == "system.cpu.time" + + transform/process-identity-windows: + error_mode: ignore + metric_statements: + - context: resource + statements: + - set(resource.attributes["node_identifier_host_name"], resource.attributes["host.name"]) + where resource.attributes["host.name"] != nil + - set(resource.attributes["job_name"], "hostmetrics-process-monitoring-windows") + - set(resource.attributes["process_name"], ConvertCase(resource.attributes["process.executable.name"], "lower")) + where resource.attributes["process.executable.name"] != nil + - delete_key(resource.attributes, "process.executable.name") + + filter/process-without-name-windows: + error_mode: ignore + metric_conditions: + - resource.attributes["process_name"] == nil + - resource.attributes["process_name"] == "" + + transform/process-prepare-windows: + error_mode: ignore + metric_statements: + - context: metric + statements: + - set(metric.name, "process.open_handles") + where metric.name == "process.handles" + - copy_metric( + name="process.count", + description="One sample per operating-system process", + unit="{process}" + ) where metric.name == "process.threads" + - context: datapoint + statements: + - set(value_int, 1) where metric.name == "process.count" + + # hostmetrics/process emits each operating-system process as a separate + # ResourceMetrics. After process identity normalization, compact equivalent + # resources so datapoints for the same logical process share Metric objects. + groupbyattrs/process-windows: + keys: [] + + transform/process-aggregate-windows: + error_mode: propagate + metric_statements: + # The process scraper assigns slightly different timestamps while + # iterating over processes. Align them to the 30-second collection + # window before aggregating datapoints. + - context: datapoint + statements: + - set( + time, + TruncateTime(time, Duration("30s")) + ) where IsMatch(metric.name, "^process\\.") + + - context: metric + statements: + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.count" + + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.threads" + + # Preserve CPU state because MAL consumes user/system separately. + - aggregate_on_attributes( + "sum", + ["state"] + ) where metric.name == "process.cpu.utilization" + + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.memory.usage" + + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.memory.virtual" + + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.memory.utilization" + + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.open_handles" + + # Logical process uptime represents the oldest process in the group. + - aggregate_on_attributes( + "max", + [] + ) where metric.name == "process.uptime" + + batch/host-windows: + timeout: 2s + send_batch_size: 8192 + + batch/export: + timeout: 2s + send_batch_size: 8192 + +exporters: + otlp: + endpoint: "${env:SW_OAP_GRPC_ADDRESS:-127.0.0.1:11800}" + tls: + insecure: true + +service: + pipelines: + metrics/host-windows: + receivers: [hostmetrics/system, windowsperfcounters/skywalking] + processors: + - resource_detection/system + - transform/host-windows + - batch/host-windows + exporters: [otlp] + metrics/processes-windows: + receivers: [hostmetrics/processes] + processors: + - resource_detection/system + - transform/process-identity-windows + - filter/process-without-name-windows + - transform/process-prepare-windows + - groupbyattrs/process-windows + - transform/process-aggregate-windows + - batch/export + exporters: [otlp] diff --git a/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/process-hostmetrics-linux.data.yaml b/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/process-hostmetrics-linux.data.yaml new file mode 100644 index 000000000000..6f3457dcee1b --- /dev/null +++ b/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/process-hostmetrics-linux.data.yaml @@ -0,0 +1,121 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +script: oap-server/server-starter/src/main/resources/otel-rules/process-hostmetrics-linux.yaml +input: + # This MAL runtime fixture deliberately supplies three same-identity samples + # to exercise sum/max grouping independently of Collector transport. In the + # production path, the Collector pre-aggregates same-name OS processes before + # OTLP export. Different thread values also ensure process.count cannot + # accidentally inherit process.threads. + process_count: + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 1.0 + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 1.0 + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 1.0 + process_threads: + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 10.0 + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 20.0 + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 30.0 + process_cpu_utilization: + - labels: {node_identifier_host_name: test-host, process_name: java, state: user} + value: 0.10 + - labels: {node_identifier_host_name: test-host, process_name: java, state: system} + value: 0.05 + - labels: {node_identifier_host_name: test-host, process_name: java, state: wait} + value: 0.02 + - labels: {node_identifier_host_name: test-host, process_name: java, state: user} + value: 0.20 + - labels: {node_identifier_host_name: test-host, process_name: java, state: system} + value: 0.10 + - labels: {node_identifier_host_name: test-host, process_name: java, state: wait} + value: 0.03 + - labels: {node_identifier_host_name: test-host, process_name: java, state: user} + value: 0.05 + - labels: {node_identifier_host_name: test-host, process_name: java, state: system} + value: 0.05 + - labels: {node_identifier_host_name: test-host, process_name: java, state: wait} + value: 0.01 + process_memory_usage: + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 1000.0 + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 2000.0 + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 3000.0 + process_memory_utilization: + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 0.01 + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 0.02 + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 0.03 + process_open_handles: + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 5.0 + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 7.0 + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 9.0 + process_uptime: + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 100.0 + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 200.0 + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 150.0 + +expected: + mp_process_linux_num_procs: + samples: + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 3.0 + mp_process_linux_num_threads: + samples: + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 60.0 + mp_process_linux_cpu_total_percent: + samples: + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 55.0 + mp_process_linux_cpu_user_percent: + samples: + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 35.0 + mp_process_linux_cpu_system_percent: + samples: + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 20.0 + mp_process_linux_memory_resident_bytes: + samples: + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 6000.0 + mp_process_linux_memory_resident_percent: + samples: + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 6.0 + mp_process_linux_open_handles: + samples: + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 21.0 + mp_process_linux_oldest_process_uptime_seconds: + samples: + - labels: {node_identifier_host_name: test-host, process_name: java} + value: 200.0 diff --git a/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/process-hostmetrics-windows.data.yaml b/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/process-hostmetrics-windows.data.yaml new file mode 100644 index 000000000000..62864d7c7e79 --- /dev/null +++ b/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/process-hostmetrics-windows.data.yaml @@ -0,0 +1,121 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +script: oap-server/server-starter/src/main/resources/otel-rules/process-hostmetrics-windows.yaml +input: + # This MAL runtime fixture deliberately supplies three same-identity samples + # to exercise sum/max grouping independently of Collector transport. In the + # production path, the Collector pre-aggregates same-name OS processes before + # OTLP export. Different thread values also ensure process.count cannot + # accidentally inherit process.threads. + process_count: + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 1.0 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 1.0 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 1.0 + process_threads: + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 10.0 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 20.0 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 30.0 + process_cpu_utilization: + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe, state: user} + value: 0.10 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe, state: system} + value: 0.05 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe, state: wait} + value: 0.02 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe, state: user} + value: 0.20 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe, state: system} + value: 0.10 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe, state: wait} + value: 0.03 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe, state: user} + value: 0.05 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe, state: system} + value: 0.05 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe, state: wait} + value: 0.01 + process_memory_usage: + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 1000.0 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 2000.0 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 3000.0 + process_memory_utilization: + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 0.01 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 0.02 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 0.03 + process_open_handles: + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 5.0 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 7.0 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 9.0 + process_uptime: + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 100.0 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 200.0 + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 150.0 + +expected: + mp_process_windows_num_procs: + samples: + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 3.0 + mp_process_windows_num_threads: + samples: + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 60.0 + mp_process_windows_cpu_total_percent: + samples: + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 55.0 + mp_process_windows_cpu_user_percent: + samples: + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 35.0 + mp_process_windows_cpu_system_percent: + samples: + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 20.0 + mp_process_windows_memory_resident_bytes: + samples: + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 6000.0 + mp_process_windows_memory_resident_percent: + samples: + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 6.0 + mp_process_windows_open_handles: + samples: + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 21.0 + mp_process_windows_oldest_process_uptime_seconds: + samples: + - labels: {node_identifier_host_name: test-host, process_name: svchost.exe} + value: 200.0 diff --git a/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/vm.data.yaml b/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/vm.data.yaml index 8672c5e22826..c28ef8f985bc 100644 --- a/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/vm.data.yaml +++ b/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/vm.data.yaml @@ -20,6 +20,21 @@ input: node_identifier_host_name: test-host mode: user value: 100.0 + # Native OpenTelemetry hostmetrics shape captured from otelcol-contrib 0.158.0. + # Values are fixed for deterministic MAL assertions. + system_cpu_logical_count: + - labels: + node_identifier_host_name: test-host + value: 4.0 + system_cpu_utilization: + - labels: + node_identifier_host_name: test-host + mode: user + value: 0.2 + - labels: + node_identifier_host_name: test-host + mode: idle + value: 0.8 node_load1: - labels: value: 100.0 @@ -100,7 +115,7 @@ expected: samples: - labels: node_identifier_host_name: test-host - value: 2500.0 + value: 2520.0 meter_vm_cpu_average_used: entities: - scope: SERVICE @@ -110,11 +125,12 @@ expected: - labels: node_identifier_host_name: test-host mode: user - value: 2500.0 + value: 2520.0 meter_vm_cpu_load1: entities: - scope: SERVICE layer: OS_LINUX + attr0: OS_LINUX samples: - labels: value: 10000.0 @@ -269,3 +285,22 @@ expected: samples: - labels: value: 100.0 + + meter_vm_cpu_cores_num: + entities: + - scope: SERVICE + service: test-host + layer: OS_LINUX + samples: + - labels: + node_identifier_host_name: test-host + value: 4.0 + meter_vm_cpu_norm_percentage: + entities: + - scope: SERVICE + service: test-host + layer: OS_LINUX + samples: + - labels: + node_identifier_host_name: test-host + value: 5.0 diff --git a/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/windows.data.yaml b/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/windows.data.yaml index 96e1bd011fec..e53357ae54f1 100644 --- a/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/windows.data.yaml +++ b/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/windows.data.yaml @@ -20,6 +20,59 @@ input: node_identifier_host_name: test-host mode: user value: 100.0 + # Native OpenTelemetry hostmetrics contract after Collector normalization. + # Values are fixed for deterministic MAL assertions. + system_cpu_logical_count: + - labels: + node_identifier_host_name: test-host + value: 4.0 + system_cpu_time: + - labels: + node_identifier_host_name: test-host + mode: user + value: 20.0 + - labels: + node_identifier_host_name: test-host + mode: idle + value: 80.0 + system_cpu_load_average_1m: + - labels: + node_identifier_host_name: test-host + value: 1.25 + system_cpu_load_average_5m: + - labels: + node_identifier_host_name: test-host + value: 0.75 + system_cpu_load_average_15m: + - labels: + node_identifier_host_name: test-host + value: 0.5 + system_filesystem_usage: + - labels: + node_identifier_host_name: test-host + mountpoint: "C:" + state: free + value: 60.0 + - labels: + node_identifier_host_name: test-host + mountpoint: "C:" + state: used + value: 40.0 + system_filehandles_allocated: + - labels: + node_identifier_host_name: test-host + value: 123.0 + system_paging_usage: + - labels: + node_identifier_host_name: test-host + device: 'C:\\pagefile.sys' + state: used + value: 30.0 + - labels: + node_identifier_host_name: test-host + device: 'C:\\pagefile.sys' + state: free + value: 70.0 windows_cs_physical_memory_bytes: - labels: value: 100.0 @@ -57,7 +110,7 @@ expected: samples: - labels: node_identifier_host_name: test-host - value: 2500.0 + value: 3000.0 meter_win_cpu_average_used: entities: - scope: SERVICE @@ -67,7 +120,7 @@ expected: - labels: node_identifier_host_name: test-host mode: user - value: 2500.0 + value: 3000.0 meter_win_memory_total: entities: - scope: SERVICE @@ -146,3 +199,95 @@ expected: - labels: node_identifier_host_name: test-host value: 0.0 + + meter_win_cpu_cores_num: + entities: + - scope: SERVICE + service: test-host + layer: OS_WINDOWS + samples: + - labels: + node_identifier_host_name: test-host + value: 4.0 + meter_win_cpu_norm_percentage: + entities: + - scope: SERVICE + service: test-host + layer: OS_WINDOWS + samples: + - labels: + node_identifier_host_name: test-host + value: 125.0 + meter_win_cpu_load1: + entities: + - scope: SERVICE + service: test-host + layer: OS_WINDOWS + samples: + - labels: + node_identifier_host_name: test-host + value: 125.0 + meter_win_cpu_load5: + entities: + - scope: SERVICE + service: test-host + layer: OS_WINDOWS + samples: + - labels: + node_identifier_host_name: test-host + value: 75.0 + meter_win_cpu_load15: + entities: + - scope: SERVICE + service: test-host + layer: OS_WINDOWS + samples: + - labels: + node_identifier_host_name: test-host + value: 50.0 + meter_win_filesystem_percentage: + entities: + - scope: SERVICE + service: test-host + layer: OS_WINDOWS + samples: + - labels: + node_identifier_host_name: test-host + mountpoint: "C:" + value: 40.0 + meter_win_filehandles_allocated: + entities: + - scope: SERVICE + service: test-host + layer: OS_WINDOWS + samples: + - labels: + node_identifier_host_name: test-host + value: 123.0 + meter_win_memory_pagefile_free: + entities: + - scope: SERVICE + service: test-host + layer: OS_WINDOWS + samples: + - labels: + node_identifier_host_name: test-host + value: 70.0 + meter_win_memory_pagefile_total: + entities: + - scope: SERVICE + service: test-host + layer: OS_WINDOWS + samples: + - labels: + node_identifier_host_name: test-host + value: 100.0 + meter_win_memory_pagefile_percentage: + entities: + - scope: SERVICE + service: test-host + layer: OS_WINDOWS + samples: + - labels: + node_identifier_host_name: test-host + value: 30.0 diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index cb8a48c96216..7148ec7085e0 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -412,7 +412,7 @@ receiver-otel: selector: ${SW_OTEL_RECEIVER:default} default: enabledHandlers: ${SW_OTEL_RECEIVER_ENABLED_HANDLERS:"otlp-traces,otlp-metrics,otlp-logs"} - enabledOtelMetricsRules: ${SW_OTEL_RECEIVER_ENABLED_OTEL_METRICS_RULES:"apisix,nginx/*,k8s/*,istio-controlplane,vm,mysql/*,postgresql/*,oap,aws-eks/*,windows,aws-s3/*,aws-dynamodb/*,aws-gateway/*,redis/*,elasticsearch/*,rabbitmq/*,mongodb/*,kafka/*,pulsar/*,bookkeeper/*,rocketmq/*,clickhouse/*,activemq/*,kong/*,flink/*,airflow/*,banyandb/*,envoy-ai-gateway/*,ios/*,miniprogram/*"} + enabledOtelMetricsRules: ${SW_OTEL_RECEIVER_ENABLED_OTEL_METRICS_RULES:"apisix,nginx/*,k8s/*,istio-controlplane,vm,process-hostmetrics-linux,mysql/*,postgresql/*,oap,aws-eks/*,windows,process-hostmetrics-windows,aws-s3/*,aws-dynamodb/*,aws-gateway/*,redis/*,elasticsearch/*,rabbitmq/*,mongodb/*,kafka/*,pulsar/*,bookkeeper/*,rocketmq/*,clickhouse/*,activemq/*,kong/*,flink/*,airflow/*,banyandb/*,envoy-ai-gateway/*,ios/*,miniprogram/*"} receiver-zipkin: selector: ${SW_RECEIVER_ZIPKIN:-} diff --git a/oap-server/server-starter/src/main/resources/otel-rules/process-hostmetrics-linux.yaml b/oap-server/server-starter/src/main/resources/otel-rules/process-hostmetrics-linux.yaml index 61912ca9807f..71356da371b6 100644 --- a/oap-server/server-starter/src/main/resources/otel-rules/process-hostmetrics-linux.yaml +++ b/oap-server/server-starter/src/main/resources/otel-rules/process-hostmetrics-linux.yaml @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Process metrics are pre-aggregated by normalized process_name in the -# Collector to bound PID cardinality. MAL performs the final SkyWalking -# service-instance mapping and metric calculation. +# The Collector normalizes process identity and pre-aggregates operating-system +# processes with the same process_name before OTLP export. MAL maps the +# resulting logical process metrics to the corresponding SkyWalking instance. filter: "{ tags -> tags.job_name == 'hostmetrics-process-monitoring-linux' }" expSuffix: |- @@ -67,7 +67,10 @@ metricsRules: - name: memory_resident_bytes exp: "process_memory_usage.sum(['node_identifier_host_name','process_name'])" - # OTel process.memory.utilization uses unit 1 (fraction), so expose % here. + # process.memory.utilization is emitted as a [0,1] fraction. Multiply by + # 100 before SkyWalking fixed-point storage to preserve two decimal places + # of percentage precision (basis points). UI expressions must divide the + # stored metric by 100. - name: memory_resident_percent exp: > process_memory_utilization diff --git a/oap-server/server-starter/src/main/resources/otel-rules/process-hostmetrics-windows.yaml b/oap-server/server-starter/src/main/resources/otel-rules/process-hostmetrics-windows.yaml index c2f299f779d6..bde9f5b73580 100644 --- a/oap-server/server-starter/src/main/resources/otel-rules/process-hostmetrics-windows.yaml +++ b/oap-server/server-starter/src/main/resources/otel-rules/process-hostmetrics-windows.yaml @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Process metrics are pre-aggregated by normalized process_name in the -# Collector to bound PID cardinality. MAL performs the final SkyWalking -# service-instance mapping and metric calculation. +# The Collector normalizes process identity and pre-aggregates operating-system +# processes with the same process_name before OTLP export. MAL maps the +# resulting logical process metrics to the corresponding SkyWalking instance. filter: "{ tags -> tags.job_name == 'hostmetrics-process-monitoring-windows' }" expSuffix: |- @@ -67,7 +67,10 @@ metricsRules: - name: memory_resident_bytes exp: "process_memory_usage.sum(['node_identifier_host_name','process_name'])" - # OTel process.memory.utilization uses unit 1 (fraction), so expose % here. + # process.memory.utilization is emitted as a [0,1] fraction. Multiply by + # 100 before SkyWalking fixed-point storage to preserve two decimal places + # of percentage precision (basis points). UI expressions must divide the + # stored metric by 100. - name: memory_resident_percent exp: > process_memory_utilization diff --git a/oap-server/server-starter/src/main/resources/otel-rules/vm.yaml b/oap-server/server-starter/src/main/resources/otel-rules/vm.yaml index 4e1e1865b1a8..3b50da8afafd 100644 --- a/oap-server/server-starter/src/main/resources/otel-rules/vm.yaml +++ b/oap-server/server-starter/src/main/resources/otel-rules/vm.yaml @@ -27,7 +27,6 @@ # one source family or the other, each expression can safely add the equivalent # node-exporter and hostmetrics branches. The output metric is registered once. filter: "{ tags -> tags.job_name == 'vm-monitoring' }" -expSuffix: service(['node_identifier_host_name'], Layer.OS_LINUX) metricPrefix: meter_vm metricsRules: @@ -38,6 +37,7 @@ metricsRules: # CPU - name: cpu_total_percentage exp: > + ( (node_cpu_seconds_total * 100) .tagNotEqual('mode','idle') .sum(['node_identifier_host_name']) @@ -46,59 +46,71 @@ metricsRules: (system_cpu_utilization * 100) .tagNotEqual('mode','idle') .sum(['node_identifier_host_name']) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: cpu_average_used exp: > + ( (node_cpu_seconds_total * 100) .sum(['node_identifier_host_name','mode']) .rate('PT1M') + (system_cpu_utilization * 100) .sum(['node_identifier_host_name','mode']) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: cpu_load1 - exp: "(node_load1 * 100) + (system_cpu_load_average_1m * 100)" + exp: "((node_load1 * 100) + (system_cpu_load_average_1m * 100)).service(['node_identifier_host_name'], Layer.OS_LINUX).decorate({ me -> me.attr0 = me.layer.name()})" - name: cpu_load5 - exp: "(node_load5 * 100) + (system_cpu_load_average_5m * 100)" + exp: "((node_load5 * 100) + (system_cpu_load_average_5m * 100)).service(['node_identifier_host_name'], Layer.OS_LINUX)" - name: cpu_load15 - exp: "(node_load15 * 100) + (system_cpu_load_average_15m * 100)" + exp: "((node_load15 * 100) + (system_cpu_load_average_15m * 100)).service(['node_identifier_host_name'], Layer.OS_LINUX)" # Physical memory - name: memory_total - exp: "node_memory_MemTotal_bytes + system_memory_limit" + exp: "(node_memory_MemTotal_bytes + system_memory_limit).service(['node_identifier_host_name'], Layer.OS_LINUX)" - name: memory_available - exp: "node_memory_MemAvailable_bytes + system_linux_memory_available" + exp: "(node_memory_MemAvailable_bytes + system_linux_memory_available).service(['node_identifier_host_name'], Layer.OS_LINUX)" - name: memory_used exp: > + ( (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) + (system_memory_limit - system_linux_memory_available) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: memory_buff_cache exp: > + ( (node_memory_Buffers_bytes + node_memory_Cached_bytes) + ( system_memory_usage.tagEqual('state','buffered').sum(['node_identifier_host_name']) + system_memory_usage.tagEqual('state','cached').sum(['node_identifier_host_name']) + - + system_memory_usage.tagEqual('state','slab_reclaimable').sum(['node_identifier_host_name']) ) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) # Swap - name: memory_swap_free exp: > + ( node_memory_SwapFree_bytes + system_paging_usage .tagEqual('state','free') .sum(['node_identifier_host_name']) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: memory_swap_total exp: > + ( node_memory_SwapTotal_bytes + ( @@ -106,9 +118,11 @@ metricsRules: + system_paging_usage.tagEqual('state','free').sum(['node_identifier_host_name']) ) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: memory_swap_percentage exp: > + ( ( 100 - ((node_memory_SwapFree_bytes * 100) / node_memory_SwapTotal_bytes) ) @@ -125,10 +139,12 @@ metricsRules: system_paging_usage.tagEqual('state','free').sum(['node_identifier_host_name']) ) ) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) # Filesystem. hostmetrics free+reserved+used is the full capacity. - name: filesystem_percentage exp: > + ( ( 100 - ( (node_filesystem_avail_bytes * 100) @@ -156,10 +172,12 @@ metricsRules: ) ) ) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) # Disk throughput - name: disk_read exp: > + ( node_disk_read_bytes_total .sum(['node_identifier_host_name']) .rate('PT1M') @@ -168,9 +186,11 @@ metricsRules: .tagEqual('direction','read') .sum(['node_identifier_host_name']) .rate('PT1M') + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: disk_written exp: > + ( node_disk_written_bytes_total .sum(['node_identifier_host_name']) .rate('PT1M') @@ -179,10 +199,12 @@ metricsRules: .tagEqual('direction','write') .sum(['node_identifier_host_name']) .rate('PT1M') + ).service(['node_identifier_host_name'], Layer.OS_LINUX) # Network throughput - name: network_receive exp: > + ( node_network_receive_bytes_total .sum(['node_identifier_host_name']) .irate() @@ -191,9 +213,11 @@ metricsRules: .tagEqual('direction','receive') .sum(['node_identifier_host_name']) .irate() + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: network_transmit exp: > + ( node_network_transmit_bytes_total .sum(['node_identifier_host_name']) .irate() @@ -202,26 +226,38 @@ metricsRules: .tagEqual('direction','transmit') .sum(['node_identifier_host_name']) .irate() + ).service(['node_identifier_host_name'], Layer.OS_LINUX) # TCP connection metrics with true hostmetrics equivalents. # Collector normalizes system.network.connections state values to lowercase. - name: tcp_curr_estab exp: > + ( node_netstat_Tcp_CurrEstab + - system_network_connections - .tagEqual('protocol','tcp') - .tagEqual('state','established') - .sum(['node_identifier_host_name']) + ( + system_network_connections + .tagEqual('protocol','tcp') + .tagEqual('state','established') + .sum(['node_identifier_host_name']) + + + system_network_connections + .tagEqual('protocol','tcp') + .tagEqual('state','close_wait') + .sum(['node_identifier_host_name']) + ) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: tcp_tw exp: > + ( node_sockstat_TCP_tw + system_network_connections .tagEqual('protocol','tcp') .tagEqual('state','time_wait') .sum(['node_identifier_host_name']) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) # --------------------------------------------------------------------------- # Existing node-exporter-only canonical metrics. @@ -229,26 +265,27 @@ metricsRules: # exporter only rather than being populated with a misleading approximation. # --------------------------------------------------------------------------- - name: tcp_alloc - exp: node_sockstat_TCP_alloc + exp: (node_sockstat_TCP_alloc).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: sockets_used - exp: node_sockstat_sockets_used + exp: (node_sockstat_sockets_used).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: udp_inuse - exp: node_sockstat_UDP_inuse + exp: (node_sockstat_UDP_inuse).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: filefd_allocated - exp: node_filefd_allocated + exp: (node_filefd_allocated).service(['node_identifier_host_name'], Layer.OS_LINUX) # --------------------------------------------------------------------------- # New hostmetrics-only capabilities. New names are used whenever semantics # do not exactly match an existing node-exporter target metric. # --------------------------------------------------------------------------- - name: cpu_cores_num - exp: system_cpu_logical_count.sum(['node_identifier_host_name']) + exp: (system_cpu_logical_count.sum(['node_identifier_host_name'])).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: cpu_norm_percentage exp: > + ( ( (system_cpu_utilization * 100) .tagNotEqual('mode','idle') @@ -256,3 +293,4 @@ metricsRules: ) / system_cpu_logical_count.sum(['node_identifier_host_name']) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/assert-collector-output.sh b/test/e2e-v2/cases/vm/otel-hostmetrics/assert-collector-output.sh new file mode 100755 index 000000000000..b36b8ee373ab --- /dev/null +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/assert-collector-output.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +set -euo pipefail + +CID="$( + docker ps \ + --filter 'label=com.docker.compose.service=vm-service' \ + --filter 'status=running' \ + --format '{{.ID}}' \ + | head -n 1 +)" + +if [ -z "${CID}" ]; then + echo "vm-service container not found" >&2 + exit 1 +fi + +TMPDIR="$(mktemp -d)" +trap 'rm -rf "${TMPDIR}"' EXIT + +docker exec "${CID}" cat /tmp/process-linux-raw.log \ + > "${TMPDIR}/process-linux.log" + +docker exec "${CID}" cat /tmp/windows-host-raw.log \ + > "${TMPDIR}/windows-host.log" + +docker exec "${CID}" cat /tmp/windows-process-raw.log \ + > "${TMPDIR}/windows-process.log" + + +# --------------------------------------------------------------------------- +# Linux process pre-OTLP contract. +# +# For every observed process.count metric block we require the Collector-side +# result to contain one datapoint with value 3 and the corrected {process} unit. +# A raw value 1 would reproduce the reviewer-reported regression. +# --------------------------------------------------------------------------- +awk ' +function finish_block() { + if (!in_count) { + return + } + + if (points == 1 && value3 && unit_process) { + good = 1 + } + + if (value1) { + bad = 1 + } + + in_count = 0 +} + +/-> Name: process.count[[:space:]]*$/ { + finish_block() + in_count = 1 + points = 0 + value3 = 0 + value1 = 0 + unit_process = 0 + next +} + +in_count && /^[[:space:]]*Metric #[0-9]+/ { + finish_block() + next +} + +in_count && /NumberDataPoints #[0-9]+/ { + points++ +} + +in_count && /-> Unit: \{process\}/ { + unit_process = 1 +} + +in_count && /Value:[[:space:]]+3(\.0+)?[[:space:]]*$/ { + value3 = 1 +} + +in_count && /Value:[[:space:]]+1(\.0+)?[[:space:]]*$/ { + value1 = 1 +} + +END { + finish_block() + exit(good && !bad ? 0 : 1) +} +' "${TMPDIR}/process-linux.log" + + +# --------------------------------------------------------------------------- +# Windows host normalization: +# native system.cpu.time state -> canonical mode. +# --------------------------------------------------------------------------- +grep -q -- '-> Name: system.cpu.time' \ + "${TMPDIR}/windows-host.log" + +grep -q -- '-> mode: Str(idle)' \ + "${TMPDIR}/windows-host.log" + +grep -q -- '-> mode: Str(user)' \ + "${TMPDIR}/windows-host.log" + +if grep -q -- '-> state:' "${TMPDIR}/windows-host.log"; then + echo "Windows system.cpu.time still contains state attribute" >&2 + exit 1 +fi + + +# --------------------------------------------------------------------------- +# Windows process normalization: +# native process.handles -> canonical process.open_handles. +# --------------------------------------------------------------------------- +grep -q -- '-> Name: process.open_handles' \ + "${TMPDIR}/windows-process.log" + +if grep -q -- '-> Name: process.handles[[:space:]]*$' \ + "${TMPDIR}/windows-process.log"; then + echo "Native process.handles survived normalization" >&2 + exit 1 +fi + + +cat <<'YAML' +collector_process_count: + datapoints_per_export: 1 + value: 3 + unit: "{process}" +windows_normalization: + cpu_state_to_mode: true + process_handles_to_open_handles: true +YAML diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/docker-compose.yml b/test/e2e-v2/cases/vm/otel-hostmetrics/docker-compose.yml index 693a4f444242..88168a2965f7 100644 --- a/test/e2e-v2/cases/vm/otel-hostmetrics/docker-compose.yml +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/docker-compose.yml @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - services: oap: extends: @@ -20,20 +19,17 @@ services: service: oap environment: SW_OTEL_RECEIVER: default - SW_OTEL_RECEIVER_ENABLED_OTEL_METRICS_RULES: vm,process-hostmetrics-linux volumes: - ./otel-rules/vm.yaml:/skywalking/config/otel-rules/vm.yaml - ./otel-rules/process-hostmetrics-linux.yaml:/skywalking/config/otel-rules/process-hostmetrics-linux.yaml ports: - 12800 - banyandb: extends: file: ../../../script/docker-compose/base-compose.yml service: banyandb ports: - 17912 - vm-service: hostname: vm-service build: @@ -45,9 +41,44 @@ services: - e2e volumes: - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml + - ./fixtures:/fixtures:ro depends_on: oap: condition: service_healthy - + windows-fixture-replay: + image: curlimages/curl:8.10.1 + depends_on: + - vm-service + volumes: + - ./fixtures:/fixtures:ro + networks: + - e2e + command: + - sh + - -c + - | + while true; do + TS="$$(date +%s)000000000" + echo "Replaying Windows OTLP fixtures timestamp=$${TS}" + sed "s/1788474000000000000/$${TS}/g" \ + /fixtures/windows-host.json \ + | curl -sS \ + -o /dev/null \ + -w "windows-host HTTP %{http_code}\n" \ + -X POST \ + -H 'Content-Type: application/json' \ + --data-binary @- \ + http://vm-service:4318/v1/metrics || true + sed "s/1788474000000000000/$${TS}/g" \ + /fixtures/windows-process.json \ + | curl -sS \ + -o /dev/null \ + -w "windows-process HTTP %{http_code}\n" \ + -X POST \ + -H 'Content-Type: application/json' \ + --data-binary @- \ + http://vm-service:4318/v1/metrics || true + sleep 10 + done networks: e2e: diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/e2e.yaml b/test/e2e-v2/cases/vm/otel-hostmetrics/e2e.yaml index fba61f3fc4f2..bb3c28839b8f 100644 --- a/test/e2e-v2/cases/vm/otel-hostmetrics/e2e.yaml +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/e2e.yaml @@ -64,3 +64,19 @@ verify: - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=mp_process_linux_oldest_process_uptime_seconds --service-name=vm-service --instance-name=sleep expected: ../expected/metrics-has-value.yml + + # Raw post-processor Collector contract. This distinguishes true + # Collector-side aggregation from a value that only becomes 3 in MAL. + - query: bash test/e2e-v2/cases/vm/otel-hostmetrics/assert-collector-output.sh + expected: expected/collector-contract.yml + + # Synthetic native-Windows OTLP fixtures are normalized by Collector 0.158 + # and then consumed by the production Windows MAL rules. + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_win_cpu_cores_num --service-name=win-fixture + expected: ../expected/metrics-has-value.yml + + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=mp_process_windows_num_procs --service-name=win-fixture --instance-name=svchost.exe + expected: ../expected/metrics-has-value.yml + + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=mp_process_windows_open_handles --service-name=win-fixture --instance-name=svchost.exe + expected: ../expected/metrics-has-value.yml diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/expected/collector-contract.yml b/test/e2e-v2/cases/vm/otel-hostmetrics/expected/collector-contract.yml new file mode 100644 index 000000000000..f7c4c0bdbd14 --- /dev/null +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/expected/collector-contract.yml @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collector_process_count: + datapoints_per_export: 1 + value: 3 + unit: "{process}" +windows_normalization: + cpu_state_to_mode: true + process_handles_to_open_handles: true diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/fixtures/windows-host.json b/test/e2e-v2/cases/vm/otel-hostmetrics/fixtures/windows-host.json new file mode 100644 index 000000000000..048391f8999a --- /dev/null +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/fixtures/windows-host.json @@ -0,0 +1 @@ +{"resourceMetrics":[{"resource":{"attributes":[{"key":"host.name","value":{"stringValue":"win-fixture"}}]},"scopeMetrics":[{"scope":{"name":"windows-host-fixture"},"metrics":[{"name":"system.cpu.logical.count","unit":"{cpu}","gauge":{"dataPoints":[{"timeUnixNano":"1788474000000000000","asInt":"4"}]}},{"name":"system.cpu.time","unit":"s","sum":{"dataPoints":[{"attributes":[{"key":"state","value":{"stringValue":"idle"}}],"timeUnixNano":"1788474000000000000","asDouble":100.0},{"attributes":[{"key":"state","value":{"stringValue":"user"}}],"timeUnixNano":"1788474000000000000","asDouble":20.0}],"aggregationTemporality":2,"isMonotonic":true}}]}]}]} diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/fixtures/windows-process.json b/test/e2e-v2/cases/vm/otel-hostmetrics/fixtures/windows-process.json new file mode 100644 index 000000000000..6b5edb6e3272 --- /dev/null +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/fixtures/windows-process.json @@ -0,0 +1 @@ +{"resourceMetrics":[{"resource":{"attributes":[{"key":"host.name","value":{"stringValue":"win-fixture"}},{"key":"process.executable.name","value":{"stringValue":"svchost.exe"}}]},"scopeMetrics":[{"scope":{"name":"windows-process-fixture"},"metrics":[{"name":"process.threads","unit":"{threads}","gauge":{"dataPoints":[{"timeUnixNano":"1788474000000000000","asInt":"10"}]}},{"name":"process.handles","unit":"{handles}","gauge":{"dataPoints":[{"timeUnixNano":"1788474000000000000","asInt":"5"}]}}]}]}]} diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/otel-collector-config.yaml b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-collector-config.yaml index 79487395ab83..e049333b394a 100644 --- a/test/e2e-v2/cases/vm/otel-hostmetrics/otel-collector-config.yaml +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-collector-config.yaml @@ -24,6 +24,13 @@ receivers: + otlp/windows-fixture-replay: + protocols: + http: + endpoint: 0.0.0.0:4318 + + # Synthetic OTLP JSON fixtures exercise the Windows normalization contract + # on the Linux E2E runner; no Windows host is required. # --------------------------------------------------------------------------- # Host metrics # --------------------------------------------------------------------------- @@ -146,6 +153,138 @@ receivers: processors: + groupbyattrs/process-linux: + keys: [] + + transform/process-aggregate-linux: + error_mode: propagate + metric_statements: + # hostmetrics/process generates each PID as a separate ResourceMetrics + # and assigns a slightly different timestamp to each process. + # + # groupbyattrs compacts ResourceMetrics with the same normalized + # process identity into common Metric objects. Align timestamps before + # aggregating their datapoints. + - context: datapoint + statements: + - set( + time, + TruncateTime(time, Duration("30s")) + ) where IsMatch(metric.name, "^process\\.") + + - context: metric + statements: + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.count" + + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.threads" + + # CPU keeps state because MAL needs user/system/wait separately. + - aggregate_on_attributes( + "sum", + ["state"] + ) where metric.name == "process.cpu.utilization" + + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.memory.usage" + + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.memory.virtual" + + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.memory.utilization" + + - aggregate_on_attributes( + "sum", + [] + ) where metric.name == "process.open_handles" + + # Logical process uptime is the oldest member of the group. + - aggregate_on_attributes( + "max", + [] + ) where metric.name == "process.uptime" + # --------------------------------------------------------------------------- + # Windows OTLP fixture normalization. These processors intentionally mirror + # the deployable Windows reference configuration. + # --------------------------------------------------------------------------- + transform/host-windows: + error_mode: ignore + metric_statements: + - context: resource + statements: + - set(resource.attributes["node_identifier_host_name"], resource.attributes["host.name"]) + where resource.attributes["host.name"] != nil + - set(resource.attributes["job_name"], "windows-monitoring") + - context: datapoint + statements: + - set(datapoint.attributes["mode"], datapoint.attributes["state"]) + where metric.name == "system.cpu.time" and datapoint.attributes["state"] != nil + - delete_key(datapoint.attributes, "state") + where metric.name == "system.cpu.time" + + transform/process-identity-windows: + error_mode: ignore + metric_statements: + - context: resource + statements: + - set(resource.attributes["node_identifier_host_name"], resource.attributes["host.name"]) + where resource.attributes["host.name"] != nil + - set(resource.attributes["job_name"], "hostmetrics-process-monitoring-windows") + - set(resource.attributes["process_name"], ConvertCase(resource.attributes["process.executable.name"], "lower")) + where resource.attributes["process.executable.name"] != nil + - delete_key(resource.attributes, "process.executable.name") + + filter/process-without-name-windows: + error_mode: ignore + metric_conditions: + - resource.attributes["process_name"] == nil + - resource.attributes["process_name"] == "" + + transform/process-prepare-windows: + error_mode: ignore + metric_statements: + - context: metric + statements: + - set(metric.name, "process.open_handles") + where metric.name == "process.handles" + - copy_metric( + name="process.count", + description="One sample per operating-system process", + unit="{process}" + ) where metric.name == "process.threads" + - context: datapoint + statements: + - set(value_int, 1) where metric.name == "process.count" + + groupbyattrs/process-windows: + keys: [] + + transform/process-aggregate-windows: + error_mode: propagate + metric_statements: + - context: datapoint + statements: + - set( + time, + TruncateTime(time, Duration("30s")) + ) where IsMatch(metric.name, "^process\\.") + - context: metric + statements: + - aggregate_on_attributes("sum", []) where metric.name == "process.count" + - aggregate_on_attributes("sum", []) where metric.name == "process.threads" + - aggregate_on_attributes("sum", []) where metric.name == "process.open_handles" # --------------------------------------------------------------------------- # Detect host.name. @@ -207,6 +346,16 @@ processors: datapoint.attributes["state"] ) where metric.name == "system.cpu.utilization" and datapoint.attributes["state"] != nil + - set( + datapoint.attributes["mode"], + "irq" + ) where metric.name == "system.cpu.utilization" and datapoint.attributes["mode"] == "interrupt" + + - set( + datapoint.attributes["mode"], + "iowait" + ) where metric.name == "system.cpu.utilization" and datapoint.attributes["mode"] == "wait" + - delete_key( datapoint.attributes, "state" @@ -292,19 +441,18 @@ processors: # --------------------------------------------------------------------------- - # Prepare process metrics for aggregation. + # Prepare individual process metrics for Collector-side aggregation. # # Linux FD: # process.open_file_descriptors -> process.open_handles # # Process count: - # create one process.count sample for each process by copying the - # process.threads metric. Later aggregate_on_attributes("count", []) - # counts those samples. + # create one process.count=1 sample per operating-system process. The + # following groupbyattrs and transform processors aggregate these samples + # by normalized process identity before OTLP export. # --------------------------------------------------------------------------- transform/process-prepare-linux: error_mode: ignore - metric_statements: - context: metric @@ -317,9 +465,16 @@ processors: - copy_metric( name="process.count", - description="Number of processes in the normalized process group" + description="One sample per operating-system process", + unit="{process}" ) where metric.name == "process.threads" + - context: datapoint + statements: + - set( + value_int, + 1 + ) where metric.name == "process.count" # --------------------------------------------------------------------------- # Batch host telemetry. @@ -329,95 +484,6 @@ processors: send_batch_size: 8192 - # --------------------------------------------------------------------------- - # Batch individual process telemetry before grouping. - # --------------------------------------------------------------------------- - batch/process-linux: - timeout: 2s - send_batch_size: 8192 - - - # --------------------------------------------------------------------------- - # Collapse ResourceMetrics belonging to the same normalized process group. - # - # process_name remains a resource attribute and therefore remains the - # process group identity. - # - # PID is not present. - # --------------------------------------------------------------------------- - groupbyattrs/process-linux: - keys: [] - - - # --------------------------------------------------------------------------- - # Aggregate all PIDs belonging to the same process_name. - # - # For the E2E workload: - # - # sleep PID A - # sleep PID B - # sleep PID C - # - # becomes one logical process group: - # - # process_name = sleep - # - # with: - # - # process.count = 3 - # - # --------------------------------------------------------------------------- - transform/process-aggregate-linux: - error_mode: ignore - - metric_statements: - - - context: metric - statements: - - # Number of actual processes in this normalized group. - - aggregate_on_attributes( - "count", - [] - ) where metric.name == "process.count" - - # Total threads across the group. - - aggregate_on_attributes( - "sum", - [] - ) where metric.name == "process.threads" - - # Preserve CPU state while aggregating all processes. - - aggregate_on_attributes( - "sum", - ["state"] - ) where metric.name == "process.cpu.utilization" - - # Total resident memory. - - aggregate_on_attributes( - "sum", - [] - ) where metric.name == "process.memory.usage" - - # Aggregate process memory utilization. - - aggregate_on_attributes( - "sum", - [] - ) where metric.name == "process.memory.utilization" - - # Total open file descriptors. - - aggregate_on_attributes( - "sum", - [] - ) where metric.name == "process.open_handles" - - # Oldest process in the group. - - aggregate_on_attributes( - "max", - [] - ) where metric.name == "process.uptime" - - # --------------------------------------------------------------------------- # Final batching before OTLP export. # --------------------------------------------------------------------------- @@ -428,6 +494,26 @@ processors: exporters: + # Detailed post-processor Collector output is asserted by + # assert-collector-output.sh before relying on OAP/MAL results. + debug/process-linux-raw: + verbosity: detailed + use_internal_logger: false + output_paths: + - /tmp/process-linux-raw.log + + debug/windows-host-raw: + verbosity: detailed + use_internal_logger: false + output_paths: + - /tmp/windows-host-raw.log + + debug/windows-process-raw: + verbosity: detailed + use_internal_logger: false + output_paths: + - /tmp/windows-process-raw.log + # --------------------------------------------------------------------------- # OAP service defined in docker-compose.yml. # @@ -465,19 +551,44 @@ service: # Process telemetry -> process-hostmetrics-linux.yaml # ------------------------------------------------------------------------- metrics/processes-linux: - receivers: - hostmetrics/processes - processors: - resource_detection/system - transform/process-identity-linux - filter/process-without-name-linux - transform/process-prepare-linux - - batch/process-linux - groupbyattrs/process-linux - transform/process-aggregate-linux - batch/export + exporters: + - otlp + - debug/process-linux-raw + # ------------------------------------------------------------------------- + # Synthetic Windows host OTLP fixture -> windows.yaml + # ------------------------------------------------------------------------- + metrics/windows-host-fixture: + receivers: + - otlp/windows-fixture-replay + processors: + - transform/host-windows + exporters: + - otlp + - debug/windows-host-raw + + # ------------------------------------------------------------------------- + # Synthetic Windows process OTLP fixture -> process-hostmetrics-windows.yaml + # ------------------------------------------------------------------------- + metrics/windows-process-fixture: + receivers: + - otlp/windows-fixture-replay + processors: + - transform/process-identity-windows + - filter/process-without-name-windows + - transform/process-prepare-windows + - groupbyattrs/process-windows + - transform/process-aggregate-windows exporters: - otlp + - debug/windows-process-raw diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/process-hostmetrics-linux.yaml b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/process-hostmetrics-linux.yaml index 61912ca9807f..71356da371b6 100644 --- a/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/process-hostmetrics-linux.yaml +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/process-hostmetrics-linux.yaml @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Process metrics are pre-aggregated by normalized process_name in the -# Collector to bound PID cardinality. MAL performs the final SkyWalking -# service-instance mapping and metric calculation. +# The Collector normalizes process identity and pre-aggregates operating-system +# processes with the same process_name before OTLP export. MAL maps the +# resulting logical process metrics to the corresponding SkyWalking instance. filter: "{ tags -> tags.job_name == 'hostmetrics-process-monitoring-linux' }" expSuffix: |- @@ -67,7 +67,10 @@ metricsRules: - name: memory_resident_bytes exp: "process_memory_usage.sum(['node_identifier_host_name','process_name'])" - # OTel process.memory.utilization uses unit 1 (fraction), so expose % here. + # process.memory.utilization is emitted as a [0,1] fraction. Multiply by + # 100 before SkyWalking fixed-point storage to preserve two decimal places + # of percentage precision (basis points). UI expressions must divide the + # stored metric by 100. - name: memory_resident_percent exp: > process_memory_utilization diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/process-hostmetrics-windows.yaml b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/process-hostmetrics-windows.yaml index c2f299f779d6..bde9f5b73580 100644 --- a/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/process-hostmetrics-windows.yaml +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/process-hostmetrics-windows.yaml @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Process metrics are pre-aggregated by normalized process_name in the -# Collector to bound PID cardinality. MAL performs the final SkyWalking -# service-instance mapping and metric calculation. +# The Collector normalizes process identity and pre-aggregates operating-system +# processes with the same process_name before OTLP export. MAL maps the +# resulting logical process metrics to the corresponding SkyWalking instance. filter: "{ tags -> tags.job_name == 'hostmetrics-process-monitoring-windows' }" expSuffix: |- @@ -67,7 +67,10 @@ metricsRules: - name: memory_resident_bytes exp: "process_memory_usage.sum(['node_identifier_host_name','process_name'])" - # OTel process.memory.utilization uses unit 1 (fraction), so expose % here. + # process.memory.utilization is emitted as a [0,1] fraction. Multiply by + # 100 before SkyWalking fixed-point storage to preserve two decimal places + # of percentage precision (basis points). UI expressions must divide the + # stored metric by 100. - name: memory_resident_percent exp: > process_memory_utilization diff --git a/test/e2e-v2/cases/vm/prometheus-node-exporter/otel-rules/vm.yaml b/test/e2e-v2/cases/vm/prometheus-node-exporter/otel-rules/vm.yaml index 4e1e1865b1a8..3b50da8afafd 100644 --- a/test/e2e-v2/cases/vm/prometheus-node-exporter/otel-rules/vm.yaml +++ b/test/e2e-v2/cases/vm/prometheus-node-exporter/otel-rules/vm.yaml @@ -27,7 +27,6 @@ # one source family or the other, each expression can safely add the equivalent # node-exporter and hostmetrics branches. The output metric is registered once. filter: "{ tags -> tags.job_name == 'vm-monitoring' }" -expSuffix: service(['node_identifier_host_name'], Layer.OS_LINUX) metricPrefix: meter_vm metricsRules: @@ -38,6 +37,7 @@ metricsRules: # CPU - name: cpu_total_percentage exp: > + ( (node_cpu_seconds_total * 100) .tagNotEqual('mode','idle') .sum(['node_identifier_host_name']) @@ -46,59 +46,71 @@ metricsRules: (system_cpu_utilization * 100) .tagNotEqual('mode','idle') .sum(['node_identifier_host_name']) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: cpu_average_used exp: > + ( (node_cpu_seconds_total * 100) .sum(['node_identifier_host_name','mode']) .rate('PT1M') + (system_cpu_utilization * 100) .sum(['node_identifier_host_name','mode']) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: cpu_load1 - exp: "(node_load1 * 100) + (system_cpu_load_average_1m * 100)" + exp: "((node_load1 * 100) + (system_cpu_load_average_1m * 100)).service(['node_identifier_host_name'], Layer.OS_LINUX).decorate({ me -> me.attr0 = me.layer.name()})" - name: cpu_load5 - exp: "(node_load5 * 100) + (system_cpu_load_average_5m * 100)" + exp: "((node_load5 * 100) + (system_cpu_load_average_5m * 100)).service(['node_identifier_host_name'], Layer.OS_LINUX)" - name: cpu_load15 - exp: "(node_load15 * 100) + (system_cpu_load_average_15m * 100)" + exp: "((node_load15 * 100) + (system_cpu_load_average_15m * 100)).service(['node_identifier_host_name'], Layer.OS_LINUX)" # Physical memory - name: memory_total - exp: "node_memory_MemTotal_bytes + system_memory_limit" + exp: "(node_memory_MemTotal_bytes + system_memory_limit).service(['node_identifier_host_name'], Layer.OS_LINUX)" - name: memory_available - exp: "node_memory_MemAvailable_bytes + system_linux_memory_available" + exp: "(node_memory_MemAvailable_bytes + system_linux_memory_available).service(['node_identifier_host_name'], Layer.OS_LINUX)" - name: memory_used exp: > + ( (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) + (system_memory_limit - system_linux_memory_available) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: memory_buff_cache exp: > + ( (node_memory_Buffers_bytes + node_memory_Cached_bytes) + ( system_memory_usage.tagEqual('state','buffered').sum(['node_identifier_host_name']) + system_memory_usage.tagEqual('state','cached').sum(['node_identifier_host_name']) + - + system_memory_usage.tagEqual('state','slab_reclaimable').sum(['node_identifier_host_name']) ) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) # Swap - name: memory_swap_free exp: > + ( node_memory_SwapFree_bytes + system_paging_usage .tagEqual('state','free') .sum(['node_identifier_host_name']) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: memory_swap_total exp: > + ( node_memory_SwapTotal_bytes + ( @@ -106,9 +118,11 @@ metricsRules: + system_paging_usage.tagEqual('state','free').sum(['node_identifier_host_name']) ) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: memory_swap_percentage exp: > + ( ( 100 - ((node_memory_SwapFree_bytes * 100) / node_memory_SwapTotal_bytes) ) @@ -125,10 +139,12 @@ metricsRules: system_paging_usage.tagEqual('state','free').sum(['node_identifier_host_name']) ) ) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) # Filesystem. hostmetrics free+reserved+used is the full capacity. - name: filesystem_percentage exp: > + ( ( 100 - ( (node_filesystem_avail_bytes * 100) @@ -156,10 +172,12 @@ metricsRules: ) ) ) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) # Disk throughput - name: disk_read exp: > + ( node_disk_read_bytes_total .sum(['node_identifier_host_name']) .rate('PT1M') @@ -168,9 +186,11 @@ metricsRules: .tagEqual('direction','read') .sum(['node_identifier_host_name']) .rate('PT1M') + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: disk_written exp: > + ( node_disk_written_bytes_total .sum(['node_identifier_host_name']) .rate('PT1M') @@ -179,10 +199,12 @@ metricsRules: .tagEqual('direction','write') .sum(['node_identifier_host_name']) .rate('PT1M') + ).service(['node_identifier_host_name'], Layer.OS_LINUX) # Network throughput - name: network_receive exp: > + ( node_network_receive_bytes_total .sum(['node_identifier_host_name']) .irate() @@ -191,9 +213,11 @@ metricsRules: .tagEqual('direction','receive') .sum(['node_identifier_host_name']) .irate() + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: network_transmit exp: > + ( node_network_transmit_bytes_total .sum(['node_identifier_host_name']) .irate() @@ -202,26 +226,38 @@ metricsRules: .tagEqual('direction','transmit') .sum(['node_identifier_host_name']) .irate() + ).service(['node_identifier_host_name'], Layer.OS_LINUX) # TCP connection metrics with true hostmetrics equivalents. # Collector normalizes system.network.connections state values to lowercase. - name: tcp_curr_estab exp: > + ( node_netstat_Tcp_CurrEstab + - system_network_connections - .tagEqual('protocol','tcp') - .tagEqual('state','established') - .sum(['node_identifier_host_name']) + ( + system_network_connections + .tagEqual('protocol','tcp') + .tagEqual('state','established') + .sum(['node_identifier_host_name']) + + + system_network_connections + .tagEqual('protocol','tcp') + .tagEqual('state','close_wait') + .sum(['node_identifier_host_name']) + ) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: tcp_tw exp: > + ( node_sockstat_TCP_tw + system_network_connections .tagEqual('protocol','tcp') .tagEqual('state','time_wait') .sum(['node_identifier_host_name']) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) # --------------------------------------------------------------------------- # Existing node-exporter-only canonical metrics. @@ -229,26 +265,27 @@ metricsRules: # exporter only rather than being populated with a misleading approximation. # --------------------------------------------------------------------------- - name: tcp_alloc - exp: node_sockstat_TCP_alloc + exp: (node_sockstat_TCP_alloc).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: sockets_used - exp: node_sockstat_sockets_used + exp: (node_sockstat_sockets_used).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: udp_inuse - exp: node_sockstat_UDP_inuse + exp: (node_sockstat_UDP_inuse).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: filefd_allocated - exp: node_filefd_allocated + exp: (node_filefd_allocated).service(['node_identifier_host_name'], Layer.OS_LINUX) # --------------------------------------------------------------------------- # New hostmetrics-only capabilities. New names are used whenever semantics # do not exactly match an existing node-exporter target metric. # --------------------------------------------------------------------------- - name: cpu_cores_num - exp: system_cpu_logical_count.sum(['node_identifier_host_name']) + exp: (system_cpu_logical_count.sum(['node_identifier_host_name'])).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: cpu_norm_percentage exp: > + ( ( (system_cpu_utilization * 100) .tagNotEqual('mode','idle') @@ -256,3 +293,4 @@ metricsRules: ) / system_cpu_logical_count.sum(['node_identifier_host_name']) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) From 5fb8cf62ea2206f7e2b302ab55e91fd470572359 Mon Sep 17 00:00:00 2001 From: mike-realuptime Date: Fri, 4 Sep 2026 21:00:04 +0000 Subject: [PATCH 4/4] Fix license headers and sync hostmetrics E2E rules --- .../otel-collector-hostmetrics-linux.yaml | 19 ++++-- .../otel-collector-hostmetrics-windows.yaml | 14 +++- .../assert-collector-output.sh | 14 ++++ .../vm/otel-hostmetrics/otel-rules/vm.yaml | 68 +++++++++++++++---- 4 files changed, 92 insertions(+), 23 deletions(-) diff --git a/docs/en/setup/backend/otel-collector-hostmetrics-linux.yaml b/docs/en/setup/backend/otel-collector-hostmetrics-linux.yaml index 37308a9e1d10..942249dbf7bb 100644 --- a/docs/en/setup/backend/otel-collector-hostmetrics-linux.yaml +++ b/docs/en/setup/backend/otel-collector-hostmetrics-linux.yaml @@ -1,10 +1,17 @@ # Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with +# contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0. -# -# Reference configuration for OpenTelemetry Collector Contrib 0.158.0. -# Native Linux host + process monitoring for SkyWalking OAP. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # Environment variables: # SW_OAP_GRPC_ADDRESS OAP OTLP/gRPC endpoint (default 127.0.0.1:11800) @@ -80,7 +87,7 @@ receivers: process.command: enabled: false process.command_line: - enabled: false + enabled: false process.owner: enabled: false process.pid: diff --git a/docs/en/setup/backend/otel-collector-hostmetrics-windows.yaml b/docs/en/setup/backend/otel-collector-hostmetrics-windows.yaml index 7371772b5aa6..30daa1672597 100644 --- a/docs/en/setup/backend/otel-collector-hostmetrics-windows.yaml +++ b/docs/en/setup/backend/otel-collector-hostmetrics-windows.yaml @@ -1,7 +1,17 @@ # Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with +# contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # Reference configuration for OpenTelemetry Collector Contrib 0.158.0. # Native Windows host + process monitoring for SkyWalking OAP. diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/assert-collector-output.sh b/test/e2e-v2/cases/vm/otel-hostmetrics/assert-collector-output.sh index b36b8ee373ab..f3e01de4d6d1 100755 --- a/test/e2e-v2/cases/vm/otel-hostmetrics/assert-collector-output.sh +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/assert-collector-output.sh @@ -1,4 +1,18 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. set -euo pipefail CID="$( diff --git a/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/vm.yaml b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/vm.yaml index 4e1e1865b1a8..3b50da8afafd 100644 --- a/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/vm.yaml +++ b/test/e2e-v2/cases/vm/otel-hostmetrics/otel-rules/vm.yaml @@ -27,7 +27,6 @@ # one source family or the other, each expression can safely add the equivalent # node-exporter and hostmetrics branches. The output metric is registered once. filter: "{ tags -> tags.job_name == 'vm-monitoring' }" -expSuffix: service(['node_identifier_host_name'], Layer.OS_LINUX) metricPrefix: meter_vm metricsRules: @@ -38,6 +37,7 @@ metricsRules: # CPU - name: cpu_total_percentage exp: > + ( (node_cpu_seconds_total * 100) .tagNotEqual('mode','idle') .sum(['node_identifier_host_name']) @@ -46,59 +46,71 @@ metricsRules: (system_cpu_utilization * 100) .tagNotEqual('mode','idle') .sum(['node_identifier_host_name']) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: cpu_average_used exp: > + ( (node_cpu_seconds_total * 100) .sum(['node_identifier_host_name','mode']) .rate('PT1M') + (system_cpu_utilization * 100) .sum(['node_identifier_host_name','mode']) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: cpu_load1 - exp: "(node_load1 * 100) + (system_cpu_load_average_1m * 100)" + exp: "((node_load1 * 100) + (system_cpu_load_average_1m * 100)).service(['node_identifier_host_name'], Layer.OS_LINUX).decorate({ me -> me.attr0 = me.layer.name()})" - name: cpu_load5 - exp: "(node_load5 * 100) + (system_cpu_load_average_5m * 100)" + exp: "((node_load5 * 100) + (system_cpu_load_average_5m * 100)).service(['node_identifier_host_name'], Layer.OS_LINUX)" - name: cpu_load15 - exp: "(node_load15 * 100) + (system_cpu_load_average_15m * 100)" + exp: "((node_load15 * 100) + (system_cpu_load_average_15m * 100)).service(['node_identifier_host_name'], Layer.OS_LINUX)" # Physical memory - name: memory_total - exp: "node_memory_MemTotal_bytes + system_memory_limit" + exp: "(node_memory_MemTotal_bytes + system_memory_limit).service(['node_identifier_host_name'], Layer.OS_LINUX)" - name: memory_available - exp: "node_memory_MemAvailable_bytes + system_linux_memory_available" + exp: "(node_memory_MemAvailable_bytes + system_linux_memory_available).service(['node_identifier_host_name'], Layer.OS_LINUX)" - name: memory_used exp: > + ( (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) + (system_memory_limit - system_linux_memory_available) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: memory_buff_cache exp: > + ( (node_memory_Buffers_bytes + node_memory_Cached_bytes) + ( system_memory_usage.tagEqual('state','buffered').sum(['node_identifier_host_name']) + system_memory_usage.tagEqual('state','cached').sum(['node_identifier_host_name']) + - + system_memory_usage.tagEqual('state','slab_reclaimable').sum(['node_identifier_host_name']) ) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) # Swap - name: memory_swap_free exp: > + ( node_memory_SwapFree_bytes + system_paging_usage .tagEqual('state','free') .sum(['node_identifier_host_name']) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: memory_swap_total exp: > + ( node_memory_SwapTotal_bytes + ( @@ -106,9 +118,11 @@ metricsRules: + system_paging_usage.tagEqual('state','free').sum(['node_identifier_host_name']) ) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: memory_swap_percentage exp: > + ( ( 100 - ((node_memory_SwapFree_bytes * 100) / node_memory_SwapTotal_bytes) ) @@ -125,10 +139,12 @@ metricsRules: system_paging_usage.tagEqual('state','free').sum(['node_identifier_host_name']) ) ) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) # Filesystem. hostmetrics free+reserved+used is the full capacity. - name: filesystem_percentage exp: > + ( ( 100 - ( (node_filesystem_avail_bytes * 100) @@ -156,10 +172,12 @@ metricsRules: ) ) ) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) # Disk throughput - name: disk_read exp: > + ( node_disk_read_bytes_total .sum(['node_identifier_host_name']) .rate('PT1M') @@ -168,9 +186,11 @@ metricsRules: .tagEqual('direction','read') .sum(['node_identifier_host_name']) .rate('PT1M') + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: disk_written exp: > + ( node_disk_written_bytes_total .sum(['node_identifier_host_name']) .rate('PT1M') @@ -179,10 +199,12 @@ metricsRules: .tagEqual('direction','write') .sum(['node_identifier_host_name']) .rate('PT1M') + ).service(['node_identifier_host_name'], Layer.OS_LINUX) # Network throughput - name: network_receive exp: > + ( node_network_receive_bytes_total .sum(['node_identifier_host_name']) .irate() @@ -191,9 +213,11 @@ metricsRules: .tagEqual('direction','receive') .sum(['node_identifier_host_name']) .irate() + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: network_transmit exp: > + ( node_network_transmit_bytes_total .sum(['node_identifier_host_name']) .irate() @@ -202,26 +226,38 @@ metricsRules: .tagEqual('direction','transmit') .sum(['node_identifier_host_name']) .irate() + ).service(['node_identifier_host_name'], Layer.OS_LINUX) # TCP connection metrics with true hostmetrics equivalents. # Collector normalizes system.network.connections state values to lowercase. - name: tcp_curr_estab exp: > + ( node_netstat_Tcp_CurrEstab + - system_network_connections - .tagEqual('protocol','tcp') - .tagEqual('state','established') - .sum(['node_identifier_host_name']) + ( + system_network_connections + .tagEqual('protocol','tcp') + .tagEqual('state','established') + .sum(['node_identifier_host_name']) + + + system_network_connections + .tagEqual('protocol','tcp') + .tagEqual('state','close_wait') + .sum(['node_identifier_host_name']) + ) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: tcp_tw exp: > + ( node_sockstat_TCP_tw + system_network_connections .tagEqual('protocol','tcp') .tagEqual('state','time_wait') .sum(['node_identifier_host_name']) + ).service(['node_identifier_host_name'], Layer.OS_LINUX) # --------------------------------------------------------------------------- # Existing node-exporter-only canonical metrics. @@ -229,26 +265,27 @@ metricsRules: # exporter only rather than being populated with a misleading approximation. # --------------------------------------------------------------------------- - name: tcp_alloc - exp: node_sockstat_TCP_alloc + exp: (node_sockstat_TCP_alloc).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: sockets_used - exp: node_sockstat_sockets_used + exp: (node_sockstat_sockets_used).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: udp_inuse - exp: node_sockstat_UDP_inuse + exp: (node_sockstat_UDP_inuse).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: filefd_allocated - exp: node_filefd_allocated + exp: (node_filefd_allocated).service(['node_identifier_host_name'], Layer.OS_LINUX) # --------------------------------------------------------------------------- # New hostmetrics-only capabilities. New names are used whenever semantics # do not exactly match an existing node-exporter target metric. # --------------------------------------------------------------------------- - name: cpu_cores_num - exp: system_cpu_logical_count.sum(['node_identifier_host_name']) + exp: (system_cpu_logical_count.sum(['node_identifier_host_name'])).service(['node_identifier_host_name'], Layer.OS_LINUX) - name: cpu_norm_percentage exp: > + ( ( (system_cpu_utilization * 100) .tagNotEqual('mode','idle') @@ -256,3 +293,4 @@ metricsRules: ) / system_cpu_logical_count.sum(['node_identifier_host_name']) + ).service(['node_identifier_host_name'], Layer.OS_LINUX)