From 27ef235a435de052a29fafe2dafbe32673836094 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Tue, 28 Jul 2026 14:24:00 +0200 Subject: [PATCH 1/2] Enable alert rendering for egress gateway test case --- tests/egress-gateway.yml | 3 +- .../templates/cilium-agent/daemonset.yaml | 2 - .../templates/cilium-agent/service.yaml | 19 ++++++++ .../cilium-agent/servicemonitor.yaml | 28 ++++++++++++ .../templates/hubble/metrics-service.yaml | 4 +- .../templates/hubble/servicemonitor.yaml | 27 +++++++++++ .../cilium/cilium/10_ebpf_alerts.yaml | 45 +++++++++++++++++++ .../cilium/10_egress_gateway_alerts.yaml | 27 +++++++++++ .../cilium/cilium/10_pods_alerts.yaml | 23 ++++++++++ 9 files changed, 172 insertions(+), 6 deletions(-) create mode 100644 tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-agent/service.yaml create mode 100644 tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-agent/servicemonitor.yaml create mode 100644 tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/hubble/servicemonitor.yaml create mode 100644 tests/golden/egress-gateway/cilium/cilium/10_ebpf_alerts.yaml create mode 100644 tests/golden/egress-gateway/cilium/cilium/10_egress_gateway_alerts.yaml create mode 100644 tests/golden/egress-gateway/cilium/cilium/10_pods_alerts.yaml diff --git a/tests/egress-gateway.yml b/tests/egress-gateway.yml index 0a7c636c6..1c1651e98 100644 --- a/tests/egress-gateway.yml +++ b/tests/egress-gateway.yml @@ -1,6 +1,7 @@ # Overwrite parameters here applications: - openshift-nmstate + - prometheus parameters: kapitan: @@ -10,7 +11,7 @@ parameters: output_path: vendor/lib/espejote.libsonnet - type: https source: https://raw.githubusercontent.com/appuio/component-openshift4-monitoring/v6.11.3/lib/openshift4-monitoring-prom.libsonnet - output_path: vendor/lib/prom.libsonnet + output_path: vendor/lib/prometheus.libsonnet - type: https source: https://raw.githubusercontent.com/appuio/component-openshift4-monitoring/v6.11.3/lib/openshift4-monitoring-alert-patching.libsonnet output_path: vendor/lib/alert-patching.libsonnet diff --git a/tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-agent/daemonset.yaml b/tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-agent/daemonset.yaml index f5c9a0e30..feed3f539 100644 --- a/tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-agent/daemonset.yaml +++ b/tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-agent/daemonset.yaml @@ -19,8 +19,6 @@ spec: container.apparmor.security.beta.kubernetes.io/clean-cilium-state: unconfined container.apparmor.security.beta.kubernetes.io/mount-cgroup: unconfined kubectl.kubernetes.io/default-container: cilium-agent - prometheus.io/port: '9962' - prometheus.io/scrape: 'true' labels: app.kubernetes.io/name: cilium-agent app.kubernetes.io/part-of: cilium diff --git a/tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-agent/service.yaml b/tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-agent/service.yaml new file mode 100644 index 000000000..fbab0df5d --- /dev/null +++ b/tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-agent/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: cilium-agent + app.kubernetes.io/part-of: cilium + k8s-app: cilium + name: cilium-agent + namespace: cilium +spec: + clusterIP: None + ports: + - name: metrics + port: 9962 + protocol: TCP + targetPort: prometheus + selector: + k8s-app: cilium + type: ClusterIP diff --git a/tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-agent/servicemonitor.yaml b/tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-agent/servicemonitor.yaml new file mode 100644 index 000000000..7eff56171 --- /dev/null +++ b/tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-agent/servicemonitor.yaml @@ -0,0 +1,28 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app.kubernetes.io/name: cilium-agent + app.kubernetes.io/part-of: cilium + name: cilium-agent + namespace: cilium +spec: + endpoints: + - honorLabels: true + interval: 10s + path: /metrics + port: metrics + relabelings: + - action: replace + replacement: ${1} + sourceLabels: + - __meta_kubernetes_pod_node_name + targetLabel: node + namespaceSelector: + matchNames: + - cilium + selector: + matchLabels: + app.kubernetes.io/name: cilium-agent + targetLabels: + - k8s-app diff --git a/tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/hubble/metrics-service.yaml b/tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/hubble/metrics-service.yaml index a3604b96c..ebeaf0fdf 100644 --- a/tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/hubble/metrics-service.yaml +++ b/tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/hubble/metrics-service.yaml @@ -1,9 +1,7 @@ apiVersion: v1 kind: Service metadata: - annotations: - prometheus.io/port: '9965' - prometheus.io/scrape: 'true' + annotations: null labels: app.kubernetes.io/name: hubble app.kubernetes.io/part-of: cilium diff --git a/tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/hubble/servicemonitor.yaml b/tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/hubble/servicemonitor.yaml new file mode 100644 index 000000000..15776d4a8 --- /dev/null +++ b/tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/hubble/servicemonitor.yaml @@ -0,0 +1,27 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app.kubernetes.io/name: hubble + app.kubernetes.io/part-of: cilium + name: hubble + namespace: cilium +spec: + endpoints: + - honorLabels: true + interval: 10s + path: /metrics + port: hubble-metrics + relabelings: + - action: replace + replacement: ${1} + sourceLabels: + - __meta_kubernetes_pod_node_name + targetLabel: node + scheme: http + namespaceSelector: + matchNames: + - cilium + selector: + matchLabels: + k8s-app: hubble diff --git a/tests/golden/egress-gateway/cilium/cilium/10_ebpf_alerts.yaml b/tests/golden/egress-gateway/cilium/cilium/10_ebpf_alerts.yaml new file mode 100644 index 000000000..b44af4391 --- /dev/null +++ b/tests/golden/egress-gateway/cilium/cilium/10_ebpf_alerts.yaml @@ -0,0 +1,45 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + annotations: {} + labels: + name: cilium-ebpf + name: cilium-ebpf +spec: + groups: + - name: cilium-ebpf.rules + rules: + - alert: CiliumBpfMapUtilizationHigh + annotations: + description: | + BPF map utilization for map {{ $labels.map_name }} has been above + 50% on node {{ $labels.node }} for the last 10m. + message: High BPF map utilization on {{ $labels.node }} + runbook_url: https://hub.syn.tools/cilium/runbooks/CiliumBpfMapPressureHigh.html + expr: cilium_bpf_map_pressure > 0.5 + for: 10m + labels: + severity: warning + - alert: CiliumBpfMapUtilizationExtremelyHigh + annotations: + description: | + BPF map utilization for map {{ $labels.map_name }} has been above + 90% on node {{ $labels.node }} for the last 10m. + message: Extremely High BPF map utilization on {{ $labels.node }} + runbook_url: https://hub.syn.tools/cilium/runbooks/CiliumBpfMapPressureExtremelyHigh.html + expr: cilium_bpf_map_pressure > 0.9 + for: 10m + labels: + severity: critical + - alert: CiliumBpfOperationErrorRateHigh + annotations: + description: | + BPF error rate for map {{ $labels.map_name }} has been above + 50% on node {{ $labels.node }} for the last 10m. + message: High BPF error rate on {{ $labels.node }} + runbook_url: https://hub.syn.tools/cilium/runbooks/CiliumBpfOperationErrorRateHigh.html + expr: (rate(cilium_bpf_map_ops_total{outcome="fail"}[1m]) / rate(cilium_bpf_map_ops_total{}[1m])) + > 0.5 + for: 10m + labels: + severity: critical diff --git a/tests/golden/egress-gateway/cilium/cilium/10_egress_gateway_alerts.yaml b/tests/golden/egress-gateway/cilium/cilium/10_egress_gateway_alerts.yaml new file mode 100644 index 000000000..68d5c90fa --- /dev/null +++ b/tests/golden/egress-gateway/cilium/cilium/10_egress_gateway_alerts.yaml @@ -0,0 +1,27 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + annotations: {} + labels: + name: cilium-egress-gateway + name: cilium-egress-gateway +spec: + groups: + - name: cilium-egress-gateway.rules + rules: + - alert: CiliumShadowRangeNodeMissing + annotations: + description: | + The cluster should have 3 nodes which host a shadow egress + IP range, but had {{ $value }} for the last 5m. Most likely + one or more nodes were replaced but the shadow range was not + updated. + + The shadow range configmap currently has nodes infra-8344, infra-87c9, infra-eba2. + message: A node which hosts a shadow egress IP range is missing + runbook_url: https://hub.syn.tools/cilium/CiliumShadowRangeNodeMissing.html + expr: sum (kube_node_info{node=~"infra-8344|infra-87c9|infra-eba2"}) != + 3 + for: 5m + labels: + severity: critical diff --git a/tests/golden/egress-gateway/cilium/cilium/10_pods_alerts.yaml b/tests/golden/egress-gateway/cilium/cilium/10_pods_alerts.yaml new file mode 100644 index 000000000..14cd7c717 --- /dev/null +++ b/tests/golden/egress-gateway/cilium/cilium/10_pods_alerts.yaml @@ -0,0 +1,23 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + annotations: {} + labels: + name: cilium-pods + name: cilium-pods +spec: + groups: + - name: cilium-pods.rules + rules: + - alert: CiliumAgentUnexpectedCount + annotations: + description: | + The cluster has had {{ $value }} Cilium agents for the last 5m + instead of the expected {{ query "count(kube_node_info)" }}. + message: Number of Cilium agents doesn't match node count + runbook_url: https://hub.syn.tools/cilium/runbooks/CiliumAgentUnexpectedCount.html + expr: count(kube_pod_labels{namespace="cilium", label_app_kubernetes_io_name="cilium-agent"}) + != count(kube_node_info) + for: 5m + labels: + severity: critical From b09eb3e6e4be0232980c45e1b06f5046c712bd8b Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Tue, 28 Jul 2026 14:24:52 +0200 Subject: [PATCH 2/2] Fix alert expression for egress shadow range node count alert --- component/alerts.jsonnet | 2 +- .../cilium/cilium/10_egress_gateway_alerts.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/component/alerts.jsonnet b/component/alerts.jsonnet index 130b6f608..496ce5096 100644 --- a/component/alerts.jsonnet +++ b/component/alerts.jsonnet @@ -201,7 +201,7 @@ local egw_group = local this = self, alert: 'CiliumShadowRangeNodeMissing', expr: - 'sum (kube_node_info{node=~"%(sel)s"}) != %(count)s' + '(sum (kube_node_info{node=~"%(sel)s"}) or vector(0)) != %(count)s' % { sel: std.join('|', std.objectFields(egw_shadow_ranges.config)), count: std.length(egw_shadow_ranges.config), diff --git a/tests/golden/egress-gateway/cilium/cilium/10_egress_gateway_alerts.yaml b/tests/golden/egress-gateway/cilium/cilium/10_egress_gateway_alerts.yaml index 68d5c90fa..5a607aa3b 100644 --- a/tests/golden/egress-gateway/cilium/cilium/10_egress_gateway_alerts.yaml +++ b/tests/golden/egress-gateway/cilium/cilium/10_egress_gateway_alerts.yaml @@ -20,8 +20,8 @@ spec: The shadow range configmap currently has nodes infra-8344, infra-87c9, infra-eba2. message: A node which hosts a shadow egress IP range is missing runbook_url: https://hub.syn.tools/cilium/CiliumShadowRangeNodeMissing.html - expr: sum (kube_node_info{node=~"infra-8344|infra-87c9|infra-eba2"}) != - 3 + expr: (sum (kube_node_info{node=~"infra-8344|infra-87c9|infra-eba2"}) or + vector(0)) != 3 for: 5m labels: severity: critical