Fix alert expression for egress shadow range node count alert

component/alerts.jsonnet

@@ -201,7 +201,7 @@ local egw_group =
         local this = self,
         alert: 'CiliumShadowRangeNodeMissing',
         expr:
-          'sum (kube_node_info{node=~"%(sel)s"}) != %(count)s'
+          '(sum (kube_node_info{node=~"%(sel)s"}) or vector(0)) != %(count)s'
           % {
             sel: std.join('|', std.objectFields(egw_shadow_ranges.config)),
             count: std.length(egw_shadow_ranges.config),

tests/egress-gateway.yml

@@ -1,6 +1,7 @@
 # Overwrite parameters here
 applications:
   - openshift-nmstate
+  - prometheus
 
 parameters:
   kapitan:
@@ -10,7 +11,7 @@ parameters:
         output_path: vendor/lib/espejote.libsonnet
       - type: https
         source: https://raw.githubusercontent.com/appuio/component-openshift4-monitoring/v6.11.3/lib/openshift4-monitoring-prom.libsonnet
-        output_path: vendor/lib/prom.libsonnet
+        output_path: vendor/lib/prometheus.libsonnet
       - type: https
         source: https://raw.githubusercontent.com/appuio/component-openshift4-monitoring/v6.11.3/lib/openshift4-monitoring-alert-patching.libsonnet
         output_path: vendor/lib/alert-patching.libsonnet

tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-agent/daemonset.yaml

@@ -19,8 +19,6 @@ spec:
         container.apparmor.security.beta.kubernetes.io/clean-cilium-state: unconfined
         container.apparmor.security.beta.kubernetes.io/mount-cgroup: unconfined
         kubectl.kubernetes.io/default-container: cilium-agent
-        prometheus.io/port: '9962'
-        prometheus.io/scrape: 'true'
       labels:
         app.kubernetes.io/name: cilium-agent
         app.kubernetes.io/part-of: cilium

tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-agent/service.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: cilium-agent
+    app.kubernetes.io/part-of: cilium
+    k8s-app: cilium
+  name: cilium-agent
+  namespace: cilium
+spec:
+  clusterIP: None
+  ports:
+    - name: metrics
+      port: 9962
+      protocol: TCP
+      targetPort: prometheus
+  selector:
+    k8s-app: cilium
+  type: ClusterIP

tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-agent/servicemonitor.yaml

@@ -0,0 +1,28 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/name: cilium-agent
+    app.kubernetes.io/part-of: cilium
+  name: cilium-agent
+  namespace: cilium
+spec:
+  endpoints:
+    - honorLabels: true
+      interval: 10s
+      path: /metrics
+      port: metrics
+      relabelings:
+        - action: replace
+          replacement: ${1}
+          sourceLabels:
+            - __meta_kubernetes_pod_node_name
+          targetLabel: node
+  namespaceSelector:
+    matchNames:
+      - cilium
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: cilium-agent
+  targetLabels:
+    - k8s-app

tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/hubble/metrics-service.yaml

@@ -1,9 +1,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  annotations:
-    prometheus.io/port: '9965'
-    prometheus.io/scrape: 'true'
+  annotations: null
   labels:
     app.kubernetes.io/name: hubble
     app.kubernetes.io/part-of: cilium

tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/hubble/servicemonitor.yaml

@@ -0,0 +1,27 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/name: hubble
+    app.kubernetes.io/part-of: cilium
+  name: hubble
+  namespace: cilium
+spec:
+  endpoints:
+    - honorLabels: true
+      interval: 10s
+      path: /metrics
+      port: hubble-metrics
+      relabelings:
+        - action: replace
+          replacement: ${1}
+          sourceLabels:
+            - __meta_kubernetes_pod_node_name
+          targetLabel: node
+      scheme: http
+  namespaceSelector:
+    matchNames:
+      - cilium
+  selector:
+    matchLabels:
+      k8s-app: hubble

tests/golden/egress-gateway/cilium/cilium/10_ebpf_alerts.yaml

@@ -0,0 +1,45 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  annotations: {}
+  labels:
+    name: cilium-ebpf
+  name: cilium-ebpf
+spec:
+  groups:
+    - name: cilium-ebpf.rules
+      rules:
+        - alert: CiliumBpfMapUtilizationHigh
+          annotations:
+            description: |
+              BPF map utilization for map {{ $labels.map_name }} has been above
+              50% on node {{ $labels.node }} for the last 10m.
+            message: High BPF map utilization on {{ $labels.node }}
+            runbook_url: https://hub.syn.tools/cilium/runbooks/CiliumBpfMapPressureHigh.html
+          expr: cilium_bpf_map_pressure > 0.5
+          for: 10m
+          labels:
+            severity: warning
+        - alert: CiliumBpfMapUtilizationExtremelyHigh
+          annotations:
+            description: |
+              BPF map utilization for map {{ $labels.map_name }} has been above
+              90% on node {{ $labels.node }} for the last 10m.
+            message: Extremely High BPF map utilization on {{ $labels.node }}
+            runbook_url: https://hub.syn.tools/cilium/runbooks/CiliumBpfMapPressureExtremelyHigh.html
+          expr: cilium_bpf_map_pressure > 0.9
+          for: 10m
+          labels:
+            severity: critical
+        - alert: CiliumBpfOperationErrorRateHigh
+          annotations:
+            description: |
+              BPF error rate for map {{ $labels.map_name }} has been above
+              50% on node {{ $labels.node }} for the last 10m.
+            message: High BPF error rate on {{ $labels.node }}
+            runbook_url: https://hub.syn.tools/cilium/runbooks/CiliumBpfOperationErrorRateHigh.html
+          expr: (rate(cilium_bpf_map_ops_total{outcome="fail"}[1m]) / rate(cilium_bpf_map_ops_total{}[1m]))
+            > 0.5
+          for: 10m
+          labels:
+            severity: critical

tests/golden/egress-gateway/cilium/cilium/10_egress_gateway_alerts.yaml

@@ -0,0 +1,27 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  annotations: {}
+  labels:
+    name: cilium-egress-gateway
+  name: cilium-egress-gateway
+spec:
+  groups:
+    - name: cilium-egress-gateway.rules
+      rules:
+        - alert: CiliumShadowRangeNodeMissing
+          annotations:
+            description: |
+              The cluster should have 3 nodes which host a shadow egress
+              IP range, but had {{ $value }} for the last 5m. Most likely
+              one or more nodes were replaced but the shadow range was not
+              updated.
+
+              The shadow range configmap currently has nodes infra-8344, infra-87c9, infra-eba2.
+            message: A node which hosts a shadow egress IP range is missing
+            runbook_url: https://hub.syn.tools/cilium/CiliumShadowRangeNodeMissing.html
+          expr: (sum (kube_node_info{node=~"infra-8344|infra-87c9|infra-eba2"}) or
+            vector(0)) != 3
+          for: 5m
+          labels:
+            severity: critical

tests/golden/egress-gateway/cilium/cilium/10_pods_alerts.yaml

@@ -0,0 +1,23 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  annotations: {}
+  labels:
+    name: cilium-pods
+  name: cilium-pods
+spec:
+  groups:
+    - name: cilium-pods.rules
+      rules:
+        - alert: CiliumAgentUnexpectedCount
+          annotations:
+            description: |
+              The cluster has had {{ $value }} Cilium agents for the last 5m
+              instead of the expected {{ query "count(kube_node_info)" }}.
+            message: Number of Cilium agents doesn't match node count
+            runbook_url: https://hub.syn.tools/cilium/runbooks/CiliumAgentUnexpectedCount.html
+          expr: count(kube_pod_labels{namespace="cilium", label_app_kubernetes_io_name="cilium-agent"})
+            != count(kube_node_info)
+          for: 5m
+          labels:
+            severity: critical