Render ThanosRuler instances

class/defaults.yml

@@ -143,6 +143,10 @@ parameters:
         enabled: false
         config: {}
         overrides: {}
+      thanosRuler:
+        enabled: false
+        config: {}
+        overrides: {}
 
     prometheusOperator:
       enabled: true

component/common.libsonnet

@@ -41,6 +41,7 @@ local instanceComponents = [
   'prometheus',
   'prometheusAdapter',
   'kubePrometheus',
+  'thanosRuler',
 ];
 
 local imageIsDockerIOShort = function(image)
@@ -233,6 +234,135 @@ local grafanaIngress(instanceName, instanceParams) = if instanceParams.grafana.i
     },
   } else {};
 
+// NOTE(mdl): kube-prometheus ships no thanosRuler component
+local thanosRuler(instanceName, instanceParams) =
+  local name = formatComponentName('thanosRuler', instanceName);
+  local namespace = instanceParams.common.namespace;
+  local metadata = {
+    name: name,
+    namespace: namespace,
+  };
+  local config = instanceParams.thanosRuler.config;
+
+  local serviceName(component) =
+    com.getValueOrDefault(
+      instanceParams[component].config, 'name', formatComponentName(component, instanceName)
+    );
+
+  local configuresEndpoint(fields) = std.any([ std.objectHas(config, f) for f in fields ]);
+  local endpointDefaults =
+    (
+      if instanceParams.prometheus.enabled && !configuresEndpoint([ 'queryEndpoints', 'queryConfig' ]) then {
+        // NOTE: The CRD recommends using `queryConfig` for Thanos >= 0.11.0.
+        // Eventually the default config should probably detect the Thanos
+        // version and adjust the config accordingly.
+        queryEndpoints: [ 'http://prometheus-%s.%s.svc:9090' % [ serviceName('prometheus'), namespace ] ],
+      } else {}
+    ) + (
+      if instanceParams.alertmanager.enabled && !configuresEndpoint([ 'alertmanagersUrl', 'alertmanagersConfig' ]) then {
+        // NOTE: The CRD recommends using `alertmanagersConfig` for Thanos >=
+        // 0.10.0.  Eventually the default config should probably detect the
+        // Thanos version and adjust the config accordingly.
+        alertmanagersUrl: [ 'http://alertmanager-%s.%s.svc:9093' % [ serviceName('alertmanager'), namespace ] ],
+      } else {}
+    );
+
+  local spec = endpointDefaults + config { serviceAccountName: name };
+  {
+    thanosRuler: if instanceParams.thanosRuler.enabled then
+      assert std.objectHas(spec, 'queryEndpoints') || std.objectHas(spec, 'queryConfig') :
+             'thanosRuler of instance `%s` needs `config.queryEndpoints` or `config.queryConfig` when the instance has no Prometheus enabled' % instanceName;
+      {
+        serviceAccount: {
+          apiVersion: 'v1',
+          kind: 'ServiceAccount',
+          metadata: metadata,
+        },
+        thanosRuler: {
+          apiVersion: 'monitoring.coreos.com/v1',
+          kind: 'ThanosRuler',
+          metadata: metadata,
+          spec: spec,
+        },
+        serviceMonitor: {
+          apiVersion: 'monitoring.coreos.com/v1',
+          kind: 'ServiceMonitor',
+          metadata: metadata,
+          spec: {
+            namespaceSelector: {
+              matchNames: [
+                namespace,
+              ],
+            },
+            selector: {
+              matchLabels: {
+                'operated-thanos-ruler': 'true',
+              },
+            },
+            endpoints: [
+              { port: 'web' },
+            ],
+          },
+        },
+        prometheusRule: {
+          apiVersion: 'monitoring.coreos.com/v1',
+          kind: 'PrometheusRule',
+          metadata: metadata,
+          spec: {
+            groups: [ {
+              name: name,
+              rules: [
+                {
+                  alert: 'ThanosRulerDown',
+                  expr: 'up{namespace="%s",service="thanos-ruler-operated"} == 0 or absent(up{namespace="%s",service="thanos-ruler-operated"})' % [ namespace, namespace ],
+                  'for': '15m',
+                  labels: {
+                    severity: 'critical',
+                  },
+                  annotations: {
+                    summary: 'Thanos Ruler %s is down, its alert rules are not being evaluated.' % name,
+                  },
+                },
+                {
+                  alert: 'ThanosRulerRuleEvaluationFailing',
+                  expr: 'increase(prometheus_rule_evaluation_failures_total{namespace="%s",service="thanos-ruler-operated"}[10m]) > 0' % namespace,
+                  'for': '15m',
+                  labels: {
+                    severity: 'warning',
+                  },
+                  annotations: {
+                    summary: 'Thanos Ruler %s is failing rule evaluations, alerts may be missed.' % name,
+                  },
+                },
+                {
+                  alert: 'ThanosRulerIsDroppingAlerts',
+                  expr: 'sum(rate(thanos_alert_sender_alerts_dropped_total{namespace="%s"}[5m])) > 0 or sum(rate(thanos_alert_queue_alerts_dropped_total{namespace="%s"}[5m])) > 0' % [ namespace, namespace ],
+                  'for': '5m',
+                  labels: {
+                    severity: 'critical',
+                  },
+                  annotations: {
+                    summary: 'Thanos Ruler %s is dropping alerts instead of delivering them to Alertmanager.' % name,
+                  },
+                },
+                {
+                  alert: 'ThanosRulerNoEvaluation',
+                  expr: 'time() - max by (rule_group) (prometheus_rule_group_last_evaluation_timestamp_seconds{namespace="%s",service="thanos-ruler-operated"}) > 10 * max by (rule_group) (prometheus_rule_group_interval_seconds{namespace="%s",service="thanos-ruler-operated"})' % [ namespace, namespace ],
+                  'for': '5m',
+                  labels: {
+                    severity: 'critical',
+                  },
+                  annotations: {
+                    summary: 'Thanos Ruler %s has not evaluated a rule group for 10 intervals.' % name,
+                  },
+                },
+              ],
+            } ],
+          },
+        },
+      } else {},
+  };
+
 local grafanaStorage(instanceName, instanceParams) = if instanceParams.grafana.persistence.enabled then
   assert instanceParams.grafana.persistence.size != '' : 'Storage size cannot be empty when persistence enabled';
   {
@@ -356,6 +486,7 @@ local stackForInstance = function(instanceName)
       },
     } + patchGrafanaDataSource(instanceName) + patchKubeControlPlaneSelectors(instanceName) + com.makeMergeable(cm),
   }
+  + thanosRuler(instanceName, confWithBase)
   + grafanaStorage(instanceName, confWithBase)
   + grafanaIngress(instanceName, confWithBase)
   + addNodeExporterContainerArgs(instanceName, confWithBase)

component/main.jsonnet

@@ -87,6 +87,7 @@ local secrets = std.foldl(
 local renderInstance = function(instanceName, stack)
   local prometheus = common.render_component(stack, 'prometheus', 20, instanceName);
   local alertmanager = common.render_component(stack, 'alertmanager', 30, instanceName);
+  local thanosRuler = common.render_component(stack, 'thanosRuler', 35, instanceName);
   local grafana = common.render_component(stack, 'grafana', 40, instanceName);
   local nodeExporter = common.render_component(stack, 'nodeExporter', 50, instanceName);
   local blackboxExporter = common.render_component(stack, 'blackboxExporter', 60, instanceName);
@@ -104,7 +105,8 @@ local renderInstance = function(instanceName, stack)
   (if p.kubernetesControlPlane.enabled then kubernetesControlPlane else {}) +
   (if p.prometheusAdapter.enabled then prometheusAdapter else {}) +
   (if p.kubeStateMetrics.enabled then kubeStateMetrics else {}) +
-  (if p.kubePrometheus.enabled then kubePrometheus else {})
+  (if p.kubePrometheus.enabled then kubePrometheus else {}) +
+  (if p.thanosRuler.enabled then thanosRuler else {})
 
 ;
 

docs/modules/ROOT/pages/references/parameters.adoc

@@ -16,6 +16,7 @@ Multiple instances of the following components can be configured:
 * kubernetesControlPlane
 * prometheusAdapter
 * kubeStateMetrics
+* thanosRuler
 
 
 == `kubernetes_version`
@@ -255,6 +256,10 @@ kubePrometheus:
   enabled: false
   config: {}
   overrides: {}
+thanosRuler:
+  enabled: false
+  config: {}
+  overrides: {}
 ----
 
 The base configuration shared by all instances.
@@ -381,6 +386,36 @@ This means configuration side effects don't apply and the configuration can cont
 The easiest way to find the allowed parameters is to look at the local `defaults` variable.
 See the kube state metrics defaults as an example: https://github.com/prometheus-operator/kube-prometheus/blob/aeb50f066eadf9831c53cdf9228e09dd4e9d28b2/jsonnet/kube-prometheus/components/kube-state-metrics.libsonnet#L7-L48[kube-prometheus/components/kube-state-metrics.libsonnet]
 
+== `base.thanosRuler`, `instances.*.thanosRuler`
+
+[horizontal]
+type:: dict
+example::
++
+[source,yaml]
+----
+thanosRuler:
+  enabled: true
+  config:
+    replicas: 2
+    queryEndpoints:
+      - http://prometheus-infra.syn-infra-monitoring:9090
+    ruleNamespaceSelector:
+      matchExpressions:
+        - key: appcat.vshn.io/servicename
+          operator: Exists
+----
+
+`config` is merged into the https://prometheus-operator.dev/docs/api-reference/api/#monitoring.coreos.com/v1.ThanosRulerSpec[`ThanosRuler` spec].
+
+If the same instance has `prometheus.enabled` or `alertmanager.enabled`, `spec.queryEndpoints` and `spec.alertmanagersUrl` default to the respective services of that instance.
+An explicit value in `config` always wins, and the default is skipped entirely when `config` sets the mutually exclusive `queryConfig` resp. `alertmanagersConfig`.
+Set them explicitly to evaluate against, or alert through, another instance or namespace.
+
+NOTE: Depending on the target cluster, evaluating against or alerting through an instance in another namespace may require custom network policies.
+
+Rendering fails if the instance has no Prometheus enabled and `config` sets neither `queryEndpoints` nor `queryConfig`.
+
 == `instances.*.(prometheus|alertmanager|grafana).networkPolicy.additionalIngressRules`
 
 [horizontal]

tests/golden/thanos/prometheus/prometheus/30_test_alertmanager_alertmanager.yaml

@@ -0,0 +1,38 @@
+apiVersion: monitoring.coreos.com/v1
+kind: Alertmanager
+metadata:
+  annotations:
+    source: https://github.com/projectsyn/component-prometheus
+  labels:
+    alertmanager: alertmanager-test
+    app.kubernetes.io/component: alert-router
+    app.kubernetes.io/managed-by: commodore
+    app.kubernetes.io/name: alertmanager
+    app.kubernetes.io/part-of: kube-prometheus
+    app.kubernetes.io/version: 0.22.2
+  name: alertmanager-test
+  namespace: syn-prometheus
+spec:
+  image: quay.io/prometheus/alertmanager:v0.22.2
+  nodeSelector:
+    kubernetes.io/os: linux
+  podMetadata:
+    labels:
+      app.kubernetes.io/component: alert-router
+      app.kubernetes.io/name: alertmanager
+      app.kubernetes.io/part-of: kube-prometheus
+      app.kubernetes.io/version: 0.22.2
+  replicas: 3
+  resources:
+    limits:
+      cpu: 100m
+      memory: 100Mi
+    requests:
+      cpu: 4m
+      memory: 100Mi
+  securityContext:
+    fsGroup: 2000
+    runAsNonRoot: true
+    runAsUser: 1000
+  serviceAccountName: alertmanager-alertmanager-test
+  version: 0.22.2

tests/golden/thanos/prometheus/prometheus/30_test_alertmanager_podDisruptionBudget.yaml

@@ -0,0 +1,21 @@
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  annotations:
+    source: https://github.com/projectsyn/component-prometheus
+  labels:
+    app.kubernetes.io/component: alert-router
+    app.kubernetes.io/managed-by: commodore
+    app.kubernetes.io/name: alertmanager
+    app.kubernetes.io/part-of: kube-prometheus
+    app.kubernetes.io/version: 0.22.2
+  name: alertmanager-alertmanager-test
+  namespace: syn-prometheus
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      alertmanager: alertmanager-test
+      app.kubernetes.io/component: alert-router
+      app.kubernetes.io/name: alertmanager
+      app.kubernetes.io/part-of: kube-prometheus