diff --git a/alerts.libsonnet b/alerts.libsonnet new file mode 100644 index 0000000..e520454 --- /dev/null +++ b/alerts.libsonnet @@ -0,0 +1,292 @@ +/** + * \file alerts.libsonnet + * \brief Generic functions that are suitable to filter and patch Prometheus alert rules and generate `PrometheusRule` contents from a Project Syn inventory parameter. + * + * This library is intended to be customized and re-exported by monitoring + * stack components (e.g. component-openshift4-monitoring and + * component-prometheus). + */ +local com = import 'lib/commodore.libjsonnet'; +local syn_teams = import 'syn-teams.libsonnet'; + +local inv = com.inventory(); + +{ + /** + * \brief global configuration used when filtering and patching alerts + * + * This field is intended to be set by a monitoring stack component (e.g. + * openshift4-monitoring) in order to apply its global alert parameters. + * This allows users of the monitoring component to apply some alert tuning + * (e.g. disabling a noisy alert or adding a custom annotation to a set of + * alerts) through a single component parameter, instead of first having to + * figure out which component defines the alert, so it can be disabled or + * modified. + * + * Example usage: + * + * ``` + * local inv = kap.inventory(); + * local alerts = (import 'syn/alerts.libsonnet') { + * global_alert_params+:: std.get( + * inv.parameters, + * 'openshift4_monitoring', // <1> + * { alerts : {} } // <2> + * ).alerts, // <3> + * }; + * ``` + * <1> Replace with the parameter key of your monitoring component. + * <2> We use std.get() with a fallback object to avoid that all component + * golden tests that use the component's re-exported alerts.libsonnet + * have to fully define the monitoring component parameter. + * <3> This example assumes that component openshift4-monitoring provides + * parameter `alerts` which has fields `ignoreNames` and + * `customAnnotations` in a format suitable for this library. + */ + global_alert_params:: { + /** + * \brief list of alert names (field `alert`) to filter out + * + * This field allows the monitoring component to inject a globally defined + * list of alerts to drop in `filterRules`, `filterPatchRules`, and + * `renderGroups`. + * + * Note that this field must contain alert names as they are processed by + * the library before any potential name patching by `patchRule`. + */ + ignoreNames: [], + /** + * \brief map of alert names (field `alert`) to object defining additional annotations for the matching alert + * + * This field allows the monitoring component to inject a globally defined + * object containing additional alert annotations. Top-level keys in this + * object are alert names and values are expected to be objects with + * string keys and values and are merged into the target alert's + * `annotations`. + * + * Note that this field must contain alert names as they are processed by + * the library before any potential name patching by `patchRule`. + */ + customAnnotations: {}, + }, + /** + * \brief filter alert rules in the provided group + * + * The function assumes that `group` is a valid entry for the PrometheusRule + * CR `.spec.groups` field. + * + * \arg group + * a PrometheusRule CR `.spec.groups` entry + * \arg ignoreNames + * A list of alert names to filter out. This argument is optional and + * defaults to the empty list. The function doesn't process the + * provided value for `ignoreNames`, except converting it to a Jsonnet + * set with `std.set()`. If you want to use `com.renderArray()` to + * allow re-enabling ignored alerts, you'll have to do so before + * providing the list to the function. + * \arg preserveRecordingRules + * Whether to keep or discard recording rules in the group. This + * argument is optional and defaults to `false`. This is useful when + * wanting to patch alerting rules which are already deployed to the + * cluster through some operator (e.g. cluster-logging, or rook-ceph). + * Generally, in such cases, we'll only want to modify alerting rules, + * but don't want to deploy duplicates of the recording rules which may + * be present in the same groups as the alerting rules in the upstream + * manifests. + * + * \returns + * The group with alert rules whose field `alert` matches an entry in + * either library field `global_alert_params.ignoreNames` or the function + * argument `ignoreNames` list removed. If `preserveRecordingRules` is + * `false`, all recording rules are also removed from the result. + */ + filterRules(group, ignoreNames=[], preserveRecordingRules=false): + local ignore_set = std.set($.global_alert_params.ignoreNames + ignoreNames); + group { + rules: + std.filter( + // Filter out unwanted rules + function(rule) + local isAlert = std.objectHas(rule, 'alert'); + if isAlert then + !std.member(ignore_set, rule.alert) + else + preserveRecordingRules, + super.rules + ), + }, + + /** + * \brief patch the provided rule to adhere to the format expected by the + * component. + * + * This function patches the provided alert rule to adhere to the format + * expected by this component. This includes adding labels which are used by + * other parts of the component to the rule (e.g. `syn=true`), as well as + * ensuring that the alert name is prefixed with `SYN_`. + * + * The function also reads any custom annotations from library field + * `global_alert_params.customAnnotations` and applies those to the alert + * rule. + * + * Custom alert patches can be provided through argument `patches`. + * + * Recording rules will always be returned unchanged + * + * \arg rule + * The rule to patch + * \arg patches + * An object with partial alert rule definitions. The function uses the + * provided rule's `alert` field to lookup potential patches. This + * parameter is optional and defaults to an empty object. + * \arg patchName + * Whether to prefix the alert name with `SYN_` if it isn't already. + * This parameter is optional and defaults to `true`. + * + * \returns The patched rule + */ + patchRule(rule, patches={}, patchName=true): + if !std.objectHas(rule, 'alert') then + rule + else + local rulepatch = std.get(patches, rule.alert, {}); + local fixupName(name) = + if patchName && !std.startsWith(name, 'SYN_') then + 'SYN_' + name + else + name; + local syn_team_label = + if std.objectHas(rule, 'labels') && std.objectHas(rule.labels, 'syn_team') + then + rule.labels.syn_team + else + syn_teams.teamForApplication(inv.parameters._instance); + rule { + // Change alert names so we don't get multiple alerts with the same + // name, as the logging operator deploys its own copy of these + // rules. + alert: fixupName(super.alert), + labels+: { + syn: 'true', + // mark alert as belonging to component instance in whose context the + // function is called. + syn_component: inv.parameters._instance, + // mark alert as belonging to the team in whose context the + // function is called. + [if syn_team_label != null then 'syn_team']: syn_team_label, + }, + annotations+: + std.get($.global_alert_params.customAnnotations, super.alert, {}), + } + com.makeMergeable(rulepatch), + + /** + * \brief Convenience wrapper around filterRules and patchRule. + * + * This function provides a convenience wrapper which filters the provided + * group using `filterRules`, and applies `patchRule` for each rule which + * isn't dropped by `filterRules`. + * + * \arg group + * a PrometheusRule CR `.spec.groups` entry + * \arg ignoreNames + * A list of alert names to filter out. This argument is optional and + * defaults to the empty list. The function doesn't process the + * provided value for `ignoreNames`, except converting it to a Jsonnet + * set with `std.set()`. If you want to use `com.renderArray()` to + * allow re-enabling ignored alerts, you'll have to do so before + * providing the list to the function. + * \arg patches + * An object with partial alert rule definitions. The function uses the + * provided rule's `alert` field to lookup potential patches. This + * parameter is optional and defaults to an empty object. + * \arg preserveRecordingRules + * Whether to keep or discard recording rules in the group. This + * argument is optional and defaults to `false`. This is useful when + * wanting to patch alerting rules which are already deployed to the + * cluster through some operator (e.g. cluster-logging, or rook-ceph). + * Generally, in such cases, we'll only want to modify alerting rules, + * but don't want to deploy duplicates of the recording rules which may + * be present in the same groups as the alerting rules in the upstream + * manifests. + * \arg patchNames + * Whether to prefix alert names with `SYN_` if they aren't already. + * This parameter is passed to `patchRule()` as argument `patchName`. + * This parameter is optional and defaults to `true`. + * + * \returns the provided `group` object with rules filtered and patched + */ + filterPatchRules(group, ignoreNames=[], patches={}, preserveRecordingRules=false, patchNames=true): + $.filterRules(group, ignoreNames, preserveRecordingRules) { + rules: std.map( + function(rule) $.patchRule(rule, patches, patchNames), + super.rules + ), + }, + + /** + * \brief Function to render rule groups defined in the hierarchy + * + * This function assumes that the rules are defined in the hierarchy in an + * object whose fields each represent a rule group. The function also + * assumes that each rule group is defined as an object which uses scheme + * '(alert:|record:)rulename' for the field names. Finally, the function + * assumes that each value of a rule key is a valid alerting or recording + * rule (matching the prefix of the key). + * + * Option + * + * \arg rules + * The object to render as rules. + * \arg ignoreNames + * Alert names to drop from the rendered rules. + * \arg patches + * Rule patches to apply to the rules. + * + * Note that this function doesn't apply the full `filterPatchRules` logic: + * We patch alerts to have the common Project Syn labels and apply any + * custom annotations defined in `global_alert_params.customAnnotations`. + * However, alert names are never patched. + * + * Additionally, we run each resulting alert group through `filterRules()` + * which enables users to disable alerts created through `renderGroups` via + * `global_alert_params.ignoreNames`. However, recording rules are always + * preserved by this `filterRules` call. + * + * Users who need name patching or recording rule filtering will need to + * apply `filterPatchRules()` with appropriate arguments to each list + * element returned by this function. + * + * \return + * A single list suitable to be used in a `PrometheusRule` manifest as + * `spec.groups`. + */ + renderGroups(rules, ignoreNames=[], patches={}): + std.filter( + function(g) std.length(g.rules) > 0, + [ + $.filterRules( + { + name: group_name, + rules: [ + local rnamekey = std.splitLimit(rname, ':', 1); + $.patchRule( + rules[group_name][rname] { + // transform source key into "alert: alertname" or + // "record: recordname" + [rnamekey[0]]: rnamekey[1], + }, + patches=patches, + patchName=false, + ) + for rname in std.objectFields(rules[group_name]) + if rules[group_name][rname] != null + ], + }, + ignoreNames=ignoreNames, + preserveRecordingRules=true, + ) + for group_name in std.objectFields(rules) + if rules[group_name] != null + ] + ), +} diff --git a/tests/alerts.jsonnet b/tests/alerts.jsonnet new file mode 100644 index 0000000..f80d899 --- /dev/null +++ b/tests/alerts.jsonnet @@ -0,0 +1,300 @@ +local com = import 'lib/commodore.libjsonnet'; + +local inv = com.inventory(); + +local expected = inv.expected; + +local alerts = (import 'alerts.libsonnet') { + // NOTE(sg): This needs to be documented properly. + global_alert_params:: inv.parameters.global_config, +}; + +local testRules = { + test_group: { + 'alert:FooIsLow': { + expr: 'foo < 10', + labels: { + severity: 'warning', + }, + annotations: { + summary: 'Foo < 10', + description: 'Foo is below 10, check Foo generator pipeline', + }, + }, + 'alert:SYN_FooIsExtremelyLow': { + expr: 'foo < 2', + labels: { + severity: 'critical', + }, + annotations: { + summary: 'Foo < 2', + description: 'Foo is below 2, immediately check Foo generator pipeline', + }, + }, + 'record:sum:foo': { + expr: 'sum(foo)', + }, + 'alert:IgnoredAlert': { + expr: 'vector(1)', + }, + }, +}; + +local testRulesRendered = [ + local rparts = std.splitLimit(r, ':', 1); + testRules.test_group[r] { + [rparts[0]]: rparts[1], + } + for r in std.objectFields(testRules.test_group) +]; + +local testPatches = { + FooIsLow: { + labels: { + test: 'label', + }, + }, + // patches are looked up before the alert name is patched. + SYN_FooIsLow: { + labels: { + test2: 'should be ignored', + }, + }, + SYN_FooIsExtremelyLow: { + labels: { + test: 'prefixed patch', + }, + }, +}; + +local testPatchRule = + local rendered = [ + alerts.patchRule(r, patches=testPatches, patchName=true) + for r in testRulesRendered + ]; + + if rendered != expected.patchedRules then + error + 'Error in testPatchRule:\n' + + 'Expected: %s\n' % [ expected.patchedRules ] + + 'Got: %s\n' % [ rendered ] + else + rendered; + +local testPatchRuleOrigName = + local rendered = [ + alerts.patchRule(r, patches=testPatches, patchName=false) + for r in testRulesRendered + ]; + local expectedRules = [ + if std.objectHas(r, 'alert') && r.alert != 'SYN_FooIsExtremelyLow' then + r { + alert: std.strReplace(r.alert, 'SYN_', ''), + } + else + r + for r in expected.patchedRules + ]; + + if rendered != expectedRules then + error + 'Error in testPatchRuleOrigName:\n' + + 'Expected: %s\n' % [ expectedRules ] + + 'Got: %s\n' % [ rendered ] + else + rendered; + +local testFilterRules = + local ignoreNames = [ 'SYN_FooIsExtremelyLow' ]; + local filtered = alerts.filterRules( + { rules: testRulesRendered }, + ignoreNames=ignoreNames, + preserveRecordingRules=true + ); + local expectedFiltered = std.filter( + function(r) + !std.member( + // NOTE(sg): IgnoredAlert is in global ignore list + [ 'IgnoredAlert' ] + ignoreNames, + std.get(r, 'alert', '') + ), + testRulesRendered + ); + + if filtered.rules != expectedFiltered then + error + 'Error in testFilterRules:\n' + + 'Expected: %s\n' % [ expectedFiltered ] + + 'Got: %s\n' % [ filtered.rules ] + else + filtered; + +local testFilterRulesNoRecording = + local filtered = alerts.filterRules( + { rules: testRulesRendered }, + ignoreNames=[], + preserveRecordingRules=false, + ); + local expectedFiltered = std.filter( + function(r) + std.objectHas(r, 'alert') && + r.alert != 'IgnoredAlert', + testRulesRendered + ); + + if filtered.rules != expectedFiltered then + error + 'Error in testFilterRulesNoRecording:\n' + + 'Expected: %s\n' % [ expectedFiltered ] + + 'Got: %s\n' % [ filtered.rules ] + else + filtered; + +local testFilterPatchRules = + local filtered = alerts.filterPatchRules( + { rules: testRulesRendered }, + ignoreNames=[ 'Ignore2' ], + patches=testPatches { + Ignore2: { + expr: 'vector(2)', + labels: { + severity: 'info', + }, + }, + }, + preserveRecordingRules=true, + patchNames=true, + ); + + local expectedFiltered = std.filter( + function(r) + !std.member( + // NOTE(sg): IgnoredAlert is in global ignore list; also we need to + // filter `expected.patchedRules` using the prefixed names. + [ 'SYN_IgnoredAlert', 'SYN_Ignore2' ], + std.get(r, 'alert', '') + ), + expected.patchedRules + ); + + if filtered.rules != expectedFiltered then + error + 'Error in testFilterPatchRules:\n' + + 'Expected: %s\n' % [ expectedFiltered ] + + 'Got: %s\n' % [ filtered.rules ] + else + filtered; + +local testFilterPatchRulesNoRecording = + local filtered = alerts.filterPatchRules( + { rules: testRulesRendered }, + ignoreNames=[ 'Ignore2' ], + patches=testPatches { + Ignore2: { + expr: 'vector(2)', + labels: { + severity: 'info', + }, + }, + }, + preserveRecordingRules=false, + patchNames=true, + ); + + local expectedFiltered = std.filter( + function(r) + std.objectHas(r, 'alert') && + !std.member( + // NOTE(sg): IgnoredAlert is in global ignore list; also we need to + // filter `expected.patchedRules` using the prefixed names. + [ 'SYN_IgnoredAlert', 'SYN_Ignore2' ], + std.get(r, 'alert', '') + ), + expected.patchedRules + ); + + if filtered.rules != expectedFiltered then + error + 'Error in testFilterPatchRulesNoRecording:\n' + + 'Expected: %s\n' % [ expectedFiltered ] + + 'Got: %s\n' % [ filtered.rules ] + else + filtered; + +local testFilterPatchRulesNoRecordingOrigName = + local filtered = alerts.filterPatchRules( + { rules: testRulesRendered }, + ignoreNames=[ 'Ignore2' ], + patches=testPatches { + Ignore2: { + expr: 'vector(2)', + labels: { + severity: 'info', + }, + }, + }, + preserveRecordingRules=false, + patchNames=false, + ); + + local expectedFiltered = [ + if r.alert != 'SYN_FooIsExtremelyLow' then + r { + alert: std.strReplace(r.alert, 'SYN_', ''), + } + else + r + for r in std.filter( + function(r) + std.objectHas(r, 'alert') && + !std.member( + // NOTE(sg): IgnoredAlert is in global ignore list; also we need to + // filter `expected.patchedRules` using the prefixed names. + [ 'SYN_IgnoredAlert', 'SYN_Ignore2' ], + std.get(r, 'alert', '') + ), + expected.patchedRules + ) + ]; + + if filtered.rules != expectedFiltered then + error + 'Error in testFilterPatchRulesNoRecordingOrigName:\n' + + 'Expected: %s\n' % [ expectedFiltered ] + + 'Got: %s\n' % [ filtered.rules ] + else + filtered; + +local testRenderGroups = + local rendered = alerts.renderGroups(testRules, patches=testPatches); + local expectedRules = [ + if std.objectHas(r, 'alert') && r.alert != 'SYN_FooIsExtremelyLow' then + r { + alert: std.strReplace(r.alert, 'SYN_', ''), + } + else + r + for r in std.filter( + function(r) std.get(r, 'alert', '') != 'SYN_IgnoredAlert', + expected.patchedRules + ) + ]; + + if rendered[0].rules != expectedRules then + error + 'Error in testRenderGroups:\n' + + 'Expected: %s\n' % [ expectedRules ] + + 'Got: %s\n' % [ rendered[0].rules ] + else + rendered; + +{ + patchedRules: testPatchRule, + patchedRulesOrigName: testPatchRuleOrigName, + filteredRules: testFilterRules, + filteredRulesNoRecording: testFilterRulesNoRecording, + filterPatchRules: testFilterPatchRules, + filterPatchRulesNoRecording: testFilterPatchRulesNoRecording, + filterPatchRulesNoRecordingOrigName: testFilterPatchRulesNoRecordingOrigName, + renderedGroups: testRenderGroups, +} diff --git a/tests/alerts.yaml b/tests/alerts.yaml new file mode 100644 index 0000000..5c62c4b --- /dev/null +++ b/tests/alerts.yaml @@ -0,0 +1,49 @@ +applications: + - alerts + +parameters: + _instance: alerts + syn: + owner: clumsy-donkeys + + global_config: + ignoreNames: + - IgnoredAlert + customAnnotations: + FooIsLow: + test: annotation + +expected: + patchedRules: + - alert: SYN_FooIsLow + expr: foo < 10 + annotations: + description: Foo is below 10, check Foo generator pipeline + summary: Foo < 10 + test: annotation + labels: + severity: warning + syn: "true" + syn_component: alerts + syn_team: clumsy-donkeys + test: label + - alert: SYN_IgnoredAlert + expr: vector(1) + annotations: {} + labels: + syn: "true" + syn_component: alerts + syn_team: clumsy-donkeys + - alert: SYN_FooIsExtremelyLow + expr: foo < 2 + annotations: + description: Foo is below 2, immediately check Foo generator pipeline + summary: Foo < 2 + labels: + severity: critical + syn: "true" + syn_component: alerts + syn_team: clumsy-donkeys + test: prefixed patch + - record: sum:foo + expr: sum(foo) diff --git a/tests/golden/alerts.yml b/tests/golden/alerts.yml new file mode 100644 index 0000000..1515230 --- /dev/null +++ b/tests/golden/alerts.yml @@ -0,0 +1,292 @@ +{ + "filterPatchRules": { + "rules": [ + { + "alert": "SYN_FooIsLow", + "annotations": { + "description": "Foo is below 10, check Foo generator pipeline", + "summary": "Foo < 10", + "test": "annotation" + }, + "expr": "foo < 10", + "labels": { + "severity": "warning", + "syn": "true", + "syn_component": "alerts", + "syn_team": "clumsy-donkeys", + "test": "label" + } + }, + { + "alert": "SYN_FooIsExtremelyLow", + "annotations": { + "description": "Foo is below 2, immediately check Foo generator pipeline", + "summary": "Foo < 2" + }, + "expr": "foo < 2", + "labels": { + "severity": "critical", + "syn": "true", + "syn_component": "alerts", + "syn_team": "clumsy-donkeys", + "test": "prefixed patch" + } + }, + { + "expr": "sum(foo)", + "record": "sum:foo" + } + ] + }, + "filterPatchRulesNoRecording": { + "rules": [ + { + "alert": "SYN_FooIsLow", + "annotations": { + "description": "Foo is below 10, check Foo generator pipeline", + "summary": "Foo < 10", + "test": "annotation" + }, + "expr": "foo < 10", + "labels": { + "severity": "warning", + "syn": "true", + "syn_component": "alerts", + "syn_team": "clumsy-donkeys", + "test": "label" + } + }, + { + "alert": "SYN_FooIsExtremelyLow", + "annotations": { + "description": "Foo is below 2, immediately check Foo generator pipeline", + "summary": "Foo < 2" + }, + "expr": "foo < 2", + "labels": { + "severity": "critical", + "syn": "true", + "syn_component": "alerts", + "syn_team": "clumsy-donkeys", + "test": "prefixed patch" + } + } + ] + }, + "filterPatchRulesNoRecordingOrigName": { + "rules": [ + { + "alert": "FooIsLow", + "annotations": { + "description": "Foo is below 10, check Foo generator pipeline", + "summary": "Foo < 10", + "test": "annotation" + }, + "expr": "foo < 10", + "labels": { + "severity": "warning", + "syn": "true", + "syn_component": "alerts", + "syn_team": "clumsy-donkeys", + "test": "label" + } + }, + { + "alert": "SYN_FooIsExtremelyLow", + "annotations": { + "description": "Foo is below 2, immediately check Foo generator pipeline", + "summary": "Foo < 2" + }, + "expr": "foo < 2", + "labels": { + "severity": "critical", + "syn": "true", + "syn_component": "alerts", + "syn_team": "clumsy-donkeys", + "test": "prefixed patch" + } + } + ] + }, + "filteredRules": { + "rules": [ + { + "alert": "FooIsLow", + "annotations": { + "description": "Foo is below 10, check Foo generator pipeline", + "summary": "Foo < 10" + }, + "expr": "foo < 10", + "labels": { + "severity": "warning" + } + }, + { + "expr": "sum(foo)", + "record": "sum:foo" + } + ] + }, + "filteredRulesNoRecording": { + "rules": [ + { + "alert": "FooIsLow", + "annotations": { + "description": "Foo is below 10, check Foo generator pipeline", + "summary": "Foo < 10" + }, + "expr": "foo < 10", + "labels": { + "severity": "warning" + } + }, + { + "alert": "SYN_FooIsExtremelyLow", + "annotations": { + "description": "Foo is below 2, immediately check Foo generator pipeline", + "summary": "Foo < 2" + }, + "expr": "foo < 2", + "labels": { + "severity": "critical" + } + } + ] + }, + "patchedRules": [ + { + "alert": "SYN_FooIsLow", + "annotations": { + "description": "Foo is below 10, check Foo generator pipeline", + "summary": "Foo < 10", + "test": "annotation" + }, + "expr": "foo < 10", + "labels": { + "severity": "warning", + "syn": "true", + "syn_component": "alerts", + "syn_team": "clumsy-donkeys", + "test": "label" + } + }, + { + "alert": "SYN_IgnoredAlert", + "annotations": { }, + "expr": "vector(1)", + "labels": { + "syn": "true", + "syn_component": "alerts", + "syn_team": "clumsy-donkeys" + } + }, + { + "alert": "SYN_FooIsExtremelyLow", + "annotations": { + "description": "Foo is below 2, immediately check Foo generator pipeline", + "summary": "Foo < 2" + }, + "expr": "foo < 2", + "labels": { + "severity": "critical", + "syn": "true", + "syn_component": "alerts", + "syn_team": "clumsy-donkeys", + "test": "prefixed patch" + } + }, + { + "expr": "sum(foo)", + "record": "sum:foo" + } + ], + "patchedRulesOrigName": [ + { + "alert": "FooIsLow", + "annotations": { + "description": "Foo is below 10, check Foo generator pipeline", + "summary": "Foo < 10", + "test": "annotation" + }, + "expr": "foo < 10", + "labels": { + "severity": "warning", + "syn": "true", + "syn_component": "alerts", + "syn_team": "clumsy-donkeys", + "test": "label" + } + }, + { + "alert": "IgnoredAlert", + "annotations": { }, + "expr": "vector(1)", + "labels": { + "syn": "true", + "syn_component": "alerts", + "syn_team": "clumsy-donkeys" + } + }, + { + "alert": "SYN_FooIsExtremelyLow", + "annotations": { + "description": "Foo is below 2, immediately check Foo generator pipeline", + "summary": "Foo < 2" + }, + "expr": "foo < 2", + "labels": { + "severity": "critical", + "syn": "true", + "syn_component": "alerts", + "syn_team": "clumsy-donkeys", + "test": "prefixed patch" + } + }, + { + "expr": "sum(foo)", + "record": "sum:foo" + } + ], + "renderedGroups": [ + { + "name": "test_group", + "rules": [ + { + "alert": "FooIsLow", + "annotations": { + "description": "Foo is below 10, check Foo generator pipeline", + "summary": "Foo < 10", + "test": "annotation" + }, + "expr": "foo < 10", + "labels": { + "severity": "warning", + "syn": "true", + "syn_component": "alerts", + "syn_team": "clumsy-donkeys", + "test": "label" + } + }, + { + "alert": "SYN_FooIsExtremelyLow", + "annotations": { + "description": "Foo is below 2, immediately check Foo generator pipeline", + "summary": "Foo < 2" + }, + "expr": "foo < 2", + "labels": { + "severity": "critical", + "syn": "true", + "syn_component": "alerts", + "syn_team": "clumsy-donkeys", + "test": "prefixed patch" + } + }, + { + "expr": "sum(foo)", + "record": "sum:foo" + } + ] + } + ] +}