diff --git a/docs/SLO_MONITORING.md b/docs/SLO_MONITORING.md new file mode 100644 index 0000000..217585e --- /dev/null +++ b/docs/SLO_MONITORING.md @@ -0,0 +1,40 @@ +# Service Level Objective Monitoring and Burn-Rate Alerts + +## Objectives + +Utility Protocol services share two production SLOs: + +- **Availability:** 99.99% successful critical-path requests over a rolling 30-day window. +- **Latency:** P99 critical-path latency below 100 ms. + +The implementation uses pre-aggregated request counters so the SLO path stays below the 100 ms P99 performance target. Services should emit counters for total requests, failed requests, latency violations, and P99 latency per rolling window. + +## Architecture + +1. Every service exports rolling-window measurements for the fast and slow burn-rate windows. +2. `meter-simulator/src/slo-monitor.js` evaluates the measurements against the shared SLO configuration. +3. Alert payloads include stable dashboard labels: `slo`, `window`, and `severity`. +4. Dashboards group the same labels by service, environment, and deployment color. +5. Runbooks route `page` alerts to the on-call engineer and `ticket` alerts to the service team backlog. + +## Burn-rate policy + +| Window | Duration | Burn-rate threshold | Severity | +| --- | ---: | ---: | --- | +| Fast | 5 minutes | 14.4x | Page | +| Slow | 60 minutes | 6x | Ticket | + +A burn rate compares the observed bad-event ratio with the 0.01% error budget implied by 99.99% availability. Bad events include failed requests and requests whose latency exceeds the critical-path latency objective. + +## Blue-green and canary rollout + +- Deploy the green environment with SLO monitoring in shadow mode first. +- Add a custom `canary` burn-rate window for the canary slice. +- Promote traffic only when the canary window is healthy for the full analysis period. +- Roll back immediately on a page-level burn-rate alert or sustained P99 latency above 100 ms. + +## Security review checklist + +- Do not emit user consumption volumes, wallet secrets, raw device signatures, or personally identifying metadata in alert labels. +- Alert labels must remain bounded-cardinality to prevent dashboard and log-index exhaustion. +- Validate all externally supplied SLO configuration before deploying it. diff --git a/meter-simulator/src/slo-monitor.js b/meter-simulator/src/slo-monitor.js new file mode 100644 index 0000000..f86d49b --- /dev/null +++ b/meter-simulator/src/slo-monitor.js @@ -0,0 +1,132 @@ +/** + * Service Level Objective (SLO) monitor with multi-window burn-rate alerts. + * + * The monitor is intentionally dependency-free so it can run in the meter + * simulator, CI smoke checks, and dashboard ingestion jobs without adding + * critical-path latency. It evaluates availability and latency objectives from + * pre-aggregated rolling-window measurements. + */ + +const DEFAULT_CONFIG = Object.freeze({ + availabilityTarget: 0.9999, + latencyP99TargetMs: 100, + objectiveWindowMinutes: 30 * 24 * 60, + burnRateWindows: Object.freeze([ + Object.freeze({ name: 'fast', minutes: 5, threshold: 14.4, severity: 'page' }), + Object.freeze({ name: 'slow', minutes: 60, threshold: 6, severity: 'ticket' }) + ]) +}); + +function validateConfig(config) { + if (config.availabilityTarget <= 0 || config.availabilityTarget >= 1) { + throw new Error('availabilityTarget must be between 0 and 1'); + } + if (!Number.isFinite(config.latencyP99TargetMs) || config.latencyP99TargetMs <= 0) { + throw new Error('latencyP99TargetMs must be greater than zero'); + } + if (!Number.isFinite(config.objectiveWindowMinutes) || config.objectiveWindowMinutes <= 0) { + throw new Error('objectiveWindowMinutes must be greater than zero'); + } + if (!Array.isArray(config.burnRateWindows) || config.burnRateWindows.length === 0) { + throw new Error('at least one burn-rate window is required'); + } +} + +function normalizeWindow(window) { + return { + name: window.name, + minutes: window.minutes, + threshold: window.threshold, + severity: window.severity || 'ticket' + }; +} + +class SLOMonitor { + constructor(options = {}) { + this.config = { + ...DEFAULT_CONFIG, + ...options, + burnRateWindows: (options.burnRateWindows || DEFAULT_CONFIG.burnRateWindows).map(normalizeWindow) + }; + validateConfig(this.config); + } + + get errorBudgetRatio() { + return 1 - this.config.availabilityTarget; + } + + evaluateWindow(measurement) { + const totalRequests = measurement.totalRequests || 0; + const failedRequests = measurement.failedRequests || 0; + const latencyViolations = measurement.latencyViolations || 0; + const p99LatencyMs = measurement.p99LatencyMs || 0; + const badEvents = failedRequests + latencyViolations; + const observedErrorRatio = totalRequests === 0 ? 0 : badEvents / totalRequests; + const burnRate = this.errorBudgetRatio === 0 ? 0 : observedErrorRatio / this.errorBudgetRatio; + + return { + windowName: measurement.windowName, + minutes: measurement.minutes, + totalRequests, + failedRequests, + latencyViolations, + badEvents, + observedErrorRatio, + burnRate, + availability: totalRequests === 0 ? 1 : 1 - failedRequests / totalRequests, + p99LatencyMs, + availabilityHealthy: totalRequests === 0 || failedRequests / totalRequests <= this.errorBudgetRatio, + latencyHealthy: p99LatencyMs <= this.config.latencyP99TargetMs + }; + } + + evaluate(measurements) { + const byName = new Map(measurements.map((measurement) => [measurement.windowName, measurement])); + + return this.config.burnRateWindows.map((window) => { + const measurement = byName.get(window.name) || { + windowName: window.name, + minutes: window.minutes, + totalRequests: 0, + failedRequests: 0, + latencyViolations: 0, + p99LatencyMs: 0 + }; + const result = this.evaluateWindow({ ...measurement, windowName: window.name, minutes: window.minutes }); + const alerting = result.burnRate >= window.threshold || !result.latencyHealthy; + + return { + ...result, + threshold: window.threshold, + severity: alerting ? window.severity : 'none', + alerting, + dashboardLabels: { + slo: 'utility-protocol-availability-latency', + window: window.name, + severity: alerting ? window.severity : 'none' + } + }; + }); + } + + buildAlert(measurements) { + const evaluations = this.evaluate(measurements); + const active = evaluations.filter((evaluation) => evaluation.alerting); + + return { + active: active.length > 0, + severity: active.some((evaluation) => evaluation.severity === 'page') ? 'page' : active[0]?.severity || 'none', + objective: { + availabilityTarget: this.config.availabilityTarget, + latencyP99TargetMs: this.config.latencyP99TargetMs, + objectiveWindowMinutes: this.config.objectiveWindowMinutes + }, + evaluations: active.length > 0 ? active : evaluations + }; + } +} + +module.exports = { + DEFAULT_CONFIG, + SLOMonitor +}; diff --git a/meter-simulator/tests/slo-monitor.test.js b/meter-simulator/tests/slo-monitor.test.js new file mode 100644 index 0000000..2addc3f --- /dev/null +++ b/meter-simulator/tests/slo-monitor.test.js @@ -0,0 +1,59 @@ +const { SLOMonitor } = require('../src/slo-monitor'); + +describe('SLOMonitor', () => { + test('keeps healthy windows below alert thresholds', () => { + const monitor = new SLOMonitor(); + const alert = monitor.buildAlert([ + { windowName: 'fast', totalRequests: 100000, failedRequests: 1, latencyViolations: 2, p99LatencyMs: 82 }, + { windowName: 'slow', totalRequests: 1000000, failedRequests: 10, latencyViolations: 20, p99LatencyMs: 91 } + ]); + + expect(alert.active).toBe(false); + expect(alert.severity).toBe('none'); + expect(alert.evaluations).toHaveLength(2); + expect(alert.evaluations[0].availabilityHealthy).toBe(true); + expect(alert.evaluations[0].latencyHealthy).toBe(true); + }); + + test('pages when the fast burn-rate window exhausts error budget too quickly', () => { + const monitor = new SLOMonitor(); + const alert = monitor.buildAlert([ + { windowName: 'fast', totalRequests: 100000, failedRequests: 100, latencyViolations: 100, p99LatencyMs: 88 }, + { windowName: 'slow', totalRequests: 1000000, failedRequests: 20, latencyViolations: 20, p99LatencyMs: 90 } + ]); + + expect(alert.active).toBe(true); + expect(alert.severity).toBe('page'); + expect(alert.evaluations[0].windowName).toBe('fast'); + expect(alert.evaluations[0].burnRate).toBeCloseTo(20, 5); + expect(alert.evaluations[0].dashboardLabels).toEqual({ + slo: 'utility-protocol-availability-latency', + window: 'fast', + severity: 'page' + }); + }); + + test('alerts on latency SLO violations even when availability is healthy', () => { + const monitor = new SLOMonitor(); + const alert = monitor.buildAlert([ + { windowName: 'fast', totalRequests: 50000, failedRequests: 0, latencyViolations: 0, p99LatencyMs: 125 } + ]); + + expect(alert.active).toBe(true); + expect(alert.severity).toBe('page'); + expect(alert.evaluations[0].latencyHealthy).toBe(false); + }); + + test('supports custom burn-rate windows for canary analysis', () => { + const monitor = new SLOMonitor({ + burnRateWindows: [{ name: 'canary', minutes: 10, threshold: 2, severity: 'page' }] + }); + const alert = monitor.buildAlert([ + { windowName: 'canary', totalRequests: 10000, failedRequests: 3, latencyViolations: 0, p99LatencyMs: 70 } + ]); + + expect(alert.active).toBe(true); + expect(alert.evaluations[0].minutes).toBe(10); + expect(alert.evaluations[0].burnRate).toBeCloseTo(3, 5); + }); +});