From 811e34904fc6c08a82c8efcec4adfc022fd8be88 Mon Sep 17 00:00:00 2001 From: gloskull Date: Fri, 17 Jul 2026 18:22:50 +0100 Subject: [PATCH] Add PagerDuty incident runbook automation --- .gitignore | 4 +- docs/INCIDENT_RUNBOOK_AUTOMATION.md | 37 +++++ meter-simulator/package.json | 3 +- meter-simulator/src/incident-runbook.js | 132 ++++++++++++++++++ .../tests/incident-runbook.test.js | 63 +++++++++ 5 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 docs/INCIDENT_RUNBOOK_AUTOMATION.md create mode 100644 meter-simulator/src/incident-runbook.js create mode 100644 meter-simulator/tests/incident-runbook.test.js diff --git a/.gitignore b/.gitignore index 1754383..95e79b5 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,6 @@ target AGENTS.md .agent/ agents/ -issue.md \ No newline at end of file +issue.md +# JavaScript dependencies +node_modules/ diff --git a/docs/INCIDENT_RUNBOOK_AUTOMATION.md b/docs/INCIDENT_RUNBOOK_AUTOMATION.md new file mode 100644 index 0000000..9600901 --- /dev/null +++ b/docs/INCIDENT_RUNBOOK_AUTOMATION.md @@ -0,0 +1,37 @@ +# Incident Response Runbook Automation + +This design adds a lightweight incident automation layer for Utility Protocol operators. It evaluates health snapshots locally, maps failures to runbook incident rules, and sends actionable PagerDuty Events API v2 triggers only after a rule matches. + +## Architecture + +1. Monitoring jobs collect contract and service signals such as pause state, TTL, oracle freshness, latency, and error rate. +2. `IncidentRunbookAutomation.evaluate()` performs deterministic in-process rule evaluation for the critical path. +3. `IncidentRunbookAutomation.process()` applies deduplication suppression and calls `PagerDutyClient.trigger()` for unsuppressed incidents. +4. PagerDuty routes alerts to the configured escalation policy, where responders follow `EMERGENCY_RUNBOOK.md` containment and recovery procedures. + +The critical-path evaluator is synchronous and does not perform network I/O. PagerDuty calls happen after incident detection, preserving the sub-100ms P99 target for health evaluation loops. + +## PagerDuty Configuration + +Set these environment variables in the monitoring runtime: + +- `PAGERDUTY_ROUTING_KEY`: Events API v2 integration routing key for the Utility Protocol service. +- `PAGERDUTY_EVENTS_API_URL`: Optional override for tests or private event gateways. Defaults to `https://events.pagerduty.com/v2/enqueue`. + +## Built-in Incident Rules + +| Rule | Severity | Condition | Runbook action | +| --- | --- | --- | --- | +| `contract-paused` | critical | Contract pause flag is true | Begin emergency governance coordination. | +| `ttl-low` | error | Remaining TTL is below 1,000 ledgers | Run TTL extension procedure. | +| `stale-oracle` | error | Oracle age exceeds 300 seconds | Fail over oracle operations and investigate feed health. | +| `latency-slo-breach` | warning | Critical path P99 latency is at least 100ms | Inspect recent deployments and canary metrics. | +| `error-rate-high` | error | Service error rate is at least 1% | Triage service logs and consider rollback. | + +## Deployment Strategy + +Deploy automation in blue-green mode beside the existing monitoring worker. During canary, set a non-production PagerDuty routing key and compare generated incidents against dashboard signals. Promote to production routing only after false positives and duplicate suppression are validated. + +## Security Notes + +Store the PagerDuty routing key in the deployment secret manager, never in source control. Treat generated PagerDuty incidents as operational metadata; do not place private keys, signatures, or customer PII in `customDetails` snapshots. diff --git a/meter-simulator/package.json b/meter-simulator/package.json index 27d7d00..bea89e4 100644 --- a/meter-simulator/package.json +++ b/meter-simulator/package.json @@ -11,7 +11,8 @@ "test": "jest", "lint": "eslint src/", "ttl-monitor": "node src/ttl-monitor.js", - "ttl-check": "node src/ttl-monitor.js --check-only" + "ttl-check": "node src/ttl-monitor.js --check-only", + "incident-runbook": "node src/incident-runbook.js" }, "keywords": ["iot", "stellar", "soroban", "utility", "meter", "simulator"], "author": "Utility-Protocol Team", diff --git a/meter-simulator/src/incident-runbook.js b/meter-simulator/src/incident-runbook.js new file mode 100644 index 0000000..e55689c --- /dev/null +++ b/meter-simulator/src/incident-runbook.js @@ -0,0 +1,132 @@ +#!/usr/bin/env node + +/** + * Incident response runbook automation with PagerDuty Events API v2 integration. + * + * The automation intentionally keeps critical-path evaluation local and synchronous; + * PagerDuty network I/O only occurs after a rule has matched. This keeps health + * checks suitable for sub-100ms polling loops while still escalating actionable + * incidents to responders. + */ + +const axios = require('axios'); + +const DEFAULT_EVENTS_API_URL = 'https://events.pagerduty.com/v2/enqueue'; + +const Severity = Object.freeze({ + CRITICAL: 'critical', + ERROR: 'error', + WARNING: 'warning', + INFO: 'info' +}); + +class PagerDutyClient { + constructor({ routingKey = process.env.PAGERDUTY_ROUTING_KEY, apiUrl = process.env.PAGERDUTY_EVENTS_API_URL || DEFAULT_EVENTS_API_URL, timeoutMs = 5000 } = {}) { + if (!routingKey) { + throw new Error('PAGERDUTY_ROUTING_KEY environment variable is required'); + } + this.routingKey = routingKey; + this.apiUrl = apiUrl; + this.timeoutMs = timeoutMs; + } + + async trigger({ dedupKey, summary, source, severity, customDetails = {} }) { + const payload = { + routing_key: this.routingKey, + event_action: 'trigger', + dedup_key: dedupKey, + payload: { + summary, + source, + severity, + timestamp: new Date().toISOString(), + custom_details: customDetails + } + }; + + const response = await axios.post(this.apiUrl, payload, { timeout: this.timeoutMs }); + return { status: response.status, dedupKey: response.data && response.data.dedup_key ? response.data.dedup_key : dedupKey }; + } +} + +class IncidentRunbookAutomation { + constructor({ pagerDutyClient, serviceName = 'utility-contracts', now = () => Date.now(), suppressMs = 15 * 60 * 1000 } = {}) { + this.pagerDutyClient = pagerDutyClient; + this.serviceName = serviceName; + this.now = now; + this.suppressMs = suppressMs; + this.lastTriggeredAt = new Map(); + } + + evaluate(snapshot) { + const incidents = []; + const add = (rule, severity, summary, details = {}) => { + incidents.push({ + rule, + severity, + summary, + source: this.serviceName, + dedupKey: `${this.serviceName}:${rule}`, + customDetails: { ...snapshot, ...details } + }); + }; + + if (snapshot.contractPaused === true) { + add('contract-paused', Severity.CRITICAL, 'Utility contract is paused'); + } + + if (typeof snapshot.remainingTtlLedgers === 'number' && snapshot.remainingTtlLedgers < 1000) { + add('ttl-low', Severity.ERROR, 'Contract TTL is below the safe operating threshold', { threshold: 1000 }); + } + + if (typeof snapshot.oracleAgeSeconds === 'number' && snapshot.oracleAgeSeconds > 300) { + add('stale-oracle', Severity.ERROR, 'Oracle price feed is stale', { thresholdSeconds: 300 }); + } + + if (typeof snapshot.p99LatencyMs === 'number' && snapshot.p99LatencyMs >= 100) { + add('latency-slo-breach', Severity.WARNING, 'Critical path P99 latency exceeds 100ms target', { targetMs: 100 }); + } + + if (typeof snapshot.errorRatePercent === 'number' && snapshot.errorRatePercent >= 1) { + add('error-rate-high', Severity.ERROR, 'Service error rate exceeds incident threshold', { thresholdPercent: 1 }); + } + + return incidents; + } + + async process(snapshot) { + const incidents = this.evaluate(snapshot); + const triggered = []; + + for (const incident of incidents) { + if (this.shouldSuppress(incident.dedupKey)) { + continue; + } + const result = await this.pagerDutyClient.trigger(incident); + this.lastTriggeredAt.set(incident.dedupKey, this.now()); + triggered.push({ ...incident, pagerDuty: result }); + } + + return { incidents, triggered }; + } + + shouldSuppress(dedupKey) { + const previous = this.lastTriggeredAt.get(dedupKey); + return typeof previous === 'number' && this.now() - previous < this.suppressMs; + } +} + +if (require.main === module) { + const snapshot = JSON.parse(process.env.INCIDENT_SNAPSHOT || '{}'); + const automation = new IncidentRunbookAutomation({ pagerDutyClient: new PagerDutyClient() }); + automation.process(snapshot) + .then((result) => { + console.log(JSON.stringify(result, null, 2)); + }) + .catch((error) => { + console.error(`Incident automation failed: ${error.message}`); + process.exit(1); + }); +} + +module.exports = { IncidentRunbookAutomation, PagerDutyClient, Severity }; diff --git a/meter-simulator/tests/incident-runbook.test.js b/meter-simulator/tests/incident-runbook.test.js new file mode 100644 index 0000000..1cce242 --- /dev/null +++ b/meter-simulator/tests/incident-runbook.test.js @@ -0,0 +1,63 @@ +const axios = require('axios'); +const { IncidentRunbookAutomation, PagerDutyClient } = require('../src/incident-runbook'); + +jest.mock('axios'); + +describe('IncidentRunbookAutomation', () => { + test('detects runbook incidents from service health snapshots', () => { + const automation = new IncidentRunbookAutomation({ pagerDutyClient: { trigger: jest.fn() } }); + + const incidents = automation.evaluate({ + contractPaused: true, + remainingTtlLedgers: 99, + oracleAgeSeconds: 301, + p99LatencyMs: 125, + errorRatePercent: 1.5 + }); + + expect(incidents.map((incident) => incident.rule)).toEqual([ + 'contract-paused', + 'ttl-low', + 'stale-oracle', + 'latency-slo-breach', + 'error-rate-high' + ]); + expect(incidents[0].dedupKey).toBe('utility-contracts:contract-paused'); + }); + + test('triggers PagerDuty once per suppression window', async () => { + let now = 1000; + const trigger = jest.fn().mockResolvedValue({ status: 202, dedupKey: 'utility-contracts:ttl-low' }); + const automation = new IncidentRunbookAutomation({ pagerDutyClient: { trigger }, now: () => now, suppressMs: 60000 }); + + await automation.process({ remainingTtlLedgers: 10 }); + await automation.process({ remainingTtlLedgers: 10 }); + now += 60001; + await automation.process({ remainingTtlLedgers: 10 }); + + expect(trigger).toHaveBeenCalledTimes(2); + }); +}); + +describe('PagerDutyClient', () => { + test('posts Events API v2 trigger payloads', async () => { + axios.post.mockResolvedValue({ status: 202, data: { dedup_key: 'custom-dedup' } }); + const client = new PagerDutyClient({ routingKey: 'routing-key', apiUrl: 'https://example.test/enqueue', timeoutMs: 1234 }); + + const result = await client.trigger({ + dedupKey: 'service:rule', + summary: 'summary', + source: 'service', + severity: 'critical', + customDetails: { contractPaused: true } + }); + + expect(axios.post).toHaveBeenCalledWith('https://example.test/enqueue', expect.objectContaining({ + routing_key: 'routing-key', + event_action: 'trigger', + dedup_key: 'service:rule', + payload: expect.objectContaining({ severity: 'critical' }) + }), { timeout: 1234 }); + expect(result).toEqual({ status: 202, dedupKey: 'custom-dedup' }); + }); +});