Incident response runbook

.gitignore

@@ -9,4 +9,6 @@ target
 AGENTS.md
 .agent/
 agents/
-issue.md
+issue.md
+# JavaScript dependencies
+node_modules/

docs/INCIDENT_RUNBOOK_AUTOMATION.md

@@ -0,0 +1,37 @@
+# Incident Response Runbook Automation
+
+This design adds a lightweight incident automation layer for Utility Protocol operators. It evaluates health snapshots locally, maps failures to runbook incident rules, and sends actionable PagerDuty Events API v2 triggers only after a rule matches.
+
+## Architecture
+
+1. Monitoring jobs collect contract and service signals such as pause state, TTL, oracle freshness, latency, and error rate.
+2. `IncidentRunbookAutomation.evaluate()` performs deterministic in-process rule evaluation for the critical path.
+3. `IncidentRunbookAutomation.process()` applies deduplication suppression and calls `PagerDutyClient.trigger()` for unsuppressed incidents.
+4. PagerDuty routes alerts to the configured escalation policy, where responders follow `EMERGENCY_RUNBOOK.md` containment and recovery procedures.
+
+The critical-path evaluator is synchronous and does not perform network I/O. PagerDuty calls happen after incident detection, preserving the sub-100ms P99 target for health evaluation loops.
+
+## PagerDuty Configuration
+
+Set these environment variables in the monitoring runtime:
+
+- `PAGERDUTY_ROUTING_KEY`: Events API v2 integration routing key for the Utility Protocol service.
+- `PAGERDUTY_EVENTS_API_URL`: Optional override for tests or private event gateways. Defaults to `https://events.pagerduty.com/v2/enqueue`.
+
+## Built-in Incident Rules
+
+| Rule | Severity | Condition | Runbook action |
+| --- | --- | --- | --- |
+| `contract-paused` | critical | Contract pause flag is true | Begin emergency governance coordination. |
+| `ttl-low` | error | Remaining TTL is below 1,000 ledgers | Run TTL extension procedure. |
+| `stale-oracle` | error | Oracle age exceeds 300 seconds | Fail over oracle operations and investigate feed health. |
+| `latency-slo-breach` | warning | Critical path P99 latency is at least 100ms | Inspect recent deployments and canary metrics. |
+| `error-rate-high` | error | Service error rate is at least 1% | Triage service logs and consider rollback. |
+
+## Deployment Strategy
+
+Deploy automation in blue-green mode beside the existing monitoring worker. During canary, set a non-production PagerDuty routing key and compare generated incidents against dashboard signals. Promote to production routing only after false positives and duplicate suppression are validated.
+
+## Security Notes
+
+Store the PagerDuty routing key in the deployment secret manager, never in source control. Treat generated PagerDuty incidents as operational metadata; do not place private keys, signatures, or customer PII in `customDetails` snapshots.

meter-simulator/package.json

@@ -11,7 +11,8 @@
     "test": "jest",
     "lint": "eslint src/",
     "ttl-monitor": "node src/ttl-monitor.js",
-    "ttl-check": "node src/ttl-monitor.js --check-only"
+    "ttl-check": "node src/ttl-monitor.js --check-only",
+    "incident-runbook": "node src/incident-runbook.js"
   },
   "keywords": ["iot", "stellar", "soroban", "utility", "meter", "simulator"],
   "author": "Utility-Protocol Team",

meter-simulator/src/incident-runbook.js

@@ -0,0 +1,132 @@
+#!/usr/bin/env node
+
+/**
+ * Incident response runbook automation with PagerDuty Events API v2 integration.
+ *
+ * The automation intentionally keeps critical-path evaluation local and synchronous;
+ * PagerDuty network I/O only occurs after a rule has matched. This keeps health
+ * checks suitable for sub-100ms polling loops while still escalating actionable
+ * incidents to responders.
+ */
+
+const axios = require('axios');
+
+const DEFAULT_EVENTS_API_URL = 'https://events.pagerduty.com/v2/enqueue';
+
+const Severity = Object.freeze({
+  CRITICAL: 'critical',
+  ERROR: 'error',
+  WARNING: 'warning',
+  INFO: 'info'
+});
+
+class PagerDutyClient {
+  constructor({ routingKey = process.env.PAGERDUTY_ROUTING_KEY, apiUrl = process.env.PAGERDUTY_EVENTS_API_URL || DEFAULT_EVENTS_API_URL, timeoutMs = 5000 } = {}) {
+    if (!routingKey) {
+      throw new Error('PAGERDUTY_ROUTING_KEY environment variable is required');
+    }
+    this.routingKey = routingKey;
+    this.apiUrl = apiUrl;
+    this.timeoutMs = timeoutMs;
+  }
+
+  async trigger({ dedupKey, summary, source, severity, customDetails = {} }) {
+    const payload = {
+      routing_key: this.routingKey,
+      event_action: 'trigger',
+      dedup_key: dedupKey,
+      payload: {
+        summary,
+        source,
+        severity,
+        timestamp: new Date().toISOString(),
+        custom_details: customDetails
+      }
+    };
+
+    const response = await axios.post(this.apiUrl, payload, { timeout: this.timeoutMs });
+    return { status: response.status, dedupKey: response.data && response.data.dedup_key ? response.data.dedup_key : dedupKey };
+  }
+}
+
+class IncidentRunbookAutomation {
+  constructor({ pagerDutyClient, serviceName = 'utility-contracts', now = () => Date.now(), suppressMs = 15 * 60 * 1000 } = {}) {
+    this.pagerDutyClient = pagerDutyClient;
+    this.serviceName = serviceName;
+    this.now = now;
+    this.suppressMs = suppressMs;
+    this.lastTriggeredAt = new Map();
+  }
+
+  evaluate(snapshot) {
+    const incidents = [];
+    const add = (rule, severity, summary, details = {}) => {
+      incidents.push({
+        rule,
+        severity,
+        summary,
+        source: this.serviceName,
+        dedupKey: `${this.serviceName}:${rule}`,
+        customDetails: { ...snapshot, ...details }
+      });
+    };
+
+    if (snapshot.contractPaused === true) {
+      add('contract-paused', Severity.CRITICAL, 'Utility contract is paused');
+    }
+
+    if (typeof snapshot.remainingTtlLedgers === 'number' && snapshot.remainingTtlLedgers < 1000) {
+      add('ttl-low', Severity.ERROR, 'Contract TTL is below the safe operating threshold', { threshold: 1000 });
+    }
+
+    if (typeof snapshot.oracleAgeSeconds === 'number' && snapshot.oracleAgeSeconds > 300) {
+      add('stale-oracle', Severity.ERROR, 'Oracle price feed is stale', { thresholdSeconds: 300 });
+    }
+
+    if (typeof snapshot.p99LatencyMs === 'number' && snapshot.p99LatencyMs >= 100) {
+      add('latency-slo-breach', Severity.WARNING, 'Critical path P99 latency exceeds 100ms target', { targetMs: 100 });
+    }
+
+    if (typeof snapshot.errorRatePercent === 'number' && snapshot.errorRatePercent >= 1) {
+      add('error-rate-high', Severity.ERROR, 'Service error rate exceeds incident threshold', { thresholdPercent: 1 });
+    }
+
+    return incidents;
+  }
+
+  async process(snapshot) {
+    const incidents = this.evaluate(snapshot);
+    const triggered = [];
+
+    for (const incident of incidents) {
+      if (this.shouldSuppress(incident.dedupKey)) {
+        continue;
+      }
+      const result = await this.pagerDutyClient.trigger(incident);
+      this.lastTriggeredAt.set(incident.dedupKey, this.now());
+      triggered.push({ ...incident, pagerDuty: result });
+    }
+
+    return { incidents, triggered };
+  }
+
+  shouldSuppress(dedupKey) {
+    const previous = this.lastTriggeredAt.get(dedupKey);
+    return typeof previous === 'number' && this.now() - previous < this.suppressMs;
+  }
+}
+
+if (require.main === module) {
+  const snapshot = JSON.parse(process.env.INCIDENT_SNAPSHOT || '{}');
+  const automation = new IncidentRunbookAutomation({ pagerDutyClient: new PagerDutyClient() });
+  automation.process(snapshot)
+    .then((result) => {
+      console.log(JSON.stringify(result, null, 2));
+    })
+    .catch((error) => {
+      console.error(`Incident automation failed: ${error.message}`);
+      process.exit(1);
+    });
+}
+
+module.exports = { IncidentRunbookAutomation, PagerDutyClient, Severity };

meter-simulator/tests/incident-runbook.test.js

@@ -0,0 +1,63 @@
+const axios = require('axios');
+const { IncidentRunbookAutomation, PagerDutyClient } = require('../src/incident-runbook');
+
+jest.mock('axios');
+
+describe('IncidentRunbookAutomation', () => {
+  test('detects runbook incidents from service health snapshots', () => {
+    const automation = new IncidentRunbookAutomation({ pagerDutyClient: { trigger: jest.fn() } });
+
+    const incidents = automation.evaluate({
+      contractPaused: true,
+      remainingTtlLedgers: 99,
+      oracleAgeSeconds: 301,
+      p99LatencyMs: 125,
+      errorRatePercent: 1.5
+    });
+
+    expect(incidents.map((incident) => incident.rule)).toEqual([
+      'contract-paused',
+      'ttl-low',
+      'stale-oracle',
+      'latency-slo-breach',
+      'error-rate-high'
+    ]);
+    expect(incidents[0].dedupKey).toBe('utility-contracts:contract-paused');
+  });
+
+  test('triggers PagerDuty once per suppression window', async () => {
+    let now = 1000;
+    const trigger = jest.fn().mockResolvedValue({ status: 202, dedupKey: 'utility-contracts:ttl-low' });
+    const automation = new IncidentRunbookAutomation({ pagerDutyClient: { trigger }, now: () => now, suppressMs: 60000 });
+
+    await automation.process({ remainingTtlLedgers: 10 });
+    await automation.process({ remainingTtlLedgers: 10 });
+    now += 60001;
+    await automation.process({ remainingTtlLedgers: 10 });
+
+    expect(trigger).toHaveBeenCalledTimes(2);
+  });
+});
+
+describe('PagerDutyClient', () => {
+  test('posts Events API v2 trigger payloads', async () => {
+    axios.post.mockResolvedValue({ status: 202, data: { dedup_key: 'custom-dedup' } });
+    const client = new PagerDutyClient({ routingKey: 'routing-key', apiUrl: 'https://example.test/enqueue', timeoutMs: 1234 });
+
+    const result = await client.trigger({
+      dedupKey: 'service:rule',
+      summary: 'summary',
+      source: 'service',
+      severity: 'critical',
+      customDetails: { contractPaused: true }
+    });
+
+    expect(axios.post).toHaveBeenCalledWith('https://example.test/enqueue', expect.objectContaining({
+      routing_key: 'routing-key',
+      event_action: 'trigger',
+      dedup_key: 'service:rule',
+      payload: expect.objectContaining({ severity: 'critical' })
+    }), { timeout: 1234 });
+    expect(result).toEqual({ status: 202, dedupKey: 'custom-dedup' });
+  });
+});