From 83d76f6dec1c61dfc86075b175c6945092ba7782 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:12:55 -0400 Subject: [PATCH 1/2] ci: retire the queued-job and actions-budget monitors (Phase 7 step 6) Deletes queued-job-monitor.yml and actions-budget-monitor.yml along with their scripts (queue-monitor.cjs, queue-monitor.test.cjs, budget-monitor.cjs, budget-monitor.test.cjs) and their documentation pages (docs/queue-monitor.md, docs/actions-budget-monitor.md). Both workflows are superseded by pool-alert.yml in github-iac, per the ci-perf Phase 7 routing retirement plan (melodic-software/github-iac#378, sub-issue #425), step 6. incident-issue.cjs is also deleted. A grep sweep of the tree after the other deletions showed its only requirers were queue-monitor.cjs, queue-monitor.test.cjs and budget-monitor.cjs, all removed in this change, so no surviving workflow or script depends on it. README.md's two links to the deleted docs are removed, and docs/releases.md's description of the release-tag verification job is updated to say "workflow-script tests" instead of naming the now-deleted queue-monitor tests specifically. release/dependencies.json's actions/create-github-app-token entry is removed because it was minted only by queued-job-monitor.yml to read the observer App installation, and Test-ReleasePins.ps1 fails closed on any manifest entry no workflow uses. docs/topics/repo-simplify-sweep/RUN-STATE.md and docs/releases.md's historical note about the v0.1.20 tag reservation still name these files; both are historical records of past runs and are left untouched. No ci-workflows pin changed. ci-runner stays pinned at v0.22.2 (5776760254f8b63cba44e896f51604cb755350d9) by decision; this change touches none of its governed references. This PR opens as a draft. Its merge gates on pool-alert.yml showing seven green scheduled runs over seven consecutive days (currently 4 of 7). Co-Authored-By: Claude Fable 5.1 --- .github/scripts/budget-monitor.cjs | 396 ----------- .github/scripts/budget-monitor.test.cjs | 666 ------------------ .github/scripts/incident-issue.cjs | 100 --- .github/scripts/queue-monitor.cjs | 272 -------- .github/scripts/queue-monitor.test.cjs | 696 ------------------- .github/workflows/actions-budget-monitor.yml | 69 -- .github/workflows/queued-job-monitor.yml | 116 ---- docs/actions-budget-monitor.md | 105 --- docs/queue-monitor.md | 173 ----- 9 files changed, 2593 deletions(-) delete mode 100644 .github/scripts/budget-monitor.cjs delete mode 100644 .github/scripts/budget-monitor.test.cjs delete mode 100644 .github/scripts/incident-issue.cjs delete mode 100644 .github/scripts/queue-monitor.cjs delete mode 100644 .github/scripts/queue-monitor.test.cjs delete mode 100644 .github/workflows/actions-budget-monitor.yml delete mode 100644 .github/workflows/queued-job-monitor.yml delete mode 100644 docs/actions-budget-monitor.md delete mode 100644 docs/queue-monitor.md diff --git a/.github/scripts/budget-monitor.cjs b/.github/scripts/budget-monitor.cjs deleted file mode 100644 index 197fe1d..0000000 --- a/.github/scripts/budget-monitor.cjs +++ /dev/null @@ -1,396 +0,0 @@ -'use strict'; - -const { - boundBodyLength, - escapeMarkdownTableCell, - findOpenIncident, -} = require('./incident-issue.cjs'); - -// Percentages of the included-minute allowance that open a pool incident. Both -// stay breached once crossed within a month (consumption only rises until the -// allowance resets), so the body reports the highest tier reached and the -// escalation between them is a comment, not a second issue. -const POOL_ALERT_THRESHOLD_PERCENTS = Object.freeze([50, 80]); - -const ACTIONS_PRODUCT = 'actions'; -const MINUTE_UNIT_TYPES = Object.freeze(['minutes', 'minute']); - -// Standard hosted runner SKUs only — excludes larger runners, self-hosted, and -// non-minute Actions products. -const STANDARD_HOSTED_SKUS = Object.freeze([ - 'Actions Linux', - 'Actions Linux Slim', - 'Actions Windows', -]); - -const MAX_SKU_TABLE_ROWS = 25; - -function isActionsProduct(item) { - return String(item.product ?? '').toLowerCase() === ACTIONS_PRODUCT; -} - -function isActionsMinuteItem(item) { - return isActionsProduct(item) - && MINUTE_UNIT_TYPES.includes(String(item.unitType ?? '').toLowerCase()); -} - -function isStandardHostedSku(item) { - return STANDARD_HOSTED_SKUS.includes(String(item.sku ?? '')); -} - -function hasRequiredFields(item) { - const quantity = Number(item.quantity); - return Boolean(item.organizationName) - && Boolean(item.repositoryName) - && Boolean(item.sku) - && Number.isFinite(quantity) - && quantity >= 0; -} - -function roundToTenth(value) { - return Math.round(value * 10) / 10; -} - -function billingMonth(now) { - const date = new Date(now); - return { year: date.getUTCFullYear(), month: date.getUTCMonth() + 1 }; -} - -function formatBillingMonth({ year, month }) { - return `${year}-${String(month).padStart(2, '0')}`; -} - -async function resolveRepositoryVisibility({ github, owner, repo, cache }) { - const key = `${owner}/${repo}`; - if (cache.has(key)) return cache.get(key); - try { - const response = await github.rest.repos.get({ owner, repo }); - const visibility = response.data.private ? 'private' : 'public'; - cache.set(key, visibility); - return visibility; - } catch { - // Frontier tiebreak (#171): when visibility is unknown — 404, permission - // error, or any other lookup failure — include the row's minutes rather - // than drop them. Under-counting would suppress alerts; over-counting is - // the safer failure mode for a budget watchdog. - cache.set(key, 'unknown'); - return 'unknown'; - } -} - -function minutesForRow(item, visibility) { - if (visibility === 'public') return 0; - const quantity = Number(item.quantity); - if (!Number.isFinite(quantity) || quantity <= 0) return 0; - return quantity; -} - -async function summarizeUsage(usageItems, { includedMinutes, github }) { - if (!Array.isArray(usageItems)) { - throw new Error('The billing usage report must expose a usageItems array.'); - } - if (!Number.isFinite(includedMinutes) || includedMinutes <= 0) { - throw new Error('Included minutes must be a positive number.'); - } - - const actionsItems = usageItems.filter(isActionsMinuteItem); - const eligibleItems = actionsItems.filter(item => isStandardHostedSku(item) && hasRequiredFields(item)); - - // Fail loud only when a standard hosted runner SKU reports a non-minute unit — - // that is a vocabulary change for the rows this monitor counts. Legitimate - // non-minute Actions products (storage, etc.) are excluded and must not block - // a zero-consumption month before any runner-minute row exists. - const runnerSkusWithUnexpectedUnit = usageItems.filter( - item => isActionsProduct(item) - && isStandardHostedSku(item) - && !isActionsMinuteItem(item), - ); - if (runnerSkusWithUnexpectedUnit.length > 0) { - throw new Error('A standard hosted Actions runner SKU was reported without a minute unit type; the usage-report vocabulary changed.'); - } - - const visibilityCache = new Map(); - const minutesBySku = new Map(); - for (const item of eligibleItems) { - const visibility = await resolveRepositoryVisibility({ - github, - owner: item.organizationName, - repo: item.repositoryName, - cache: visibilityCache, - }); - const minutes = minutesForRow(item, visibility); - if (minutes <= 0) continue; - const sku = String(item.sku); - minutesBySku.set(sku, (minutesBySku.get(sku) || 0) + minutes); - } - - const consumedMinutes = [...minutesBySku.values()].reduce((total, minutes) => total + minutes, 0); - const percentUsed = (consumedMinutes / includedMinutes) * 100; - - return { - includedMinutes, - consumedMinutes: roundToTenth(consumedMinutes), - percentUsed: roundToTenth(percentUsed), - breachedThresholds: POOL_ALERT_THRESHOLD_PERCENTS.filter(threshold => percentUsed >= threshold), - perSku: [...minutesBySku.entries()] - .map(([sku, minutes]) => ({ sku, minutes: roundToTenth(minutes) })) - .sort((left, right) => right.minutes - left.minutes || left.sku.localeCompare(right.sku)), - }; -} - -async function fetchUsage({ github, org, year, month }) { - const response = await github.request('GET /organizations/{org}/settings/billing/usage', { - org, - year, - month, - }); - return response.data?.usageItems; -} - -async function run({ github, core, env = process.env, now = Date.now() }) { - const org = env.BILLING_ORG; - const includedMinutes = Number(env.INCLUDED_MINUTES); - - if (env.BUDGET_MONITOR_ARMED !== 'true') { - core.warning('Actions budget monitor is disarmed (BUDGET_MONITOR_ARMED != true); skipping measurement.'); - return; - } - - let summary; - let month; - try { - if (!org) { - throw new Error('BILLING_ORG is required to scope the billing usage report.'); - } - if (env.BILLING_TOKEN_PRESENT !== 'true') { - throw new Error('The billing observer token is not provisioned: set the CI_RUNNER_BILLING_OBSERVER_TOKEN secret. Refusing to report consumption without it.'); - } - month = billingMonth(now); - summary = await summarizeUsage(await fetchUsage({ github, org, ...month }), { includedMinutes, github }); - } catch (error) { - core.setFailed(error instanceof Error ? error.message : String(error)); - return; - } - - const budget = { org, month: formatBillingMonth(month), ...summary }; - - core.setOutput('budget', JSON.stringify(budget)); - - await core.summary - .addHeading('Actions included-minute consumption') - .addRaw(`\`${org}\` used ${budget.consumedMinutes} of ${includedMinutes} included minute(s) in ${budget.month} (${budget.percentUsed}%).`) - .write(); -} - -function poolIncidentTitle(org) { - return `[Alert] Actions included-minute consumption — ${org}`; -} - -function poolIncidentMarker(org) { - return ``; -} - -function tierMarker(month, threshold) { - return ``; -} - -function renderSkuMarkdownTable(perSku, { maxRows = MAX_SKU_TABLE_ROWS } = {}) { - const header = '| SKU | Included minutes consumed |\n| --- | --- |'; - const shown = perSku.slice(0, maxRows); - const rows = shown.map(entry => `| ${escapeMarkdownTableCell(entry.sku)} | ${entry.minutes} |`); - const lines = [header, ...rows]; - const remaining = perSku.length - shown.length; - if (remaining > 0) { - lines.push('', `_...and ${remaining} more SKU(s)._`); - } - return lines.join('\n'); -} - -const poolRecoverySummary = ` -Included-minute consumption is a hard-\`$0\` posture signal, not a runner-capacity failure. Confirm which repositories and workflows are drawing on the allowance, and move eligible work onto free capacity: standard GitHub-hosted runners are free for public repositories, and self-hosted runners consume no included minutes. A new billing month resets the allowance and closes this incident on its own. -`; - -function parseBudget(budgetJson) { - if (budgetJson === undefined || budgetJson === '') { - throw new Error('BUDGET_JSON is required: the measurement step must set it via core.setOutput.'); - } - let budget; - try { - budget = JSON.parse(budgetJson); - } catch (error) { - throw new Error(`BUDGET_JSON is not valid JSON: ${error instanceof Error ? error.message : String(error)}`); - } - if (budget === null || typeof budget !== 'object' || Array.isArray(budget)) { - throw new Error('BUDGET_JSON must decode to an object.'); - } - if (!Array.isArray(budget.breachedThresholds)) { - throw new Error('BUDGET_JSON must carry a breachedThresholds array.'); - } - return budget; -} - -async function closeIncident({ github, homeOwner, homeRepo, core, existing, recoveryBody }) { - await github.rest.issues.createComment({ - owner: homeOwner, - repo: homeRepo, - issue_number: existing.number, - body: recoveryBody, - }); - await github.rest.issues.update({ - owner: homeOwner, - repo: homeRepo, - issue_number: existing.number, - state: 'closed', - state_reason: 'completed', - }); - core.info(`Closed incident #${existing.number}.`); -} - -function reportedThresholdsForMonth(body, month, breachedThresholds) { - return breachedThresholds.filter(threshold => (body ?? '').includes(tierMarker(month, threshold))); -} - -function buildPoolIncidentBody({ budget, nowIso, reportedThresholds, marker }) { - const highestThreshold = Math.max(...budget.breachedThresholds); - const tierMarkers = reportedThresholds.map(threshold => tierMarker(budget.month, threshold)).join(''); - const bodyWithoutMarker = [ - tierMarkers, - '', - `Actions included-minute consumption alert for \`${budget.org}\`: ${highestThreshold}% of the included allowance reached.`, - '', - `Billing month ${budget.month}: ${budget.consumedMinutes} of ${budget.includedMinutes} included minute(s) consumed (${budget.percentUsed}%). Last confirmed ${nowIso}.`, - '', - renderSkuMarkdownTable(budget.perSku), - '', - poolRecoverySummary.trim(), - ].join('\n'); - return boundBodyLength(bodyWithoutMarker, marker); -} - -async function upsertPoolIncident({ github, core, homeOwner, homeRepo, issueAuthorLogin, budget, nowIso }) { - const marker = poolIncidentMarker(budget.org); - const existing = await findOpenIncident({ github, homeOwner, homeRepo, marker, issueAuthorLogin }); - - if (budget.breachedThresholds.length === 0) { - if (!existing) { - core.info(`No open pool incident for ${budget.org}; included-minute consumption is under every threshold.`); - return; - } - await closeIncident({ - github, - homeOwner, - homeRepo, - core, - existing, - recoveryBody: `Recovered: \`${budget.org}\` has consumed ${budget.consumedMinutes} of ${budget.includedMinutes} included minute(s) in ${budget.month} (${budget.percentUsed}%), under every alert threshold, as of ${nowIso}.`, - }); - return; - } - - const highestThreshold = Math.max(...budget.breachedThresholds); - - if (!existing) { - const body = buildPoolIncidentBody({ - budget, - nowIso, - reportedThresholds: budget.breachedThresholds, - marker, - }); - const created = await github.rest.issues.create({ - owner: homeOwner, - repo: homeRepo, - title: poolIncidentTitle(budget.org), - body, - labels: ['automated'], - }); - core.info(`Opened pool incident #${created.data.number} for ${budget.org}.`); - return; - } - - const reportedThresholds = reportedThresholdsForMonth(existing.body, budget.month, budget.breachedThresholds); - const escalatedThresholds = budget.breachedThresholds.filter(threshold => !reportedThresholds.includes(threshold)); - - if (escalatedThresholds.length === 0) { - const body = buildPoolIncidentBody({ - budget, - nowIso, - reportedThresholds: budget.breachedThresholds, - marker, - }); - await github.rest.issues.update({ owner: homeOwner, repo: homeRepo, issue_number: existing.number, body }); - core.info(`Updated pool incident #${existing.number} for ${budget.org}.`); - return; - } - - // Retry-safe escalation: refresh the body and notify before persisting tier - // markers for thresholds whose comment has not yet succeeded. If the comment - // fails, the next run still sees the threshold as unreported for this month. - const bodyBeforeEscalation = buildPoolIncidentBody({ - budget, - nowIso, - reportedThresholds, - marker, - }); - await github.rest.issues.update({ owner: homeOwner, repo: homeRepo, issue_number: existing.number, body: bodyBeforeEscalation }); - await github.rest.issues.createComment({ - owner: homeOwner, - repo: homeRepo, - issue_number: existing.number, - body: `Escalated to ${highestThreshold}% of the included allowance: ${budget.consumedMinutes} of ${budget.includedMinutes} minute(s) consumed in ${budget.month} as of ${nowIso}.`, - }); - const bodyAfterEscalation = buildPoolIncidentBody({ - budget, - nowIso, - reportedThresholds: budget.breachedThresholds, - marker, - }); - await github.rest.issues.update({ owner: homeOwner, repo: homeRepo, issue_number: existing.number, body: bodyAfterEscalation }); - core.info(`Updated pool incident #${existing.number} for ${budget.org}.`); -} - -async function upsertIncident({ github, core, env = process.env, now = Date.now() }) { - if (env.BUDGET_MONITOR_ARMED !== 'true') { - core.warning('Actions budget monitor is disarmed (BUDGET_MONITOR_ARMED != true); skipping incident upsert.'); - return; - } - - const [homeOwner, homeRepo] = (env.GITHUB_REPOSITORY || '').split('/'); - const issueAuthorLogin = env.ISSUE_AUTHOR_LOGIN; - if (!homeOwner || !homeRepo) { - throw new Error('GITHUB_REPOSITORY must be set to the owner/repo of the monitor workflow.'); - } - if (!issueAuthorLogin) { - throw new Error('ISSUE_AUTHOR_LOGIN is required to restrict incident-issue adoption to this workflow\'s own identity.'); - } - const budget = parseBudget(env.BUDGET_JSON); - if (!budget.org) { - throw new Error('BUDGET_JSON must carry the organization it measured.'); - } - const nowIso = new Date(now).toISOString(); - - await upsertPoolIncident({ github, core, homeOwner, homeRepo, issueAuthorLogin, budget, nowIso }); -} - -module.exports = { - STANDARD_HOSTED_SKUS, - billingMonth, - fetchUsage, - formatBillingMonth, - hasRequiredFields, - isStandardHostedSku, - MAX_SKU_TABLE_ROWS, - minutesForRow, - parseBudget, - POOL_ALERT_THRESHOLD_PERCENTS, - poolIncidentMarker, - poolIncidentTitle, - poolRecoverySummary, - renderSkuMarkdownTable, - resolveRepositoryVisibility, - run, - summarizeUsage, - buildPoolIncidentBody, - reportedThresholdsForMonth, - tierMarker, - upsertIncident, -}; diff --git a/.github/scripts/budget-monitor.test.cjs b/.github/scripts/budget-monitor.test.cjs deleted file mode 100644 index 69e0f8a..0000000 --- a/.github/scripts/budget-monitor.test.cjs +++ /dev/null @@ -1,666 +0,0 @@ -'use strict'; - -const assert = require('node:assert/strict'); -const test = require('node:test'); - -const { - billingMonth, - formatBillingMonth, - hasRequiredFields, - isStandardHostedSku, - MAX_SKU_TABLE_ROWS, - minutesForRow, - parseBudget, - POOL_ALERT_THRESHOLD_PERCENTS, - poolIncidentMarker, - poolIncidentTitle, - poolRecoverySummary, - renderSkuMarkdownTable, - reportedThresholdsForMonth, - resolveRepositoryVisibility, - run, - STANDARD_HOSTED_SKUS, - summarizeUsage, - tierMarker, - upsertIncident, -} = require('./budget-monitor.cjs'); -const { incidentMarker: queueIncidentMarker } = require('./queue-monitor.cjs'); - -const ISSUE_AUTHOR_LOGIN = 'github-actions[bot]'; -const ownIssue = (overrides) => ({ user: { login: ISSUE_AUTHOR_LOGIN, type: 'Bot' }, pull_request: undefined, ...overrides }); - -function usageItem(overrides = {}) { - return { - date: '2026-07-15', - product: 'actions', - sku: 'Actions Linux', - quantity: 100, - unitType: 'minutes', - pricePerUnit: 0.008, - grossAmount: 0.8, - discountAmount: 0.8, - netAmount: 0, - organizationName: 'melodic-software', - repositoryName: 'medley', - ...overrides, - }; -} - -function fakeCore() { - const calls = { setOutput: [], setFailed: [], info: [], warning: [] }; - const summary = { - addHeading() { return summary; }, - addRaw() { return summary; }, - async write() {}, - }; - return { - calls, - summary, - setOutput(name, value) { calls.setOutput.push([name, value]); }, - setFailed(message) { calls.setFailed.push(message); }, - warning(message) { calls.warning.push(message); }, - info(message) { calls.info.push(message); }, - }; -} - -function fakeGitHub({ - usageItems = [], - requestFailure, - existingIssues = [], - repoVisibility = {}, - repoLookupFailure = {}, - commentFailure = false, -} = {}) { - const calls = []; - const listForRepo = Symbol('listForRepo'); - return { - calls, - rest: { - repos: { - async get(parameters) { - calls.push(['repos.get', parameters]); - const key = `${parameters.owner}/${parameters.repo}`; - if (repoLookupFailure[key]) throw new Error('repo lookup failed'); - const privateRepo = repoVisibility[key] ?? true; - return { data: { private: privateRepo } }; - }, - }, - issues: { - listForRepo, - async create(parameters) { calls.push(['create', parameters]); return { data: { number: 101 } }; }, - async update(parameters) { calls.push(['update', parameters]); }, - async createComment(parameters) { - calls.push(['createComment', parameters]); - if (commentFailure) throw new Error('comment failed'); - }, - }, - }, - async request(route, parameters) { - calls.push(['request', route, parameters]); - if (requestFailure) throw new Error('billing usage request failed'); - return { data: { usageItems } }; - }, - async paginate(endpoint, parameters) { - calls.push(['paginate', parameters]); - if (endpoint === listForRepo) return existingIssues; - throw new Error('unexpected endpoint'); - }, - }; -} - -const baseUpsertEnv = { - GITHUB_REPOSITORY: 'melodic-software/ci-runner', - ISSUE_AUTHOR_LOGIN, - BUDGET_MONITOR_ARMED: 'true', -}; - -function budgetFixture(overrides = {}) { - return { - org: 'melodic-software', - month: '2026-07', - includedMinutes: 3000, - consumedMinutes: 1500, - percentUsed: 50, - breachedThresholds: [50], - perSku: [{ sku: 'Actions Linux', minutes: 1500 }], - ...overrides, - }; -} - -test('isStandardHostedSku accepts only the three standard hosted runner SKUs', () => { - for (const sku of STANDARD_HOSTED_SKUS) { - assert.ok(isStandardHostedSku(usageItem({ sku }))); - } - assert.ok(!isStandardHostedSku(usageItem({ sku: 'Actions Linux 16-core' }))); - assert.ok(!isStandardHostedSku(usageItem({ sku: 'Self-hosted' }))); -}); - -test('hasRequiredFields rejects rows missing organization, repository, sku, or quantity', () => { - assert.ok(hasRequiredFields(usageItem())); - assert.ok(!hasRequiredFields(usageItem({ organizationName: '' }))); - assert.ok(!hasRequiredFields(usageItem({ repositoryName: '' }))); - assert.ok(!hasRequiredFields(usageItem({ sku: '' }))); - assert.ok(!hasRequiredFields(usageItem({ quantity: undefined }))); - assert.ok(!hasRequiredFields(usageItem({ quantity: -1 }))); -}); - -test('minutesForRow uses raw quantity for private and unknown visibility, zero for public', () => { - assert.equal(minutesForRow(usageItem({ quantity: 250 }), 'private'), 250); - assert.equal(minutesForRow(usageItem({ quantity: 250 }), 'unknown'), 250); - assert.equal(minutesForRow(usageItem({ quantity: 250 }), 'public'), 0); -}); - -test('resolveRepositoryVisibility includes minutes on lookup failure', async () => { - const github = fakeGitHub({ repoLookupFailure: { 'melodic-software/medley': true } }); - const cache = new Map(); - const visibility = await resolveRepositoryVisibility({ - github, - owner: 'melodic-software', - repo: 'medley', - cache, - }); - assert.equal(visibility, 'unknown'); - assert.equal(cache.get('melodic-software/medley'), 'unknown'); -}); - -test('summarizeUsage counts raw quantity on private repos for standard hosted SKUs only', async () => { - const github = fakeGitHub({ - repoVisibility: { - 'melodic-software/medley': true, - 'melodic-software/public-app': false, - }, - }); - const summary = await summarizeUsage([ - usageItem({ sku: 'Actions Linux', quantity: 1000, repositoryName: 'medley' }), - usageItem({ sku: 'Actions Windows', quantity: 200, repositoryName: 'medley' }), - usageItem({ sku: 'Actions Linux', quantity: 500, repositoryName: 'public-app' }), - usageItem({ sku: 'Actions Linux 16-core', quantity: 999, repositoryName: 'medley' }), - usageItem({ product: 'packages', sku: 'Packages', unitType: 'GigabyteHours', quantity: 50 }), - ], { includedMinutes: 3000, github }); - - assert.equal(summary.consumedMinutes, 1200); - assert.equal(summary.percentUsed, 40); - assert.deepEqual(summary.perSku, [ - { sku: 'Actions Linux', minutes: 1000 }, - { sku: 'Actions Windows', minutes: 200 }, - ]); -}); - -test('summarizeUsage includes quantity when repository visibility lookup fails', async () => { - const github = fakeGitHub({ - repoLookupFailure: { 'melodic-software/medley': true }, - }); - const summary = await summarizeUsage([usageItem({ quantity: 400 })], { includedMinutes: 3000, github }); - assert.equal(summary.consumedMinutes, 400); -}); - -test('summarizeUsage ignores discountAmount and uses quantity even when allowance did not pay', async () => { - const github = fakeGitHub(); - const summary = await summarizeUsage([ - usageItem({ quantity: 300, discountAmount: 0, pricePerUnit: 0.008 }), - ], { includedMinutes: 3000, github }); - assert.equal(summary.consumedMinutes, 300); -}); - -test('summarizeUsage drops rows missing required fields', async () => { - const github = fakeGitHub(); - const summary = await summarizeUsage([ - usageItem({ quantity: 100 }), - usageItem({ quantity: 200, repositoryName: '' }), - ], { includedMinutes: 3000, github }); - assert.equal(summary.consumedMinutes, 100); -}); - -test('summarizeUsage refuses to report a quiet zero when a runner SKU carries an unrecognized unit', async () => { - await assert.rejects( - () => summarizeUsage([usageItem({ sku: 'Actions Linux', unitType: 'compute-credits' })], { includedMinutes: 3000, github: fakeGitHub() }), - /standard hosted Actions runner SKU was reported without a minute unit type/, - ); -}); - -test('summarizeUsage tolerates legitimate non-minute Actions products such as storage', async () => { - const summary = await summarizeUsage([ - usageItem({ sku: 'Actions Storage', unitType: 'GigabyteHours', quantity: 12 }), - ], { includedMinutes: 3000, github: fakeGitHub() }); - assert.equal(summary.consumedMinutes, 0); - assert.deepEqual(summary.breachedThresholds, []); -}); - -test('summarizeUsage tolerates the documented field names under any value casing', async () => { - const summary = await summarizeUsage( - [usageItem({ product: 'Actions', unitType: 'Minutes' })], - { includedMinutes: 3000, github: fakeGitHub() }, - ); - assert.equal(summary.consumedMinutes, 100); -}); - -test('summarizeUsage reports a clean month with no Actions usage at all', async () => { - const summary = await summarizeUsage([], { includedMinutes: 3000, github: fakeGitHub() }); - assert.equal(summary.consumedMinutes, 0); - assert.deepEqual(summary.breachedThresholds, []); -}); - -test('summarizeUsage breaches each configured threshold in order as consumption rises', async () => { - const at = async (percent) => summarizeUsage( - [usageItem({ quantity: 3000 * (percent / 100) })], - { includedMinutes: 3000, github: fakeGitHub() }, - ).then(result => result.breachedThresholds); - - assert.deepEqual(POOL_ALERT_THRESHOLD_PERCENTS, [50, 80]); - assert.deepEqual(await at(49.9), []); - assert.deepEqual(await at(50), [50]); - assert.deepEqual(await at(79.9), [50]); - assert.deepEqual(await at(80), [50, 80]); - assert.deepEqual(await at(140), [50, 80]); -}); - -test('summarizeUsage rejects a payload that is not the documented usageItems array', async () => { - const github = fakeGitHub(); - await assert.rejects(() => summarizeUsage(undefined, { includedMinutes: 3000, github }), /must expose a usageItems array/); - await assert.rejects(() => summarizeUsage({}, { includedMinutes: 3000, github }), /must expose a usageItems array/); - await assert.rejects(() => summarizeUsage([], { includedMinutes: 0, github }), /positive number/); - await assert.rejects(() => summarizeUsage([], { includedMinutes: Number.NaN, github }), /positive number/); -}); - -test('the measured summary carries no currency figure, keeping billing data out of this public repository', async () => { - const summary = await summarizeUsage([ - usageItem({ pricePerUnit: 0.008, grossAmount: 12.34, discountAmount: 12.34 }), - usageItem({ product: 'packages', unitType: 'GigabyteHours', netAmount: 99.99, quantity: 1 }), - ], { includedMinutes: 3000, github: fakeGitHub() }); - - const serialized = JSON.stringify(summary); - for (const amount of ['12.34', '99.99', '0.008']) { - assert.ok(!serialized.includes(amount), `the summary must not carry the currency figure ${amount}`); - } -}); - -test('billingMonth resolves the UTC billing month regardless of local time', () => { - assert.deepEqual(billingMonth(Date.parse('2026-07-31T23:30:00Z')), { year: 2026, month: 7 }); - assert.deepEqual(billingMonth(Date.parse('2026-08-01T00:30:00Z')), { year: 2026, month: 8 }); - assert.equal(formatBillingMonth({ year: 2026, month: 8 }), '2026-08'); -}); - -test('run() queries the documented usage endpoint for the current billing month', async () => { - const github = fakeGitHub({ usageItems: [usageItem()] }); - const core = fakeCore(); - await run({ - github, - core, - env: { - BILLING_ORG: 'melodic-software', - INCLUDED_MINUTES: '3000', - BILLING_TOKEN_PRESENT: 'true', - BUDGET_MONITOR_ARMED: 'true', - }, - now: Date.parse('2026-07-22T10:00:00Z'), - }); - - const [, route, parameters] = github.calls.find(([action]) => action === 'request'); - assert.equal(route, 'GET /organizations/{org}/settings/billing/usage'); - assert.deepEqual(parameters, { org: 'melodic-software', year: 2026, month: 7 }); - assert.deepEqual(core.calls.setFailed, []); - const [name, value] = core.calls.setOutput[0]; - assert.equal(name, 'budget'); - assert.deepEqual(JSON.parse(value).month, '2026-07'); -}); - -test('run() skips with a warning when the monitor is disarmed', async () => { - const github = fakeGitHub({ usageItems: [usageItem()] }); - const core = fakeCore(); - await run({ - github, - core, - env: { - BILLING_ORG: 'melodic-software', - INCLUDED_MINUTES: '3000', - BILLING_TOKEN_PRESENT: 'true', - BUDGET_MONITOR_ARMED: 'false', - }, - }); - - assert.equal(core.calls.warning.length, 1); - assert.match(core.calls.warning[0], /disarmed/); - assert.deepEqual(core.calls.setOutput, []); - assert.deepEqual(core.calls.setFailed, []); - assert.equal(github.calls.filter(([action]) => action === 'request').length, 0); -}); - -test('run() fails loudly when armed but the billing token is not provisioned', async () => { - const github = fakeGitHub({ usageItems: [] }); - const core = fakeCore(); - await run({ - github, - core, - env: { - BILLING_ORG: 'melodic-software', - INCLUDED_MINUTES: '3000', - BILLING_TOKEN_PRESENT: 'false', - BUDGET_MONITOR_ARMED: 'true', - }, - now: Date.parse('2026-07-22T10:00:00Z'), - }); - - assert.equal(core.calls.setFailed.length, 1); - assert.match(core.calls.setFailed[0], /CI_RUNNER_BILLING_OBSERVER_TOKEN/); - assert.deepEqual(core.calls.setOutput, []); - assert.equal(github.calls.filter(([action]) => action === 'request').length, 0); -}); - -test('run() propagates a genuine execution error via setFailed and skips the budget output', async () => { - const github = fakeGitHub({ requestFailure: true }); - const core = fakeCore(); - await run({ - github, - core, - env: { - BILLING_ORG: 'melodic-software', - INCLUDED_MINUTES: '3000', - BILLING_TOKEN_PRESENT: 'true', - BUDGET_MONITOR_ARMED: 'true', - }, - }); - - assert.equal(core.calls.setFailed.length, 1); - assert.deepEqual(core.calls.setOutput, []); -}); - -test('run() rejects incomplete configuration', async () => { - const core = fakeCore(); - await run({ - github: fakeGitHub(), - core, - env: { INCLUDED_MINUTES: '3000', BILLING_TOKEN_PRESENT: 'true', BUDGET_MONITOR_ARMED: 'true' }, - }); - assert.match(core.calls.setFailed[0], /BILLING_ORG is required/); - - const secondCore = fakeCore(); - await run({ - github: fakeGitHub(), - core: secondCore, - env: { - BILLING_ORG: 'melodic-software', - INCLUDED_MINUTES: 'not-a-number', - BILLING_TOKEN_PRESENT: 'true', - BUDGET_MONITOR_ARMED: 'true', - }, - }); - assert.match(secondCore.calls.setFailed[0], /positive number/); -}); - -test('renderSkuMarkdownTable keeps a stable shape, caps rows, and escapes untrusted SKU text', () => { - const table = renderSkuMarkdownTable([{ sku: 'Actions Linux', minutes: 1500 }]); - assert.match(table, /^\| SKU \| Included minutes consumed \|/); - assert.match(table, /\| Actions Linux \| 1500 \|/); - - const injected = renderSkuMarkdownTable([{ sku: `evil ${poolIncidentMarker('owner-b')} | name`, minutes: 1 }]); - assert.ok(!injected.includes(poolIncidentMarker('owner-b')), 'a crafted SKU must not smuggle a functional marker into the body'); - assert.match(injected, /\\\|/); - - const many = Array.from({ length: MAX_SKU_TABLE_ROWS + 4 }, (_, index) => ({ sku: `sku-${index}`, minutes: 1 })); - const capped = renderSkuMarkdownTable(many); - assert.equal(capped.split('\n').filter(line => line.startsWith('| sku-')).length, MAX_SKU_TABLE_ROWS); - assert.match(capped, /_\.\.\.and 4 more SKU\(s\)\._/); -}); - -test('this monitor\'s markers cannot collide with the queue monitor\'s markers for the same owner', () => { - const owner = 'melodic-software'; - const markers = [poolIncidentMarker(owner), queueIncidentMarker(owner)]; - for (const left of markers) { - for (const right of markers) { - if (left === right) continue; - assert.ok(!left.includes(right), `'${left}' must not contain '${right}' as a substring`); - } - } -}); - -test('marker and title shapes stay greppable and prefix-safe across owners', () => { - assert.equal(poolIncidentTitle('melodic-software'), '[Alert] Actions included-minute consumption — melodic-software'); - assert.ok(!poolIncidentMarker('melodic-software-fork').includes(poolIncidentMarker('melodic-software'))); -}); - -test('recovery guidance names free capacity for the pool', () => { - assert.match(poolRecoverySummary, /free for public repositories/); - assert.match(poolRecoverySummary, /self-hosted runners consume no included minutes/); - assert.match(poolRecoverySummary, /new billing month resets the allowance/); -}); - -test('tierMarker scopes escalation state to the billing month', () => { - assert.equal(tierMarker('2026-07', 50), ''); - assert.notEqual(tierMarker('2026-07', 50), tierMarker('2026-08', 50)); -}); - -test('reportedThresholdsForMonth ignores prior-month markers on a still-open incident', () => { - const julyBody = `${tierMarker('2026-07', 50)}${tierMarker('2026-07', 80)}`; - assert.deepEqual(reportedThresholdsForMonth(julyBody, '2026-08', [50, 80]), []); - assert.deepEqual(reportedThresholdsForMonth(julyBody, '2026-07', [50, 80]), [50, 80]); -}); - -test('upsertIncident opens a pool incident carrying month-scoped tier markers', async () => { - const github = fakeGitHub(); - const core = fakeCore(); - await upsertIncident({ - github, - core, - env: { ...baseUpsertEnv, BUDGET_JSON: JSON.stringify(budgetFixture()) }, - now: Date.parse('2026-07-22T10:00:00Z'), - }); - - const created = github.calls.find(([action]) => action === 'create'); - const [, parameters] = created; - assert.equal(parameters.title, poolIncidentTitle('melodic-software')); - assert.deepEqual(parameters.labels, ['automated']); - assert.ok(parameters.body.includes(tierMarker('2026-07', 50))); - assert.ok(!parameters.body.includes(tierMarker('2026-07', 80))); - assert.ok(parameters.body.endsWith(poolIncidentMarker('melodic-software'))); - assert.match(parameters.body, /1500 of 3000 included minute\(s\) consumed \(50%\)/); -}); - -test('upsertIncident silently updates a pool incident that is still at the same tier', async () => { - const marker = poolIncidentMarker('melodic-software'); - const github = fakeGitHub({ - existingIssues: [ownIssue({ number: 55, body: `${tierMarker('2026-07', 50)}\nstale ${marker}` })], - }); - const core = fakeCore(); - await upsertIncident({ - github, - core, - env: { ...baseUpsertEnv, BUDGET_JSON: JSON.stringify(budgetFixture({ consumedMinutes: 1800, percentUsed: 60 })) }, - now: Date.parse('2026-07-23T10:00:00Z'), - }); - - assert.equal(github.calls.filter(([action]) => action === 'update').length, 1); - assert.equal(github.calls.filter(([action]) => action === 'createComment').length, 0); -}); - -test('upsertIncident comments once when the pool incident escalates to a higher tier', async () => { - const marker = poolIncidentMarker('melodic-software'); - const github = fakeGitHub({ - existingIssues: [ownIssue({ number: 55, body: `${tierMarker('2026-07', 50)}\nstale ${marker}` })], - }); - const core = fakeCore(); - await upsertIncident({ - github, - core, - env: { - ...baseUpsertEnv, - BUDGET_JSON: JSON.stringify(budgetFixture({ consumedMinutes: 2460, percentUsed: 82, breachedThresholds: [50, 80] })), - }, - now: Date.parse('2026-07-24T10:00:00Z'), - }); - - const updates = github.calls.filter(([action]) => action === 'update'); - assert.equal(updates.length, 2, 'escalation refreshes the body, comments, then persists month-scoped markers'); - assert.ok(!updates[0][1].body.includes(tierMarker('2026-07', 80)), 'new tier marker must not precede the comment'); - assert.ok(updates[1][1].body.includes(tierMarker('2026-07', 80)), 'new tier marker is persisted only after the comment'); - const comments = github.calls.filter(([action]) => action === 'createComment'); - assert.equal(comments.length, 1); - assert.match(comments[0][1].body, /Escalated to 80%/); -}); - -test('upsertIncident retries escalation when the comment fails before markers are persisted', async () => { - const marker = poolIncidentMarker('melodic-software'); - const github = fakeGitHub({ - existingIssues: [ownIssue({ number: 55, body: `${tierMarker('2026-07', 50)}\nstale ${marker}` })], - commentFailure: true, - }); - const core = fakeCore(); - await assert.rejects( - upsertIncident({ - github, - core, - env: { - ...baseUpsertEnv, - BUDGET_JSON: JSON.stringify(budgetFixture({ consumedMinutes: 2460, percentUsed: 82, breachedThresholds: [50, 80] })), - }, - now: Date.parse('2026-07-24T10:00:00Z'), - }), - /comment failed/, - ); - - const updates = github.calls.filter(([action]) => action === 'update'); - assert.equal(updates.length, 1); - assert.ok(!updates[0][1].body.includes(tierMarker('2026-07', 80)), 'failed escalation must not persist the new tier marker'); -}); - -test('upsertIncident emits a new-month escalation when prior-month markers remain on an open incident', async () => { - const marker = poolIncidentMarker('melodic-software'); - const github = fakeGitHub({ - existingIssues: [ownIssue({ number: 55, body: `${tierMarker('2026-07', 50)}${tierMarker('2026-07', 80)}\nstale ${marker}` })], - }); - const core = fakeCore(); - await upsertIncident({ - github, - core, - env: { - ...baseUpsertEnv, - BUDGET_JSON: JSON.stringify(budgetFixture({ - month: '2026-08', - consumedMinutes: 2500, - percentUsed: 83.3, - breachedThresholds: [50, 80], - })), - }, - now: Date.parse('2026-08-02T10:00:00Z'), - }); - - const comments = github.calls.filter(([action]) => action === 'createComment'); - assert.equal(comments.length, 1, 'August thresholds must not be treated as already reported by July markers'); - assert.match(comments[0][1].body, /Escalated to 80%/); - const finalUpdate = github.calls.filter(([action]) => action === 'update').at(-1); - assert.ok(finalUpdate[1].body.includes(tierMarker('2026-08', 80))); - assert.ok(!finalUpdate[1].body.includes(tierMarker('2026-07', 80)), 'prior-month markers must not satisfy the new month'); -}); - -test('upsertIncident closes the pool incident when a new billing month resets consumption', async () => { - const marker = poolIncidentMarker('melodic-software'); - const github = fakeGitHub({ - existingIssues: [ownIssue({ number: 55, body: `${tierMarker('2026-07', 50)}${tierMarker('2026-07', 80)}\nstale ${marker}` })], - }); - const core = fakeCore(); - await upsertIncident({ - github, - core, - env: { - ...baseUpsertEnv, - BUDGET_JSON: JSON.stringify(budgetFixture({ month: '2026-08', consumedMinutes: 12, percentUsed: 0.4, breachedThresholds: [] })), - }, - now: Date.parse('2026-08-01T10:00:00Z'), - }); - - const commented = github.calls.find(([action]) => action === 'createComment'); - const updated = github.calls.find(([action]) => action === 'update'); - assert.match(commented[1].body, /under every alert threshold/); - assert.equal(updated[1].issue_number, 55); - assert.equal(updated[1].state, 'closed'); - assert.equal(updated[1].state_reason, 'completed'); -}); - -test('upsertIncident skips when disarmed', async () => { - const github = fakeGitHub(); - const core = fakeCore(); - await upsertIncident({ - github, - core, - env: { ...baseUpsertEnv, BUDGET_MONITOR_ARMED: 'false', BUDGET_JSON: JSON.stringify(budgetFixture()) }, - }); - - assert.equal(core.calls.warning.length, 1); - assert.deepEqual(github.calls.filter(([action]) => action !== 'paginate'), []); -}); - -test('upsertIncident is a no-op when nothing is breached and no incident is open', async () => { - const github = fakeGitHub(); - const core = fakeCore(); - await upsertIncident({ - github, - core, - env: { ...baseUpsertEnv, BUDGET_JSON: JSON.stringify(budgetFixture({ breachedThresholds: [] })) }, - now: Date.parse('2026-08-01T10:00:00Z'), - }); - - assert.deepEqual(github.calls.filter(([action]) => action !== 'paginate'), []); -}); - -test('upsertIncident does not adopt a decoy issue carrying the marker under another author', async () => { - const github = fakeGitHub({ - existingIssues: [{ number: 66, body: `decoy ${poolIncidentMarker('melodic-software')}`, pull_request: undefined, user: { login: 'kyle-sexton', type: 'User' } }], - }); - const core = fakeCore(); - await upsertIncident({ - github, - core, - env: { ...baseUpsertEnv, BUDGET_JSON: JSON.stringify(budgetFixture()) }, - now: Date.parse('2026-07-22T10:00:00Z'), - }); - - assert.ok(github.calls.find(([action]) => action === 'create')); - assert.equal(github.calls.filter(([action]) => action === 'update').length, 0); -}); - -test('upsertIncident rejects a missing home repository or issue-author login', async () => { - const github = fakeGitHub(); - const core = fakeCore(); - const budgetJson = JSON.stringify(budgetFixture()); - await assert.rejects( - upsertIncident({ github, core, env: { ISSUE_AUTHOR_LOGIN, BUDGET_JSON: budgetJson, BUDGET_MONITOR_ARMED: 'true' } }), - /GITHUB_REPOSITORY must be set/, - ); - await assert.rejects( - upsertIncident({ github, core, env: { GITHUB_REPOSITORY: 'melodic-software/ci-runner', BUDGET_JSON: budgetJson, BUDGET_MONITOR_ARMED: 'true' } }), - /ISSUE_AUTHOR_LOGIN is required/, - ); -}); - -test('parseBudget rejects a missing, malformed, or structurally wrong payload instead of assuming recovery', async () => { - assert.throws(() => parseBudget(undefined), /BUDGET_JSON is required/); - assert.throws(() => parseBudget(''), /BUDGET_JSON is required/); - assert.throws(() => parseBudget('{not json'), /not valid JSON/); - assert.throws(() => parseBudget('[]'), /must decode to an object/); - assert.throws(() => parseBudget('{}'), /breachedThresholds/); - - const github = fakeGitHub({ - existingIssues: [ownIssue({ number: 55, body: `open ${poolIncidentMarker('melodic-software')}` })], - }); - await assert.rejects(upsertIncident({ github, core: fakeCore(), env: baseUpsertEnv }), /BUDGET_JSON is required/); - assert.equal(github.calls.filter(([action]) => action === 'update' || action === 'createComment').length, 0); -}); - -test('pool alert bodies publish no currency amounts', async () => { - const github = fakeGitHub(); - const core = fakeCore(); - await upsertIncident({ - github, - core, - env: { - ...baseUpsertEnv, - BUDGET_JSON: JSON.stringify(budgetFixture({ breachedThresholds: [50, 80] })), - }, - now: Date.parse('2026-07-22T10:00:00Z'), - }); - - const created = github.calls.find(([action]) => action === 'create'); - const body = created[1].body; - for (const amount of body.match(/\$\d+(?:\.\d+)?/g) ?? []) { - assert.equal(amount, '$0', `no currency amount may reach a public issue body (found ${amount})`); - } -}); diff --git a/.github/scripts/incident-issue.cjs b/.github/scripts/incident-issue.cjs deleted file mode 100644 index dd4baf2..0000000 --- a/.github/scripts/incident-issue.cjs +++ /dev/null @@ -1,100 +0,0 @@ -'use strict'; - -// The marker-deduped incident-issue alert channel shared by this repository's -// scheduled monitors (the fleet's established alert-per-incident pattern; see -// link-check.yml and queue-monitor-liveness.yml in ci-workflows). Each monitor -// owns its own marker namespace, title, and body; everything here is the part -// that must behave identically for all of them, because a divergence in the -// adoption or escaping rules is a security regression rather than a style -// difference. - -// GitHub's issue/comment write endpoints reject a body over 65536 -// characters (observed API error "Body is too long (maximum is 65536 -// characters)"; not a value GitHub's docs state as a formal field limit, -// but consistently reproduced — see -// https://github.com/orgs/community/discussions/41331). Callers cap their -// own rendered detail well below this; this is the defense-in-depth -// backstop, so that a body that somehow still exceeded the limit truncates -// instead of throwing on issues.create/update — which would make the -// monitor's own alert delivery the thing that breaks it. Stays well under -// the observed limit for safety margin. -const MAX_BODY_LENGTH = 60000; - -// Neutralizes '<' and '>' (not just '|') because cell content is untrusted: -// it originates outside this workflow's own source — monitored-repo job and -// workflow names, upstream API-reported identifiers. Left unescaped, a -// crafted value could inject a literal '' sequence into a -// bot-authored incident body — including another incident's marker, which -// findOpenIncident matches as a raw substring — causing a cross-incident -// collision. HTML-entity encoding renders as the literal characters (GitHub's -// Markdown renders '<'/'>' back to '<'/'>' visually) while never -// forming a real '' sequence in the raw body text this -// repository's own code searches. -function escapeMarkdownTableCell(value) { - return String(value) - .replace(//g, '>') - .replace(/\|/g, '\\|'); -} - -// Truncates the assembled body (everything except the trailing marker) if it -// would still exceed maxLength after the caller's own row cap — the marker must -// never be truncated away, since findOpenIncident's future upserts and -// closes depend on it surviving intact. -function boundBodyLength(bodyWithoutMarker, marker, maxLength = MAX_BODY_LENGTH) { - const separator = '\n\n'; - const budget = maxLength - marker.length - separator.length; - if (bodyWithoutMarker.length <= budget) { - return `${bodyWithoutMarker}${separator}${marker}`; - } - const notice = '\n\n_...truncated to stay under GitHub\'s issue body limit._'; - const truncated = bodyWithoutMarker.slice(0, Math.max(0, budget - notice.length)) + notice; - return `${truncated}${separator}${marker}`; -} - -function isOwnIncidentAuthor(issue, issueAuthorLogin) { - return issue.user?.login === issueAuthorLogin && issue.user?.type === 'Bot'; -} - -// The marker and title strings are public (embedded verbatim in each -// monitor's source, in a public repository), so matching on them alone -// would let anyone with issue-creation permission here craft a decoy this -// automation adopts, silently updates, or closes as recovered — suppressing -// a real alert. Restricting candidates to issues opened by the monitor's -// own token identity closes that: an attacker's issue is never a candidate, -// no matter what text it carries. Mirrors the hardening in ci-workflows#213 -// (standards-sync-stuck-automerge-alert.yml). Ambiguity (more than one -// candidate carrying the marker) fails closed rather than guessing. -// -// Matching stays marker-only (not marker + title), matching the fleet -// precedent's deliberate "a marker survives a retitle" property. Untrusted -// text reaching a rendered body could in principle try to inject a foreign -// marker; escapeMarkdownTableCell neutralizes that at the source (see its own -// comment) instead of layering a title guard here that would trade -// retitle-survival for redundant protection. -async function findOpenIncident({ github, homeOwner, homeRepo, marker, issueAuthorLogin }) { - // Every incident issue these monitors create carries the 'automated' label - // (see each caller's create call); filtering server-side keeps this scan - // from paginating every open issue in the repo. - const openIssues = await github.paginate(github.rest.issues.listForRepo, { - owner: homeOwner, - repo: homeRepo, - state: 'open', - labels: 'automated', - per_page: 100, - }); - const candidates = openIssues.filter(issue => !issue.pull_request && isOwnIncidentAuthor(issue, issueAuthorLogin)); - const matches = candidates.filter(issue => (issue.body ?? '').includes(marker)); - if (matches.length > 1) { - throw new Error(`Found ${matches.length} open incident issues carrying marker '${marker}'; reconcile manually.`); - } - return matches[0] || null; -} - -module.exports = { - boundBodyLength, - escapeMarkdownTableCell, - findOpenIncident, - isOwnIncidentAuthor, - MAX_BODY_LENGTH, -}; diff --git a/.github/scripts/queue-monitor.cjs b/.github/scripts/queue-monitor.cjs deleted file mode 100644 index a66745c..0000000 --- a/.github/scripts/queue-monitor.cjs +++ /dev/null @@ -1,272 +0,0 @@ -'use strict'; - -const { - boundBodyLength, - escapeMarkdownTableCell, - findOpenIncident, -} = require('./incident-issue.cjs'); - -const nonterminalRunStatuses = Object.freeze([ - 'queued', - 'in_progress', - 'requested', - 'waiting', - 'pending', -]); - -const routingRecoverySummary = ` -Confirm the managed runner host is running a release that includes the latest -capacity and reconciliation fixes (at least \`v0.1.21\` / current \`main\` tip) -before changing routing. An unrebuilt host on an older controller will keep -queuing work against dead capacity even when \`main\` already carries the fix. - -Then follow the [audited CI routing-control procedure](https://github.com/melodic-software/github-iac/blob/main/README.md#local-ci-routing-governance) to make the affected repository's effective \`CI_RUNNER_POLICY\` value \`hosted-only\` and verify the readback. Cancel the affected run, choose **Re-run all jobs** to guarantee that the selector executes again, and confirm that it selects hosted capacity. Do not use a failed-job or single-job rerun for this recovery because partial-rerun dependency behavior does not guarantee a fresh selector decision. A \`workflow_dispatch\` creates a separate run with different event and ref context; it does not recover the original pull-request check. -`; - -// A worst-case queue-wide outage could stall far more than a handful of -// managed jobs; capping the rendered table keeps the incident body bounded -// regardless of how many jobs are stuck. -const MAX_STUCK_TABLE_ROWS = 50; - -function splitList(value) { - return (value || '') - .split(/[\s,]+/) - .filter(Boolean); -} - -async function inspectQueuedJobs({ - github, - owner, - repositories, - managedLabels, - thresholdMinutes, - now = Date.now(), -}) { - if (!owner || repositories.length === 0 || managedLabels.size === 0) { - throw new Error('Queue monitor configuration is incomplete: owner, repositories, and managed labels are required.'); - } - if (!Number.isFinite(thresholdMinutes) || thresholdMinutes <= 0) { - throw new Error('Queue monitor threshold must be a positive number of minutes.'); - } - - const cutoff = now - thresholdMinutes * 60 * 1000; - const stuck = []; - for (const repo of repositories) { - const runsById = new Map(); - for (const status of nonterminalRunStatuses) { - const runs = await github.paginate(github.rest.actions.listWorkflowRunsForRepo, { - owner, - repo, - status, - per_page: 100, - }); - for (const run of runs) runsById.set(run.id, run); - } - - for (const run of runsById.values()) { - const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { - owner, - repo, - run_id: run.id, - filter: 'latest', - per_page: 100, - }); - for (const job of jobs) { - const labels = job.labels || []; - const isManaged = labels.some(label => managedLabels.has(label)); - const queuedAt = Date.parse(job.created_at); - if (job.status === 'queued' && isManaged && Number.isFinite(queuedAt) && queuedAt <= cutoff) { - stuck.push({ - repository: `${owner}/${repo}`, - workflow: run.name || run.path, - job: job.name, - queuedMinutes: Math.floor((now - queuedAt) / 60000), - url: job.html_url || run.html_url, - labels: labels.join(', '), - }); - } - } - } - } - return stuck; -} - -async function run({ github, core, env = process.env, now = Date.now() }) { - const owner = env.MONITOR_OWNER; - const repositories = splitList(env.MONITORED_REPOSITORIES); - const managedLabels = new Set(splitList(env.MANAGED_LABELS)); - const thresholdMinutes = Number(env.QUEUE_THRESHOLD_MINUTES); - - let stuck; - try { - stuck = await inspectQueuedJobs({ - github, - owner, - repositories, - managedLabels, - thresholdMinutes, - now, - }); - } catch (error) { - core.setFailed(error instanceof Error ? error.message : String(error)); - return; - } - - // Handed to the incident-upsert step via job output; a successful execution - // stays green from here regardless of what it found (see upsertIncident). - core.setOutput('stuck', JSON.stringify(stuck)); - - if (stuck.length === 0) { - await core.summary.addHeading('Managed runner queue').addRaw('No managed job has been queued for more than five minutes.').write(); - return; - } - - const rows = stuck.map(item => [ - item.repository, - item.workflow, - item.job, - String(item.queuedMinutes), - item.labels, - `[open job](${item.url})`, - ]); - await core.summary - .addHeading('Managed runner queue alert') - .addTable([ - [{ data: 'Repository', header: true }, { data: 'Workflow', header: true }, { data: 'Job', header: true }, { data: 'Minutes', header: true }, { data: 'Labels', header: true }, { data: 'Link', header: true }], - ...rows, - ]) - .addRaw(routingRecoverySummary) - .write(); -} - -function incidentTitle(targetOwner) { - return `[Alert] Managed runner queue capacity — ${targetOwner}`; -} - -function incidentMarker(targetOwner) { - return ``; -} - -function renderStuckMarkdownTable(stuck, { maxRows = MAX_STUCK_TABLE_ROWS, runUrl } = {}) { - const header = '| Repository | Workflow | Job | Minutes | Labels | Link |\n| --- | --- | --- | --- | --- | --- |'; - const shown = stuck.slice(0, maxRows); - const rows = shown.map(item => `| ${escapeMarkdownTableCell(item.repository)} | ${escapeMarkdownTableCell(item.workflow)} | ${escapeMarkdownTableCell(item.job)} | ${item.queuedMinutes} | ${escapeMarkdownTableCell(item.labels)} | [open job](${item.url}) |`); - const lines = [header, ...rows]; - const remaining = stuck.length - shown.length; - if (remaining > 0) { - const runLink = runUrl ? ` — see the [workflow run](${runUrl}) for the full list` : ''; - lines.push('', `_...and ${remaining} more managed job(s)${runLink}._`); - } - return lines.join('\n'); -} - -// Marker-deduped issue-per-incident alert channel (the fleet's established -// pattern; see link-check.yml and queue-monitor-liveness.yml in ci-workflows). -// Runs on the job's own GITHUB_TOKEN against the monitor's home repository — -// distinct from the read-only, target-scoped observer token the detection -// step uses, which cannot write issues here. Any thrown error here fails the -// run: an incident-issue write failure is the monitor breaking, not a queue -// alert. -async function upsertIncident({ github, core, env = process.env, now = Date.now() }) { - const targetOwner = env.TARGET_OWNER; - const [homeOwner, homeRepo] = (env.GITHUB_REPOSITORY || '').split('/'); - const issueAuthorLogin = env.ISSUE_AUTHOR_LOGIN; - if (!homeOwner || !homeRepo) { - throw new Error('GITHUB_REPOSITORY must be set to the owner/repo of the monitor workflow.'); - } - if (!targetOwner) { - throw new Error('TARGET_OWNER is required to key the incident issue.'); - } - if (!issueAuthorLogin) { - throw new Error('ISSUE_AUTHOR_LOGIN is required to restrict incident-issue adoption to this workflow\'s own identity.'); - } - // A missing or empty STUCK_JSON is never "zero stuck jobs" — that case is - // always an explicit "[]" from run()'s core.setOutput. Falling back to '[]' - // here would silently treat a missing detection-step output (a wiring bug, - // a skipped step) as a healthy recovery and close a real open incident. - if (env.STUCK_JSON === undefined || env.STUCK_JSON === '') { - throw new Error('STUCK_JSON is required: the detection step must set it via core.setOutput, even for zero stuck jobs.'); - } - let stuck; - try { - stuck = JSON.parse(env.STUCK_JSON); - } catch (error) { - throw new Error(`STUCK_JSON is not valid JSON: ${error instanceof Error ? error.message : String(error)}`); - } - if (!Array.isArray(stuck)) { - throw new Error('STUCK_JSON must decode to an array.'); - } - - const title = incidentTitle(targetOwner); - const marker = incidentMarker(targetOwner); - const existing = await findOpenIncident({ github, homeOwner, homeRepo, marker, issueAuthorLogin }); - const nowIso = new Date(now).toISOString(); - - if (stuck.length === 0) { - if (!existing) { - core.info(`No open incident for ${targetOwner}; queue is healthy.`); - return; - } - await github.rest.issues.createComment({ - owner: homeOwner, - repo: homeRepo, - issue_number: existing.number, - body: `Recovered: no managed job for \`${targetOwner}\` has been queued past the threshold as of ${nowIso}. Capacity window: ${existing.created_at} – ${nowIso}.`, - }); - await github.rest.issues.update({ - owner: homeOwner, - repo: homeRepo, - issue_number: existing.number, - state: 'closed', - state_reason: 'completed', - }); - core.info(`Closed incident #${existing.number} for ${targetOwner}.`); - return; - } - - // GITHUB_SERVER_URL/GITHUB_RUN_ID are GitHub Actions' own default env - // vars, always set in a real run; undefined only in tests that don't - // provide them, in which case renderStuckMarkdownTable's remainder note - // simply omits the run link rather than producing a broken one. - const runUrl = env.GITHUB_SERVER_URL && env.GITHUB_RUN_ID - ? `${env.GITHUB_SERVER_URL}/${homeOwner}/${homeRepo}/actions/runs/${env.GITHUB_RUN_ID}` - : undefined; - - const windowStart = existing ? existing.created_at : nowIso; - const bodyWithoutMarker = [ - `Managed runner queue capacity alert for \`${targetOwner}\`.`, - '', - `Capacity window: constrained since ${windowStart} (last confirmed ${nowIso}). Affected queue depth: ${stuck.length} managed job(s).`, - '', - renderStuckMarkdownTable(stuck, { runUrl }), - '', - routingRecoverySummary.trim(), - ].join('\n'); - const body = boundBodyLength(bodyWithoutMarker, marker); - - if (existing) { - // A silent body update, not a comment: this step runs every ~15 minutes - // while an incident stays open, and GitHub notifies watchers on every - // comment but not on a body edit. Commenting here would re-create the - // notification noise this alert channel replaces. The body's "last - // confirmed" timestamp already carries freshness. - await github.rest.issues.update({ owner: homeOwner, repo: homeRepo, issue_number: existing.number, body }); - core.info(`Updated incident #${existing.number} for ${targetOwner}.`); - } else { - const created = await github.rest.issues.create({ owner: homeOwner, repo: homeRepo, title, body, labels: ['automated'] }); - core.info(`Opened incident #${created.data.number} for ${targetOwner}.`); - } -} - -module.exports = { - incidentMarker, - incidentTitle, - inspectQueuedJobs, - MAX_STUCK_TABLE_ROWS, - nonterminalRunStatuses, - renderStuckMarkdownTable, - routingRecoverySummary, - run, - splitList, - upsertIncident, -}; diff --git a/.github/scripts/queue-monitor.test.cjs b/.github/scripts/queue-monitor.test.cjs deleted file mode 100644 index d0df36e..0000000 --- a/.github/scripts/queue-monitor.test.cjs +++ /dev/null @@ -1,696 +0,0 @@ -'use strict'; - -const assert = require('node:assert/strict'); -const test = require('node:test'); - -const { - boundBodyLength, - findOpenIncident, - MAX_BODY_LENGTH, -} = require('./incident-issue.cjs'); -const { - incidentMarker, - incidentTitle, - inspectQueuedJobs, - MAX_STUCK_TABLE_ROWS, - nonterminalRunStatuses, - renderStuckMarkdownTable, - routingRecoverySummary, - run, - splitList, - upsertIncident, -} = require('./queue-monitor.cjs'); - -const ISSUE_AUTHOR_LOGIN = 'github-actions[bot]'; -const ownIssue = (overrides) => ({ user: { login: ISSUE_AUTHOR_LOGIN, type: 'Bot' }, pull_request: undefined, ...overrides }); - -function fakeGitHub({ runs = {}, jobs = {}, failure } = {}) { - const calls = []; - const listWorkflowRunsForRepo = Symbol('runs'); - const listJobsForWorkflowRun = Symbol('jobs'); - return { - calls, - rest: { actions: { listWorkflowRunsForRepo, listJobsForWorkflowRun } }, - async paginate(endpoint, parameters) { - calls.push({ endpoint, parameters }); - if (failure && failure(endpoint, parameters)) throw new Error('API page failed'); - if (endpoint === listWorkflowRunsForRepo) return runs[parameters.status] || []; - if (endpoint === listJobsForWorkflowRun) return jobs[parameters.run_id] || []; - throw new Error('unexpected endpoint'); - }, - }; -} - -test('splitList accepts comma and newline separated configuration', () => { - assert.deepEqual(splitList('medley, standards\nci-runner'), ['medley', 'standards', 'ci-runner']); -}); - -test('recovery requires a verified hosted-only cutoff and fresh selector evaluation', () => { - assert.match(routingRecoverySummary, /audited CI routing-control procedure/); - assert.match(routingRecoverySummary, /effective `CI_RUNNER_POLICY` value `hosted-only`/); - assert.match(routingRecoverySummary, /Re-run all jobs/); - assert.match(routingRecoverySummary, /guarantee that the selector executes again/); - assert.match(routingRecoverySummary, /partial-rerun dependency behavior/); - assert.match(routingRecoverySummary, /does not recover the original pull-request check/); - assert.match(routingRecoverySummary, /at least `v0\.1\.21`/); - assert.match(routingRecoverySummary, /Confirm the managed runner host is running a release/); - assert.doesNotMatch(routingRecoverySummary, /retry(?:ing)? (?:the )?workload/i); -}); - -test('queries every GitHub nonterminal run status and deduplicates runs', async () => { - const run = { id: 42, name: 'CI', html_url: 'https://example.test/run/42' }; - const github = fakeGitHub({ - runs: Object.fromEntries(nonterminalRunStatuses.map(status => [status, [run]])), - jobs: { 42: [] }, - }); - - const stuck = await inspectQueuedJobs({ - github, - owner: 'melodic-software', - repositories: ['medley'], - managedLabels: new Set(['melodic-ubuntu-24.04-x64']), - thresholdMinutes: 5, - now: Date.parse('2026-07-10T12:00:00Z'), - }); - - assert.deepEqual(stuck, []); - assert.deepEqual( - github.calls.filter(call => call.endpoint === github.rest.actions.listWorkflowRunsForRepo).map(call => call.parameters.status), - nonterminalRunStatuses, - ); - assert.equal(github.calls.filter(call => call.endpoint === github.rest.actions.listJobsForWorkflowRun).length, 1); -}); - -test('alerts only runner-eligible queued jobs with an exact managed label', async () => { - const now = Date.parse('2026-07-10T12:00:00Z'); - const old = '2026-07-10T11:50:00Z'; - const github = fakeGitHub({ - runs: { in_progress: [{ id: 7, name: 'CI', html_url: 'https://example.test/run/7' }] }, - jobs: { - 7: [ - { name: 'eligible', status: 'queued', created_at: old, labels: ['melodic-ubuntu-24.04-x64'], html_url: 'https://example.test/job/1' }, - { name: 'needs-chain', status: 'waiting', created_at: old, labels: ['melodic-ubuntu-24.04-x64'] }, - { name: 'concurrency', status: 'pending', created_at: old, labels: ['melodic-ubuntu-24.04-x64'] }, - { name: 'not-requested', status: 'requested', created_at: old, labels: ['melodic-ubuntu-24.04-x64'] }, - { name: 'hosted', status: 'queued', created_at: old, labels: ['ubuntu-24.04'] }, - { name: 'similar-label', status: 'queued', created_at: old, labels: ['melodic-ubuntu-24.04-x64-extra'] }, - ], - }, - }); - - const stuck = await inspectQueuedJobs({ - github, - owner: 'melodic-software', - repositories: ['medley'], - managedLabels: new Set(['melodic-ubuntu-24.04-x64']), - thresholdMinutes: 5, - now, - }); - - assert.equal(stuck.length, 1); - assert.equal(stuck[0].job, 'eligible'); - assert.equal(stuck[0].queuedMinutes, 10); -}); - -test('does not alert a newly queued managed job', async () => { - const now = Date.parse('2026-07-10T12:00:00Z'); - const github = fakeGitHub({ - runs: { queued: [{ id: 8, name: 'CI' }] }, - jobs: { 8: [{ name: 'new', status: 'queued', created_at: '2026-07-10T11:58:00Z', labels: ['managed'] }] }, - }); - const stuck = await inspectQueuedJobs({ - github, - owner: 'owner', - repositories: ['repo'], - managedLabels: new Set(['managed']), - thresholdMinutes: 5, - now, - }); - assert.deepEqual(stuck, []); -}); - -test('propagates a pagination failure so the monitor becomes visibly red', async () => { - const github = fakeGitHub({ failure: (_endpoint, parameters) => parameters.status === 'waiting' }); - await assert.rejects( - inspectQueuedJobs({ - github, - owner: 'owner', - repositories: ['repo'], - managedLabels: new Set(['managed']), - thresholdMinutes: 5, - }), - /API page failed/, - ); -}); - -function fakeCore() { - const calls = { setOutput: [], setFailed: [], info: [] }; - const summaryCalls = []; - const summary = { - addHeading(text) { summaryCalls.push(['addHeading', text]); return summary; }, - addRaw(text) { summaryCalls.push(['addRaw', text]); return summary; }, - addTable(rows) { summaryCalls.push(['addTable', rows]); return summary; }, - async write() { summaryCalls.push(['write']); }, - }; - return { - calls, - summaryCalls, - summary, - setOutput(name, value) { calls.setOutput.push([name, value]); }, - setFailed(message) { calls.setFailed.push(message); }, - info(message) { calls.info.push(message); }, - }; -} - -function fakeGithubIssues({ existingIssues = [] } = {}) { - const calls = []; - const listForRepo = Symbol('listForRepo'); - return { - calls, - rest: { - issues: { - listForRepo, - async create(parameters) { calls.push(['create', parameters]); return { data: { number: 101 } }; }, - async update(parameters) { calls.push(['update', parameters]); }, - async createComment(parameters) { calls.push(['createComment', parameters]); }, - }, - }, - async paginate(endpoint, parameters) { - calls.push(['paginate', parameters]); - if (endpoint === listForRepo) return existingIssues; - throw new Error('unexpected endpoint'); - }, - }; -} - -test('run() stays green and hands the stuck list to the incident step on detection', async () => { - const github = fakeGitHub({ - runs: { queued: [{ id: 9, name: 'CI', html_url: 'https://example.test/run/9' }] }, - jobs: { - 9: [{ name: 'build', status: 'queued', created_at: '2026-07-22T09:48:00Z', labels: ['melodic-ubuntu-24.04-x64'], html_url: 'https://example.test/job/9' }], - }, - }); - const core = fakeCore(); - await run({ - github, - core, - env: { - MONITOR_OWNER: 'melodic-software', - MONITORED_REPOSITORIES: 'medley', - MANAGED_LABELS: 'melodic-ubuntu-24.04-x64', - QUEUE_THRESHOLD_MINUTES: '5', - }, - now: Date.parse('2026-07-22T10:00:00Z'), - }); - - assert.deepEqual(core.calls.setFailed, []); - assert.equal(core.calls.setOutput.length, 1); - const [name, value] = core.calls.setOutput[0]; - assert.equal(name, 'stuck'); - assert.equal(JSON.parse(value).length, 1); -}); - -test('run() propagates a genuine execution error via setFailed and skips the stuck output', async () => { - const github = fakeGitHub({ failure: () => true }); - const core = fakeCore(); - await run({ - github, - core, - env: { - MONITOR_OWNER: 'melodic-software', - MONITORED_REPOSITORIES: 'medley', - MANAGED_LABELS: 'melodic-ubuntu-24.04-x64', - QUEUE_THRESHOLD_MINUTES: '5', - }, - }); - - assert.equal(core.calls.setFailed.length, 1); - assert.deepEqual(core.calls.setOutput, []); -}); - -test('incidentTitle and renderStuckMarkdownTable keep a stable, greppable shape', () => { - assert.equal(incidentTitle('melodic-software'), '[Alert] Managed runner queue capacity — melodic-software'); - const table = renderStuckMarkdownTable([ - { repository: 'melodic-software/medley', workflow: 'CI', job: 'build', queuedMinutes: 12, labels: 'melodic-ubuntu-24.04-x64', url: 'https://example.test/job/1' }, - ]); - assert.match(table, /^\| Repository \| Workflow \| Job \| Minutes \| Labels \| Link \|/); - assert.match(table, /\| melodic-software\/medley \| CI \| build \| 12 \| melodic-ubuntu-24.04-x64 \| \[open job\]\(https:\/\/example\.test\/job\/1\) \|/); -}); - -test('renderStuckMarkdownTable escapes pipes so a job or workflow name cannot corrupt the table', () => { - const table = renderStuckMarkdownTable([ - { repository: 'melodic-software/medley', workflow: 'CI | matrix', job: 'build | test', queuedMinutes: 6, labels: 'a|b', url: 'https://example.test/job/2' }, - ]); - assert.match(table, /\| CI \\\| matrix \| build \\\| test \| 6 \| a\\\|b \|/); -}); - -test('renderStuckMarkdownTable neutralizes HTML comment sequences so a crafted job name cannot inject a literal into the body', () => { - const foreignMarker = incidentMarker('owner-b'); - const table = renderStuckMarkdownTable([ - { repository: 'melodic-software/medley', workflow: 'CI', job: `evil ${foreignMarker} name`, queuedMinutes: 5, labels: 'melodic-ubuntu-24.04-x64', url: 'https://example.test/job/1' }, - ]); - assert.ok(!table.includes(foreignMarker), 'the raw marker substring must not survive rendering'); - assert.ok(table.includes('<!--'), 'the HTML comment opener must be neutralized to an entity'); - assert.ok(table.includes('-->'), 'the HTML comment closer must be neutralized to an entity'); -}); - -function fakeStuckList(count) { - return Array.from({ length: count }, (_, index) => ({ - repository: 'melodic-software/medley', - workflow: 'CI', - job: `build-${index}`, - queuedMinutes: 10, - labels: 'melodic-ubuntu-24.04-x64', - url: `https://example.test/job/${index}`, - })); -} - -test('renderStuckMarkdownTable caps rows at MAX_STUCK_TABLE_ROWS and reports the correct remainder with a run link', () => { - const total = MAX_STUCK_TABLE_ROWS + 7; - const table = renderStuckMarkdownTable(fakeStuckList(total), { runUrl: 'https://example.test/actions/runs/123' }); - const rowLines = table.split('\n').filter(line => line.startsWith('| melodic-software/medley')); - assert.equal(rowLines.length, MAX_STUCK_TABLE_ROWS, 'must render at most MAX_STUCK_TABLE_ROWS data rows'); - assert.match(table, /_\.\.\.and 7 more managed job\(s\) — see the \[workflow run\]\(https:\/\/example\.test\/actions\/runs\/123\) for the full list\._/); -}); - -test('renderStuckMarkdownTable omits the run link when none is provided but still reports the remainder count', () => { - const total = MAX_STUCK_TABLE_ROWS + 3; - const table = renderStuckMarkdownTable(fakeStuckList(total)); - assert.match(table, /_\.\.\.and 3 more managed job\(s\)\._/); - assert.doesNotMatch(table, /workflow run/); -}); - -test('renderStuckMarkdownTable does not add a remainder note when the stuck count is within the cap', () => { - const table = renderStuckMarkdownTable(fakeStuckList(MAX_STUCK_TABLE_ROWS)); - assert.doesNotMatch(table, /more managed job/); -}); - -test('boundBodyLength leaves a body under the limit untouched, appending only the marker', () => { - const marker = incidentMarker('melodic-software'); - const body = boundBodyLength('short body', marker); - assert.equal(body, `short body\n\n${marker}`); -}); - -test('boundBodyLength truncates an oversized body while preserving the marker fully intact', () => { - const marker = incidentMarker('melodic-software'); - const oversized = 'x'.repeat(MAX_BODY_LENGTH * 2); - const body = boundBodyLength(oversized, marker, MAX_BODY_LENGTH); - assert.ok(body.length <= MAX_BODY_LENGTH, `bounded body must not exceed MAX_BODY_LENGTH (was ${body.length})`); - assert.ok(body.endsWith(marker), 'the marker must survive intact at the end of a truncated body'); - assert.match(body, /truncated to stay under GitHub's issue body limit/); -}); - -test('upsertIncident renders a capped, length-bounded body with the run link for an oversized stuck array', async () => { - const github = fakeGithubIssues(); - const core = fakeCore(); - const total = MAX_STUCK_TABLE_ROWS + 12; - await upsertIncident({ - github, - core, - env: { - TARGET_OWNER: 'melodic-software', - GITHUB_REPOSITORY: 'melodic-software/ci-runner', - GITHUB_SERVER_URL: 'https://github.com', - GITHUB_RUN_ID: '999999', - ISSUE_AUTHOR_LOGIN, - STUCK_JSON: JSON.stringify(fakeStuckList(total)), - }, - now: Date.parse('2026-07-22T10:00:00Z'), - }); - - const created = github.calls.find(([action]) => action === 'create'); - assert.ok(created, 'expected an issue create call'); - const [, parameters] = created; - assert.ok(parameters.body.length <= MAX_BODY_LENGTH, `body must stay under MAX_BODY_LENGTH (was ${parameters.body.length})`); - assert.match(parameters.body, /_\.\.\.and 12 more managed job\(s\) — see the \[workflow run\]\(https:\/\/github\.com\/melodic-software\/ci-runner\/actions\/runs\/999999\) for the full list\._/); - assert.ok(parameters.body.endsWith(incidentMarker('melodic-software')), 'the marker must survive at the end of the body'); -}); - -test('findOpenIncident matches an own-authored issue carrying the marker and ignores pull requests', async () => { - const marker = incidentMarker('melodic-software'); - const github = fakeGithubIssues({ - existingIssues: [ - ownIssue({ number: 1, title: 'unrelated', body: 'no marker here' }), - ownIssue({ number: 2, body: `has marker ${marker}`, pull_request: { url: 'x' } }), - ownIssue({ number: 3, body: `has marker ${marker}` }), - ], - }); - const found = await findOpenIncident({ github, homeOwner: 'melodic-software', homeRepo: 'ci-runner', marker, issueAuthorLogin: ISSUE_AUTHOR_LOGIN }); - assert.equal(found.number, 3); -}); - -test('findOpenIncident filters to the automated label server-side', async () => { - const marker = incidentMarker('melodic-software'); - const github = fakeGithubIssues({ existingIssues: [] }); - await findOpenIncident({ github, homeOwner: 'melodic-software', homeRepo: 'ci-runner', marker, issueAuthorLogin: ISSUE_AUTHOR_LOGIN }); - const [, parameters] = github.calls.find(([action]) => action === 'paginate'); - assert.equal(parameters.labels, 'automated'); - assert.equal(parameters.state, 'open'); -}); - -test('findOpenIncident rejects a decoy issue that carries the marker but was not opened by this workflow\'s own identity', async () => { - const marker = incidentMarker('melodic-software'); - const github = fakeGithubIssues({ - existingIssues: [ - { number: 9, body: `decoy ${marker}`, pull_request: undefined, user: { login: 'kyle-sexton', type: 'User' } }, - { number: 10, body: `decoy ${marker}`, pull_request: undefined, user: { login: 'some-other-bot', type: 'Bot' } }, - ], - }); - const found = await findOpenIncident({ github, homeOwner: 'melodic-software', homeRepo: 'ci-runner', marker, issueAuthorLogin: ISSUE_AUTHOR_LOGIN }); - assert.equal(found, null); -}); - -test('findOpenIncident fails closed when more than one own-authored issue carries the marker', async () => { - const marker = incidentMarker('melodic-software'); - const github = fakeGithubIssues({ - existingIssues: [ - ownIssue({ number: 3, body: `has marker ${marker}` }), - ownIssue({ number: 4, body: `has marker ${marker}` }), - ], - }); - await assert.rejects( - findOpenIncident({ github, homeOwner: 'melodic-software', homeRepo: 'ci-runner', marker, issueAuthorLogin: ISSUE_AUTHOR_LOGIN }), - /Found 2 open incident issues carrying marker/, - ); -}); - -test('a crafted job name embedding another owner\'s marker cannot cause cross-owner incident adoption', async () => { - const foreignOwner = 'owner-b'; - const foreignMarker = incidentMarker(foreignOwner); - const github = fakeGithubIssues(); - const core = fakeCore(); - - // owner-a's detection includes a job whose name is crafted to contain - // owner-b's marker verbatim. - await upsertIncident({ - github, - core, - env: { - TARGET_OWNER: 'owner-a', - GITHUB_REPOSITORY: 'melodic-software/ci-runner', - ISSUE_AUTHOR_LOGIN, - STUCK_JSON: JSON.stringify([{ repository: 'melodic-software/medley', workflow: 'CI', job: `evil ${foreignMarker} name`, queuedMinutes: 5, labels: 'melodic-ubuntu-24.04-x64', url: 'https://example.test/job/1' }]), - }, - now: Date.parse('2026-07-22T10:00:00Z'), - }); - - const created = github.calls.find(([action]) => action === 'create'); - assert.ok(created, 'expected owner-a\'s incident to be created'); - const [, parameters] = created; - - // Feed owner-a's freshly created issue body back in and search for - // owner-b's incident: the injected marker must not have survived - // rendering, so owner-b must not adopt owner-a's issue. - const githubForOwnerB = fakeGithubIssues({ - existingIssues: [ownIssue({ number: 999, title: parameters.title, body: parameters.body })], - }); - const found = await findOpenIncident({ - github: githubForOwnerB, - homeOwner: 'melodic-software', - homeRepo: 'ci-runner', - marker: foreignMarker, - issueAuthorLogin: ISSUE_AUTHOR_LOGIN, - }); - assert.equal(found, null, 'owner-b must not adopt owner-a\'s incident even though a crafted job name tried to embed owner-b\'s marker'); -}); - -test('a job name that exactly equals a legitimate marker string still renders inert after escaping', () => { - const marker = incidentMarker('melodic-software'); - const table = renderStuckMarkdownTable([ - { repository: 'melodic-software/medley', workflow: 'CI', job: marker, queuedMinutes: 3, labels: 'melodic-ubuntu-24.04-x64', url: 'https://example.test/job/exact' }, - ]); - assert.ok(!table.includes(marker), 'a job name that is exactly a marker string must not survive rendering as a functional marker'); -}); - -test('pipe-escaping and HTML-comment-escaping compose without interfering with each other', () => { - const table = renderStuckMarkdownTable([ - { repository: 'melodic-software/medley', workflow: 'CI matrix', job: 'build', queuedMinutes: 4, labels: 'x', url: 'https://example.test/job/combo' }, - ]); - assert.match(table, /CI <!-- \\\| --> matrix/); - assert.ok(!table.includes(''), 'no raw HTML comment sequence may survive alongside an escaped pipe'); -}); - -// End-to-end proof (per independent review, escalated CRITICAL against an -// earlier commit that lacked this escaping) that a crafted job name cannot -// achieve any of three failure modes against a DIFFERENT owner that has a -// genuine, currently open incident: adopt-and-overwrite it, false-close it, -// or trigger a false fail-closed ambiguity error that reds out that owner's -// run. Bodies below are built through the real renderStuckMarkdownTable / -// upsertIncident body-assembly shape, not hand-written, so the test exercises -// the actual escaping pipeline end to end. -function buildRealIncidentBody(owner, jobName, createdIso, nowIso) { - return [ - `Managed runner queue capacity alert for \`${owner}\`.`, - '', - `Capacity window: constrained since ${createdIso} (last confirmed ${nowIso}). Affected queue depth: 1 managed job(s).`, - '', - renderStuckMarkdownTable([{ repository: 'melodic-software/medley', workflow: 'CI', job: jobName, queuedMinutes: 10, labels: 'melodic-ubuntu-24.04-x64', url: 'https://example.test/job/real' }]), - '', - routingRecoverySummary.trim(), - '', - incidentMarker(owner), - ].join('\n'); -} - -test('findOpenIncident resolves exactly owner-b\'s real incident without ambiguity, even though owner-a\'s incident carries an injected owner-b marker', async () => { - const ownerBMarker = incidentMarker('owner-b'); - const github = fakeGithubIssues({ - existingIssues: [ - ownIssue({ number: 100, title: incidentTitle('owner-b'), body: buildRealIncidentBody('owner-b', 'legit-build', '2026-07-22T08:00:00.000Z', '2026-07-22T08:00:00.000Z'), created_at: '2026-07-22T08:00:00.000Z' }), - ownIssue({ number: 101, title: incidentTitle('owner-a'), body: buildRealIncidentBody('owner-a', `evil ${ownerBMarker} name`, '2026-07-22T09:00:00.000Z', '2026-07-22T09:00:00.000Z'), created_at: '2026-07-22T09:00:00.000Z' }), - ], - }); - const found = await findOpenIncident({ github, homeOwner: 'melodic-software', homeRepo: 'ci-runner', marker: ownerBMarker, issueAuthorLogin: ISSUE_AUTHOR_LOGIN }); - assert.equal(found.number, 100, 'must resolve to owner-b\'s own real issue, never owner-a\'s, and never throw ambiguity'); -}); - -test('upsertIncident updates only owner-b\'s real issue while still stuck, never adopting owner-a\'s issue via an injected marker (prevents adopt-overwrite)', async () => { - const ownerBMarker = incidentMarker('owner-b'); - const github = fakeGithubIssues({ - existingIssues: [ - ownIssue({ number: 100, title: incidentTitle('owner-b'), body: buildRealIncidentBody('owner-b', 'legit-build', '2026-07-22T08:00:00.000Z', '2026-07-22T08:00:00.000Z'), created_at: '2026-07-22T08:00:00.000Z' }), - ownIssue({ number: 101, title: incidentTitle('owner-a'), body: buildRealIncidentBody('owner-a', `evil ${ownerBMarker} name`, '2026-07-22T09:00:00.000Z', '2026-07-22T09:00:00.000Z'), created_at: '2026-07-22T09:00:00.000Z' }), - ], - }); - const core = fakeCore(); - await upsertIncident({ - github, - core, - env: { - TARGET_OWNER: 'owner-b', - GITHUB_REPOSITORY: 'melodic-software/ci-runner', - ISSUE_AUTHOR_LOGIN, - STUCK_JSON: JSON.stringify([{ repository: 'melodic-software/medley', workflow: 'CI', job: 'still-stuck', queuedMinutes: 12, labels: 'melodic-ubuntu-24.04-x64', url: 'https://example.test/job/still' }]), - }, - now: Date.parse('2026-07-22T10:00:00Z'), - }); - - const updateCalls = github.calls.filter(([action]) => action === 'update'); - assert.equal(updateCalls.length, 1, 'exactly one issue must be updated'); - assert.equal(updateCalls[0][1].issue_number, 100, 'the update must target owner-b\'s own real issue, never owner-a\'s'); - assert.equal(github.calls.filter(([action]) => action === 'create').length, 0, 'no duplicate incident should be created when owner-b\'s real one is correctly found'); -}); - -test('upsertIncident closes only owner-b\'s real issue on recovery, never owner-a\'s issue via an injected marker (prevents false-close)', async () => { - const ownerBMarker = incidentMarker('owner-b'); - const github = fakeGithubIssues({ - existingIssues: [ - ownIssue({ number: 100, title: incidentTitle('owner-b'), body: buildRealIncidentBody('owner-b', 'legit-build', '2026-07-22T08:00:00.000Z', '2026-07-22T08:00:00.000Z'), created_at: '2026-07-22T08:00:00.000Z' }), - ownIssue({ number: 101, title: incidentTitle('owner-a'), body: buildRealIncidentBody('owner-a', `evil ${ownerBMarker} name`, '2026-07-22T09:00:00.000Z', '2026-07-22T09:00:00.000Z'), created_at: '2026-07-22T09:00:00.000Z' }), - ], - }); - const core = fakeCore(); - await upsertIncident({ - github, - core, - env: { TARGET_OWNER: 'owner-b', GITHUB_REPOSITORY: 'melodic-software/ci-runner', ISSUE_AUTHOR_LOGIN, STUCK_JSON: '[]' }, - now: Date.parse('2026-07-22T10:00:00Z'), - }); - - const updateCalls = github.calls.filter(([action]) => action === 'update'); - assert.equal(updateCalls.length, 1, 'exactly one issue must be closed'); - assert.equal(updateCalls[0][1].issue_number, 100, 'recovery must close owner-b\'s own real issue, never owner-a\'s'); - assert.equal(updateCalls[0][1].state, 'closed'); - const commentCalls = github.calls.filter(([action]) => action === 'createComment'); - assert.equal(commentCalls.length, 1); - assert.equal(commentCalls[0][1].issue_number, 100); -}); - -test('incidentMarker keeps prefix-related owners from substring-colliding', () => { - const shortOwner = incidentMarker('melodic-software'); - const longOwner = incidentMarker('melodic-software-fork'); - assert.ok(!longOwner.includes(shortOwner), 'the longer owner\'s marker must not contain the shorter owner\'s marker as a substring'); - assert.ok(!shortOwner.includes(longOwner), 'the shorter owner\'s marker must not contain the longer owner\'s marker as a substring'); -}); - -test('upsertIncident opens a new incident issue when none is open and jobs are stuck', async () => { - const github = fakeGithubIssues(); - const core = fakeCore(); - const now = Date.parse('2026-07-22T10:00:00Z'); - await upsertIncident({ - github, - core, - env: { - TARGET_OWNER: 'melodic-software', - GITHUB_REPOSITORY: 'melodic-software/ci-runner', - ISSUE_AUTHOR_LOGIN, - STUCK_JSON: JSON.stringify([{ repository: 'melodic-software/medley', workflow: 'CI', job: 'build', queuedMinutes: 12, labels: 'melodic-ubuntu-24.04-x64', url: 'https://example.test/job/1' }]), - }, - now, - }); - - const created = github.calls.find(([action]) => action === 'create'); - assert.ok(created, 'expected an issue create call'); - const [, parameters] = created; - assert.equal(parameters.owner, 'melodic-software'); - assert.equal(parameters.repo, 'ci-runner'); - assert.equal(parameters.title, incidentTitle('melodic-software')); - assert.deepEqual(parameters.labels, ['automated']); - assert.match(parameters.body, /Affected queue depth: 1 managed job\(s\)/); - assert.match(parameters.body, /constrained since 2026-07-22T10:00:00\.000Z/); - assert.ok(parameters.body.includes(incidentMarker('melodic-software')), 'body must carry the incident marker'); - assert.equal(github.calls.filter(([action]) => action === 'update' || action === 'createComment').length, 0); -}); - -test('upsertIncident silently updates an already-open incident, preserving the window start and without commenting', async () => { - const title = incidentTitle('melodic-software'); - const marker = incidentMarker('melodic-software'); - const github = fakeGithubIssues({ - existingIssues: [ownIssue({ number: 55, title, body: `stale ${marker}`, created_at: '2026-07-22T09:00:00.000Z' })], - }); - const core = fakeCore(); - await upsertIncident({ - github, - core, - env: { - TARGET_OWNER: 'melodic-software', - GITHUB_REPOSITORY: 'melodic-software/ci-runner', - ISSUE_AUTHOR_LOGIN, - STUCK_JSON: JSON.stringify([ - { repository: 'melodic-software/medley', workflow: 'CI', job: 'build', queuedMinutes: 12, labels: 'melodic-ubuntu-24.04-x64', url: 'https://example.test/job/1' }, - { repository: 'melodic-software/standards', workflow: 'CI', job: 'lint', queuedMinutes: 8, labels: 'melodic-ubuntu-24.04-x64', url: 'https://example.test/job/2' }, - ]), - }, - now: Date.parse('2026-07-22T10:15:00Z'), - }); - - const updated = github.calls.find(([action]) => action === 'update'); - assert.ok(updated, 'expected an update call'); - assert.equal(updated[1].issue_number, 55); - assert.match(updated[1].body, /constrained since 2026-07-22T09:00:00\.000Z/); - assert.match(updated[1].body, /Affected queue depth: 2 managed job\(s\)/); - assert.equal(github.calls.filter(([action]) => action === 'create' || action === 'createComment').length, 0); -}); - -test('upsertIncident does not adopt a decoy issue: opens a fresh incident instead', async () => { - const title = incidentTitle('melodic-software'); - const github = fakeGithubIssues({ - existingIssues: [{ number: 66, title, body: 'no marker, wrong author', pull_request: undefined, user: { login: 'kyle-sexton', type: 'User' } }], - }); - const core = fakeCore(); - await upsertIncident({ - github, - core, - env: { - TARGET_OWNER: 'melodic-software', - GITHUB_REPOSITORY: 'melodic-software/ci-runner', - ISSUE_AUTHOR_LOGIN, - STUCK_JSON: JSON.stringify([{ repository: 'melodic-software/medley', workflow: 'CI', job: 'build', queuedMinutes: 12, labels: 'melodic-ubuntu-24.04-x64', url: 'https://example.test/job/1' }]), - }, - now: Date.parse('2026-07-22T10:00:00Z'), - }); - - const created = github.calls.find(([action]) => action === 'create'); - assert.ok(created, 'expected a fresh issue create call, not an update of the decoy'); - assert.equal(github.calls.filter(([action]) => action === 'update').length, 0); -}); - -test('upsertIncident closes and comments the incident on recovery', async () => { - const title = incidentTitle('melodic-software'); - const marker = incidentMarker('melodic-software'); - const github = fakeGithubIssues({ - existingIssues: [ownIssue({ number: 55, title, body: `stale ${marker}`, created_at: '2026-07-22T09:00:00.000Z' })], - }); - const core = fakeCore(); - await upsertIncident({ - github, - core, - env: { TARGET_OWNER: 'melodic-software', GITHUB_REPOSITORY: 'melodic-software/ci-runner', ISSUE_AUTHOR_LOGIN, STUCK_JSON: '[]' }, - now: Date.parse('2026-07-22T11:00:00Z'), - }); - - const commented = github.calls.find(([action]) => action === 'createComment'); - const updated = github.calls.find(([action]) => action === 'update'); - assert.ok(commented && updated, 'expected a recovery comment and a close update'); - assert.equal(commented[1].issue_number, 55); - assert.match(commented[1].body, /2026-07-22T09:00:00\.000Z.*2026-07-22T11:00:00\.000Z/s); - assert.equal(updated[1].issue_number, 55); - assert.equal(updated[1].state, 'closed'); - assert.equal(updated[1].state_reason, 'completed'); -}); - -test('upsertIncident is a no-op when recovered and no incident is open', async () => { - const github = fakeGithubIssues(); - const core = fakeCore(); - await upsertIncident({ - github, - core, - env: { TARGET_OWNER: 'melodic-software', GITHUB_REPOSITORY: 'melodic-software/ci-runner', ISSUE_AUTHOR_LOGIN, STUCK_JSON: '[]' }, - now: Date.parse('2026-07-22T11:00:00Z'), - }); - - assert.deepEqual(github.calls.filter(([action]) => action !== 'paginate'), []); - assert.equal(core.calls.info.length, 1); -}); - -test('upsertIncident rejects a missing home repository, target owner, or issue-author login', async () => { - const github = fakeGithubIssues(); - const core = fakeCore(); - await assert.rejects( - upsertIncident({ github, core, env: { TARGET_OWNER: 'melodic-software', ISSUE_AUTHOR_LOGIN, STUCK_JSON: '[]' } }), - /GITHUB_REPOSITORY must be set/, - ); - await assert.rejects( - upsertIncident({ github, core, env: { GITHUB_REPOSITORY: 'melodic-software/ci-runner', ISSUE_AUTHOR_LOGIN, STUCK_JSON: '[]' } }), - /TARGET_OWNER is required/, - ); - await assert.rejects( - upsertIncident({ github, core, env: { TARGET_OWNER: 'melodic-software', GITHUB_REPOSITORY: 'melodic-software/ci-runner', STUCK_JSON: '[]' } }), - /ISSUE_AUTHOR_LOGIN is required/, - ); -}); - -test('upsertIncident rejects a missing or empty STUCK_JSON instead of silently treating it as recovered', async () => { - const github = fakeGithubIssues({ - existingIssues: [ownIssue({ number: 55, body: `open ${incidentMarker('melodic-software')}`, created_at: '2026-07-22T09:00:00.000Z' })], - }); - const core = fakeCore(); - const baseEnv = { TARGET_OWNER: 'melodic-software', GITHUB_REPOSITORY: 'melodic-software/ci-runner', ISSUE_AUTHOR_LOGIN }; - - await assert.rejects(upsertIncident({ github, core, env: baseEnv }), /STUCK_JSON is required/); - await assert.rejects(upsertIncident({ github, core, env: { ...baseEnv, STUCK_JSON: '' } }), /STUCK_JSON is required/); - - // Neither rejection may have closed the pre-existing open incident. - assert.equal(github.calls.filter(([action]) => action === 'update' || action === 'createComment').length, 0); -}); - -test('upsertIncident rejects malformed or non-array STUCK_JSON', async () => { - const github = fakeGithubIssues(); - const core = fakeCore(); - const baseEnv = { TARGET_OWNER: 'melodic-software', GITHUB_REPOSITORY: 'melodic-software/ci-runner', ISSUE_AUTHOR_LOGIN }; - - await assert.rejects(upsertIncident({ github, core, env: { ...baseEnv, STUCK_JSON: '{not json' } }), /STUCK_JSON is not valid JSON/); - await assert.rejects(upsertIncident({ github, core, env: { ...baseEnv, STUCK_JSON: '{}' } }), /STUCK_JSON must decode to an array/); -}); - -test('rejects incomplete configuration and invalid thresholds', async () => { - const github = fakeGitHub(); - await assert.rejects( - inspectQueuedJobs({ github, owner: '', repositories: [], managedLabels: new Set(), thresholdMinutes: 5 }), - /configuration is incomplete/, - ); - await assert.rejects( - inspectQueuedJobs({ github, owner: 'owner', repositories: ['repo'], managedLabels: new Set(['managed']), thresholdMinutes: 0 }), - /threshold must be a positive/, - ); -}); diff --git a/.github/workflows/actions-budget-monitor.yml b/.github/workflows/actions-budget-monitor.yml deleted file mode 100644 index edffd1f..0000000 --- a/.github/workflows/actions-budget-monitor.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Actions Budget Monitor - -on: - schedule: - # Daily, not the queue monitor's quarter-hourly cadence: included-minute - # consumption is a month-scale signal, and this workflow's own minutes come - # out of the pool it watches. - - cron: '41 6 * * *' - workflow_dispatch: - -permissions: - contents: read # Checkout only; no write grants at the workflow root. - -concurrency: - group: actions-budget-monitor - cancel-in-progress: false - -env: - ISSUE_AUTHOR_LOGIN: github-actions[bot] - -jobs: - disarmed-notice: - name: Skip when disarmed - if: vars.CI_RUNNER_BUDGET_MONITOR_ARMED != 'true' - runs-on: ubuntu-24.04 - timeout-minutes: 2 - steps: - - name: Warn that the monitor is disarmed - run: | - echo "::warning::Actions budget monitor is disarmed (CI_RUNNER_BUDGET_MONITOR_ARMED != true); skipping measurement and incident upsert." - - monitor: - name: Alert on Actions included-minute consumption - if: vars.CI_RUNNER_BUDGET_MONITOR_ARMED == 'true' - runs-on: ubuntu-24.04 - timeout-minutes: 10 - permissions: - contents: read # Checkout the monitor scripts for github-script require(). - issues: write # Upsert the marker-deduped budget incident issue in this repo. - steps: - - name: Checkout monitor implementation - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Measure included-minute consumption - id: measure - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - BILLING_ORG: ${{ github.repository_owner }} - INCLUDED_MINUTES: '3000' - BILLING_TOKEN_PRESENT: ${{ secrets.CI_RUNNER_BILLING_OBSERVER_TOKEN != '' }} - BUDGET_MONITOR_ARMED: 'true' - with: - github-token: ${{ secrets.CI_RUNNER_BILLING_OBSERVER_TOKEN }} - script: | - const monitor = require(`${process.env.GITHUB_WORKSPACE}/.github/scripts/budget-monitor.cjs`); - await monitor.run({ github, core }); - - - name: Upsert budget incident - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - BUDGET_JSON: ${{ steps.measure.outputs.budget }} - BUDGET_MONITOR_ARMED: 'true' - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const monitor = require(`${process.env.GITHUB_WORKSPACE}/.github/scripts/budget-monitor.cjs`); - await monitor.upsertIncident({ github, core }); diff --git a/.github/workflows/queued-job-monitor.yml b/.github/workflows/queued-job-monitor.yml deleted file mode 100644 index 5ab8833..0000000 --- a/.github/workflows/queued-job-monitor.yml +++ /dev/null @@ -1,116 +0,0 @@ -name: Managed Runner Queue Monitor - -on: - schedule: - - cron: '7,22,37,52 * * * *' - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: managed-runner-queue-monitor - # false, not true: this serializes scheduled runs (never runs two at once) - # instead of cancelling an in-flight one. The incident-issue upsert's - # create-if-absent race freedom for a given target owner depends on this — - # two runs racing on the same owner's issue could both find none open and - # both create one. - cancel-in-progress: false - -env: - # The identity that opens/updates/closes this workflow's own incident - # issues: the default GITHUB_TOKEN's REST-created content is authored by - # this exact bot. The incident title and marker are public (embedded - # verbatim in this workflow's source, in a public repo), so restricting - # adoption to this author — not just marker/title text — is what stops a - # non-maintainer from opening a decoy issue that gets adopted, silently - # closed on the next healthy run, or used to fail the lookup closed and - # suppress a real alert. Mirrors ci-workflows#213 - # (standards-sync-stuck-automerge-alert.yml). - ISSUE_AUTHOR_LOGIN: github-actions[bot] - -jobs: - monitor: - name: Alert on ${{ matrix.target.owner }} jobs queued over five minutes - runs-on: ubuntu-24.04 - timeout-minutes: 10 - permissions: - contents: read - issues: write # Upsert the marker-deduped incident issue in this repo. - strategy: - fail-fast: false - matrix: - # Config-integrity assumption: each entry's owner is expected to be - # unique. Matrix jobs run in parallel *within* a run (only cross-run - # concurrency is serialized above), so two entries sharing an owner - # would race the incident-issue upsert for that owner — both could - # find none open and both create one. - target: ${{ fromJSON(vars.CI_RUNNER_MONITOR_TARGETS_JSON) }} - steps: - - name: Checkout monitor implementation - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Mint read-only observer token - id: app-token - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - client-id: ${{ vars.CI_RUNNER_OBSERVER_CLIENT_ID }} - private-key: ${{ secrets.CI_RUNNER_OBSERVER_PRIVATE_KEY }} - owner: ${{ matrix.target.owner }} - repositories: ${{ matrix.target.repositories }} - permission-actions: read - - - name: Inspect queued jobs - id: inspect - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - MONITOR_OWNER: ${{ matrix.target.owner }} - MONITORED_REPOSITORIES: ${{ matrix.target.repositories }} - MANAGED_LABELS: ${{ matrix.target.managedLabels }} - QUEUE_THRESHOLD_MINUTES: '5' - with: - github-token: ${{ steps.app-token.outputs.token }} - script: | - const monitor = require(`${process.env.GITHUB_WORKSPACE}/.github/scripts/queue-monitor.cjs`); - await monitor.run({ github, core }); - - # Alerts by upserting a marker-deduped incident issue instead of failing - # this run (a successful execution is green even when it detects - # strain): schedule-triggered runs have no actor to notify, and a - # by-design red here pollutes fleet-wide failure statistics. Runs on the - # job's own GITHUB_TOKEN, scoped to this repository, because the - # detection step's observer token is read-only and scoped to the - # monitored repositories, not this one. - - name: Upsert queue capacity incident - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - TARGET_OWNER: ${{ matrix.target.owner }} - STUCK_JSON: ${{ steps.inspect.outputs.stuck }} - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const monitor = require(`${process.env.GITHUB_WORKSPACE}/.github/scripts/queue-monitor.cjs`); - await monitor.upsertIncident({ github, core }); - - heartbeat: - name: Ping dead-man's-switch - runs-on: ubuntu-24.04 - needs: monitor - timeout-minutes: 5 - steps: - - name: Ping healthchecks.io - env: - CI_RUNNER_HEARTBEAT_URL: ${{ secrets.CI_RUNNER_HEARTBEAT_URL }} - run: | - if [ -z "${CI_RUNNER_HEARTBEAT_URL}" ]; then - echo "::notice::CI_RUNNER_HEARTBEAT_URL is not configured; heartbeat is inert until armed." - exit 0 - fi - if curl --fail --silent --show-error --max-time 10 --retry 5 --retry-connrefused \ - "${CI_RUNNER_HEARTBEAT_URL}"; then - echo "Heartbeat ping succeeded." - else - echo "::warning::Dead-man's-switch heartbeat ping failed after retries; the monitor may be stale or the check URL may be misconfigured." - fi diff --git a/docs/actions-budget-monitor.md b/docs/actions-budget-monitor.md deleted file mode 100644 index 1d05221..0000000 --- a/docs/actions-budget-monitor.md +++ /dev/null @@ -1,105 +0,0 @@ -# Actions budget monitor - -The public repository runs a hosted, alert-only monitor once a day that reports -how much of the organization's included GitHub Actions minute allowance the -current billing month has consumed on private repositories. Public standard -hosted execution is free, so the monitor does not draw on the allowance it -watches. - -Included-minute consumption is a month-scale signal, so this monitor runs daily -rather than at the queue monitor's quarter-hourly cadence. - -## Arming - -The monitor is gated on the repository variable -`CI_RUNNER_BUDGET_MONITOR_ARMED == 'true'`. When disarmed, the workflow emits a -warning and skips measurement and incident upsert. When armed but -`CI_RUNNER_BILLING_OBSERVER_TOKEN` is not yet provisioned, the workflow fails -loudly — an unprovisioned credential must never read as a healthy zero-consumption -month. Deciding which credential mints or extends billing-usage read scope stays -in [#113](https://github.com/melodic-software/ci-runner/issues/113); this -workflow does not mint credentials. - -## Alert channel - -Detection upserts a marker-deduped incident issue in this repository through the -same channel, adoption rules, and escaping the queue monitor uses; that behavior -and its rationale are documented once, in -[Managed runner queue monitor](queue-monitor.md). - -| Condition | Title | Marker | -| --- | --- | --- | -| 50% or 80% of the included allowance consumed | `[Alert] Actions included-minute consumption — ` | `` | - -A repeat detection at the same threshold silently updates the incident body, as -in the queue monitor. Crossing from 50% to 80% instead adds a comment: an edited -body notifies nobody, so an escalation would otherwise be invisible. Which -thresholds an open incident has already reported is recorded in its body as -`` markers, so the -comment fires exactly once per threshold. - -A new billing month resets consumption, which closes the incident through the -ordinary recovery path. - -## Billing data stays out of this public repository - -The alert reports consumed minutes, percentage of the allowance, and the -affected SKUs. It never publishes an amount, a price, or a per-repository cost. -This is the standing constraint recorded under -[Independent monitor and cost evidence](roadmap.md#independent-monitor-and-cost-evidence); -a test asserts it against the rendered bodies rather than leaving it to review. - -## Configuration - -| Name | Kind | Meaning | -| --- | --- | --- | -| `CI_RUNNER_BUDGET_MONITOR_ARMED` | variable | When `'true'`, run measurement and upsert; otherwise skip with a warning | -| `CI_RUNNER_BILLING_OBSERVER_TOKEN` | secret | Token that may read organization billing usage and resolve repository visibility | -| `INCLUDED_MINUTES` | workflow `env` | The plan's monthly included-minute allowance (3,000 for GitHub Team) | - -The measured organization is this repository's own owner; no variable declares -it. `INCLUDED_MINUTES` is workflow configuration rather than a constant because -the allowance is plan-dependent: GitHub documents 2,000 minutes per month for -GitHub Free and GitHub Free for organizations, 3,000 for GitHub Pro and GitHub -Team, and 50,000 for GitHub Enterprise Cloud. See -[GitHub Actions billing](https://docs.github.com/en/billing/concepts/product-billing/github-actions). - -## How consumption is measured - -The monitor reads the enhanced billing platform's usage report, -[`GET /organizations/{org}/settings/billing/usage`](https://docs.github.com/en/rest/billing/usage), -scoped to the current UTC billing month. - -For each usage row it: - -1. Keeps only `product == actions`, `unitType == Minutes`, and the standard - hosted SKUs (`Actions Linux`, `Actions Linux Slim`, `Actions Windows`). - Larger runners, self-hosted labels, and non-minute Actions products are - excluded. -2. Drops rows missing required fields (`organizationName`, `repositoryName`, - `sku`, `quantity`). -3. Resolves repository visibility via `GET /repos/{owner}/{repo}`. Rows on - **private** repositories contribute their raw `quantity`. Rows on **public** - repositories contribute nothing. When visibility is unknown (404, permission - error, or any lookup failure), the row's minutes are **included** rather than - dropped — under-counting would suppress alerts; over-counting is the safer - failure mode for a budget watchdog. -4. Sums minutes per SKU and compares the total against `INCLUDED_MINUTES` at - 50% and 80%. - -No runner minute multiplier is applied. No `discountAmount` / `pricePerUnit` -ratio is used; the numerator is raw `quantity` on eligible private-repo rows. - -Actions usage reported under a unit type the monitor does not recognize as -minutes fails the run rather than resolving to zero consumption, so a change in -the usage report's vocabulary cannot silently read as a healthy month. - -## Schedule availability boundary - -The same scheduler caveats the queue monitor documents apply here, including -GitHub's automatic disabling of schedules in a public repository after 60 days -without repository activity: see -[Schedule availability boundary](queue-monitor.md#schedule-availability-boundary). -A daily monitor tolerates a delayed or dropped scheduled run more comfortably -than a queue watchdog does, because month-scale consumption changes slowly, but -no elapsed-time guarantee is claimed while the workflow is disabled. diff --git a/docs/queue-monitor.md b/docs/queue-monitor.md deleted file mode 100644 index 060d99d..0000000 --- a/docs/queue-monitor.md +++ /dev/null @@ -1,173 +0,0 @@ -# Managed runner queue monitor - -The public repository runs a hosted, alert-only monitor at minutes 7, 22, 37, -and 52 of every hour. Public standard hosted execution is free, and keeping this -control plane off the managed fleet means it still reports when both local hosts -are unavailable. - -The monitor mints a short-lived installation token, scoped to only repository -Actions read permission on the monitored repositories, to inspect queue depth. -Writing the incident issue in this repository uses the job's own default -`GITHUB_TOKEN` instead, so no additional IaC configuration is needed for -alerting. Configure these values through IaC: - -| Name | Kind | Meaning | -| --- | --- | --- | -| `CI_RUNNER_OBSERVER_CLIENT_ID` | variable | Observer GitHub App client ID | -| `CI_RUNNER_OBSERVER_PRIVATE_KEY` | secret | Observer App private key | -| `CI_RUNNER_MONITOR_TARGETS_JSON` | variable | Installation-target JSON array | -| `CI_RUNNER_HEARTBEAT_URL` | secret | healthchecks.io ping URL (optional; see [Dead-man's-switch heartbeat](#dead-mans-switch-heartbeat)) | - -`CI_RUNNER_OBSERVER_CLIENT_ID` deliberately uses the GitHub App client ID. -`actions/create-github-app-token` deprecates its numeric `app-id` input; the -numeric `CI_RUNNER_OBSERVER_APP_ID` name is therefore rejected rather than kept -as an alias. - -Each target contains `owner`, comma/newline-separated `repositories`, and -comma/newline-separated exact `managedLabels`. The organization-only value is: - -```json -[ - { - "owner": "melodic-software", - "repositories": "medley,standards,claude-code-plugins,github-iac", - "managedLabels": "melodic-ubuntu-24.04-x64" - } -] -``` - -An installation token belongs to exactly one GitHub App installation. The -personal phase therefore appends a `kyle-sexton` object instead of changing the -workflow. Each matrix job mints an independent Actions-read token for its owner. -Repository lists remain data owned by IaC; the example documents shape, not a -hard-coded runtime inventory. - -For each configured repository the monitor paginates every nonterminal workflow -run status GitHub exposes (`queued`, `in_progress`, `requested`, `waiting`, and -`pending`), deduplicates runs, paginates their latest jobs, and finds -runner-eligible `queued` jobs carrying an exact managed label. Jobs still -`requested`, `waiting` on dependencies, or `pending` behind concurrency are not -treated as runner-capacity failures. - -A successful execution reports green regardless of what it finds: a queued job -older than five minutes is a capacity alert, not a monitor failure, so it no -longer fails the run. By-design reds here used to pollute fleet-wide failure -dashboards and, since scheduled runs have no actor, reached nobody who wasn't -watching the Actions tab. Detection instead upserts a marker-deduped incident -issue in this repository — the fleet's established alert-per-incident pattern -(see `link-check.yml`, `queue-monitor-liveness.yml`, and -`standards-sync-stuck-automerge-alert.yml` in `melodic-software/ci-workflows`): -one open issue per target owner, titled -`[Alert] Managed runner queue capacity — ` and carrying a hidden -`` marker in its body, -silently updated in place on repeat detections (an edited issue body notifies -nobody, unlike a comment) and closed with a recovery comment once the queue -clears. Because this repository is public, the marker and title text are -themselves public (embedded verbatim in this workflow's source), so adoption -is additionally restricted to issues authored by this workflow's own -`GITHUB_TOKEN` identity (`github-actions[bot]`) — otherwise a non-maintainer -could open a decoy issue that gets silently adopted, updated, or closed as -recovered, suppressing a real alert. Matching stays marker-only, matching the -fleet precedent's deliberate "a marker survives a retitle" property: the -detection table embeds monitored-repo job and workflow names verbatim, so a -crafted job name could otherwise inject a different owner's marker string -into a bot-authored body and cause a cross-owner collision — the table -neutralizes `<` and `>` at render time (HTML-entity-encoded, so they still -display literally) rather than layering a second, title-based guard that -would trade away retitle-survival for redundant protection. More than one -own-authored issue carrying the same marker fails the run closed rather than -guessing which one is authoritative. The job's own `GITHUB_TOKEN` (job-level -`issues: write`) -writes that issue, separately from the read-only, target-scoped observer -token used to inspect queued jobs, which cannot write here. Normal GitHub -notifications on issue creation answer the "no actor" constraint. The issue -body carries the detection table, the capacity window (`constrained since` -the issue's creation timestamp), and the affected queue depth, so no-runner -failures elsewhere can be cross-referenced against it. The table caps at 50 -rows with a "…and N more, see the workflow run" remainder note linking back -to the run, and the whole body is bounded well under GitHub's issue-body -write limit (empirically 65536 characters) while always preserving the -trailing marker intact, so a queue-wide outage with many stuck jobs cannot -itself break the alert. Only a genuine execution error — a bad configuration, -a GitHub API failure, an ambiguous marker match — still fails the run; that -is the monitor breaking, not a queue alert. - -Each alert links directly to the affected jobs and carries this recovery -instruction: - -> Confirm the managed runner host is running a release that includes the latest -> capacity and reconciliation fixes (at least `v0.1.21` / current `main` tip) -> before changing routing. An unrebuilt host on an older controller will keep -> queuing work against dead capacity even when `main` already carries the fix. -> -> Then follow the audited CI routing-control procedure to make the affected -> repository's effective `CI_RUNNER_POLICY` value `hosted-only` and verify the -> readback. Cancel the affected run, choose **Re-run all jobs** to guarantee -> that the selector executes again, and confirm that it selects hosted capacity. -> Do not use a failed-job or single-job rerun for this recovery because -> partial-rerun dependency behavior does not guarantee a fresh selector -> decision. A `workflow_dispatch` creates a separate run with different event -> and ref context; it does not recover the original pull-request check. - -The monitor never changes policy, cancels, reruns, dispatches, or mutates a -workload. The central selector applies its policy-driven routing rules on every -attempt; it has no rerun-only hosted branch. Because a repository variable -overrides an organization variable with the same name, recovery verifies the -effective value instead of assuming an organization update is sufficient. The -canonical procedure is the -[`github-iac` local-CI routing runbook](https://github.com/melodic-software/github-iac/blob/main/README.md#local-ci-routing-governance); -GitHub documents the [full and partial rerun -operations](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/re-run-workflows-and-jobs), -the [`workflow_dispatch` event -context](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#workflow_dispatch), -and the precedence rule in its [variables -reference](https://docs.github.com/en/actions/reference/workflows-and-actions/variables#configuration-variable-precedence). - -Watch this repository's issues (or the `[Alert] Managed runner queue capacity` -title prefix) rather than the failed-workflow email GitHub sends for the -account that owns the schedule: a healthy detection run is green and sends no -such notification. See [workflow-run -notifications](https://docs.github.com/en/actions/concepts/workflows-and-actions/notifications-for-workflow-runs) -for the (now unused, by design) failure-notification path. - -## Schedule availability boundary - -The four off-the-hour entries reduce contention; they do not form a hard -15-minute watchdog. GitHub documents that scheduled events can be delayed during -high load and, under sufficiently high load, some queued scheduled jobs can be -dropped. GitHub also automatically disables schedules in a public repository -after 60 days without repository activity. See [scheduled-event -behavior](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#schedule) -and [automatic schedule disabling](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/disable-and-enable-workflows). - -Accordingly, the five-minute queue threshold applies when a monitor invocation -runs; no elapsed-time SLO is claimed while GitHub's scheduler is delayed or the -workflow is disabled. The production rollout checklist must verify that this -workflow is enabled and has a recent successful run. - -## Dead-man's-switch heartbeat - -An off-GitHub dead-man's switch closes the silent-failure gap left by GitHub's -scheduler: schedule drops, the public-repository 60-day inactivity auto-disable, -workflow or repository disablement, and GitHub outages all stop pings without -emitting an error inside this repository. The monitor's `heartbeat` job pings -[healthchecks.io](https://healthchecks.io/) (Hobbyist/free tier) after every -successful monitor run — when all matrix targets complete green — and the -service alerts on **absence** of the ping. - -Operator setup (outside this repository): - -1. Create a check on healthchecks.io and configure **email** as the alert - channel. -2. Set the check **Period** to about one hour and **Grace Time** to about - thirty minutes. The workflow schedule fires roughly every fifteen minutes, - so several pings land inside each period; the grace window absorbs GitHub - scheduler delay without false positives. -3. Add the check's ping URL as repository secret `CI_RUNNER_HEARTBEAT_URL`. - -Until that secret is present the heartbeat job emits a workflow notice and exits -green (fail-soft / inert). When armed, it `curl`s the URL from the environment -(the URL is never logged). Transient ping failures after retries produce a -workflow warning annotation but do **not** fail the run — a missed ping must -not mask a genuine monitor execution error, and healthchecks.io's absence alert -is the authoritative signal for a stale monitor. From 77caf2f06a31cbccf65c072ce87b3c59bff78328 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:14:11 -0400 Subject: [PATCH 2/2] ci: fix stale references after monitor retirement Companion to 83d76f6dec1c61dfc86075b175c6945092ba7782, staged separately because git rm only staged the deletions themselves. - README.md: remove the two "Further documentation" links that pointed at the deleted docs/queue-monitor.md and docs/actions-budget-monitor.md. - docs/releases.md: the release-tag verification job description named "queue-monitor tests" specifically; renamed to "workflow-script tests" since queue-monitor.test.cjs no longer exists. - release/dependencies.json: removed the actions/create-github-app-token entry. Test-ReleasePins.ps1 fails closed on any githubActions manifest entry not used by a workflow, and queued-job-monitor.yml was the only caller of that action. Co-Authored-By: Claude Fable 5.1 --- README.md | 2 -- docs/releases.md | 2 +- release/dependencies.json | 5 ----- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/README.md b/README.md index 783dd3d..86ba013 100644 --- a/README.md +++ b/README.md @@ -435,8 +435,6 @@ failed job or worker; see ## Further documentation - [Worker image and isolation contract](docs/worker-image.md) -- [Queue-monitor behavior and scheduler limits](docs/queue-monitor.md) -- [Actions budget monitor](docs/actions-budget-monitor.md) - [OpenTelemetry observability](docs/observability.md) - [Immutable releases, freshness, and rollback](docs/releases.md) - [Deferred capabilities and non-workaround boundaries](docs/roadmap.md) diff --git a/docs/releases.md b/docs/releases.md index 5de02c9..fce4fc7 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -29,7 +29,7 @@ the exact-SHA-pinned reusable Go-quality workflow. Its repository-local build lane only cross-compiles the Windows executables, and active 30-second fuzzing per target runs only on the weekly schedule or a manual dispatch. A release tag still starts with a separate read-only job that reruns module verification, vet, -tests, race tests, vulnerability scanning, queue-monitor tests, Actionlint with +tests, race tests, vulnerability scanning, workflow-script tests, Actionlint with ShellCheck, Zizmor, Windows compilation, official-source dependency freshness, and a local worker-image contract build against that exact tag. Only the dependent publication job receives the combined job-scoped `contents:write`, diff --git a/release/dependencies.json b/release/dependencies.json index 82f5d08..63a4b3e 100644 --- a/release/dependencies.json +++ b/release/dependencies.json @@ -83,11 +83,6 @@ "version": "7.0.1", "commit": "3d3c42e5aac5ba805825da76410c181273ba90b1" }, - { - "repository": "actions/create-github-app-token", - "version": "3.2.0", - "commit": "bcd2ba49218906704ab6c1aa796996da409d3eb1" - }, { "repository": "actions/dependency-review-action", "version": "5.0.0",