diff --git a/.ai/skills/hack-repo-verify/SKILL.md b/.ai/skills/hack-repo-verify/SKILL.md index 39a1657d3..4b01408db 100644 --- a/.ai/skills/hack-repo-verify/SKILL.md +++ b/.ai/skills/hack-repo-verify/SKILL.md @@ -70,7 +70,7 @@ untracked candidate additions. Keep private artifacts excluded from the index. | Native authored command slices | `tests/native-authored-command.test.ts` (unsupported matrix), `tests/native-authored-command-dispatch.test.ts`, `tests/native-authored-project-{start,run,observe,logs,exec,recovery}.test.ts` and the [command contract](../../../docs/reference/native-authored-commands.md); the dispatch "source CLI" cases need the prepared compiler (`bun run build:config-compiler` or absolute `HACK_CONFIG_COMPILER_BINARY`) and fail without it; `native-authored-project-start` and `native-authored-project-exec` are nondeterministic on a local macOS host, so a single local pass or failure is not proof; live acceptance runs host-serialized on one prepared macOS provider slot | | Native VM/routing/reclamation | [candidate guide](../../../docs/guides/native-candidate.md), relevant acceptance gate and isolated host fixture; Linux checks cannot qualify macOS effects | | Resource/performance claims | [performance skill](../hack-repo-performance/SKILL.md), matched workload and measured boundaries | -| Bun runtime crashes | Read-only history: `bun run ci:bun-crash-census -- --since --until ` (failed `test` job logs; `--scan-passing` for all). Matched reproduction: the gated [experiment](../../../.github/workflows/bun-crash-experiment.yml), which spends hosted macOS minutes and needs an explicit go-ahead; never loop on a shared host. Both share [the classifier](../../../scripts/lib/bun-crash-classify.ts) | +| Bun runtime crashes | Read-only history: `bun run ci:bun-crash-census -- --since --until ` (failed `test` job logs; `--scan-passing` for all). Matched reproduction: the gated [experiment](../../../.github/workflows/bun-crash-experiment.yml), which spends hosted macOS minutes and needs an explicit go-ahead; never loop on a shared host. Its default `focused` mode loops `tests/native-https-owner.test.ts` (where both original segfaults happened); `full` needs a budget of at least 40 minutes because one full suite takes 30-35. A run never starts unless its estimate fits the remaining budget, so read `summary.json` for the verdict and per-mode crash, failure and skipped-for-budget counts. Both share [the classifier](../../../scripts/lib/bun-crash-classify.ts) | Read [CI](../../../.github/workflows/ci.yml) for current hosted gates. A local pass does not mean CI ran. Existing unsupported remote/gateway/native-app checks are diff --git a/.github/workflows/bun-crash-experiment.yml b/.github/workflows/bun-crash-experiment.yml index 170bd2e11..6802c92b4 100644 --- a/.github/workflows/bun-crash-experiment.yml +++ b/.github/workflows/bun-crash-experiment.yml @@ -8,8 +8,15 @@ name: Bun crash experiment # go-ahead. # # A labeled pull request runs the matched pair on the pull request's head: the -# 1.3.9 baseline and the 1.4.2 candidate pinned on next, with the same job and -# budget. A dispatch runs one Bun version at a chosen source revision. +# 1.3.9 baseline and the 1.4.2 candidate pinned on next, with the same job, +# budget and the default `focused` mode. A dispatch runs one Bun version at a +# chosen source revision and mode. +# +# Modes: `focused` loops tests/native-https-owner.test.ts, the file in which both +# original segfaults happened (about 16 s per run); `full` loops the full +# `bun test` (about 30-35 min per run on hosted macOS); `alternate` alternates +# the two. The loop never starts a run whose estimate exceeds the remaining +# budget; it records the command as skipped for budget instead. on: workflow_dispatch: inputs: @@ -22,9 +29,17 @@ on: type: string default: "" budget_minutes: - description: Loop budget in minutes (5-240); the job timeout is budget + 15 + description: Loop budget in minutes (5-240; full mode needs at least 40); the job timeout is budget + 15 type: number default: 30 + mode: + description: focused loops the owner test file, full loops the full suite, alternate alternates them + type: choice + options: + - focused + - full + - alternate + default: focused pull_request: types: [labeled] @@ -46,7 +61,8 @@ jobs: permissions: contents: read outputs: - bun_versions: ${{ steps.plan.outputs.bun_versions }} + runs: ${{ steps.plan.outputs.runs }} + mode: ${{ steps.plan.outputs.mode }} source_revision: ${{ steps.plan.outputs.source_revision }} budget_seconds: ${{ steps.plan.outputs.budget_seconds }} timeout_minutes: ${{ steps.plan.outputs.timeout_minutes }} @@ -58,6 +74,7 @@ jobs: INPUT_BUN_VERSION: ${{ inputs.bun_version }} INPUT_SOURCE_REVISION: ${{ inputs.source_revision }} INPUT_BUDGET_MINUTES: ${{ inputs.budget_minutes }} + INPUT_MODE: ${{ inputs.mode }} PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} EVENT_SHA: ${{ github.sha }} run: | @@ -67,13 +84,15 @@ jobs: echo "Invalid bun_version" exit 1 fi - versions="[\"$INPUT_BUN_VERSION\"]" + runs="[{\"bun_version\":\"$INPUT_BUN_VERSION\",\"role\":\"\"}]" revision="${INPUT_SOURCE_REVISION:-$EVENT_SHA}" budget="$INPUT_BUDGET_MINUTES" + mode="$INPUT_MODE" else - versions='["1.3.9","1.4.2"]' + runs='[{"bun_version":"1.3.9","role":"baseline"},{"bun_version":"1.4.2","role":"candidate"}]' revision="$PR_HEAD_SHA" budget=30 + mode=focused fi if ! [[ "$revision" =~ ^[0-9a-f]{40}$ ]]; then echo "Invalid source_revision" @@ -83,20 +102,35 @@ jobs: echo "Invalid budget_minutes" exit 1 fi + case "$mode" in + focused | full | alternate) ;; + *) + echo "Invalid mode" + exit 1 + ;; + esac + # The loop's full-suite prior is 40 minutes (DEFAULT_FULL_ESTIMATE_SECONDS + # in scripts/bun-crash-loop.ts); a smaller full-mode budget could never + # start a run and would only spend hosted macOS minutes on setup. + if [ "$mode" = "full" ] && [ "$budget" -lt 40 ]; then + echo "full mode needs budget_minutes of at least 40" + exit 1 + fi { - echo "bun_versions=$versions" + echo "runs=$runs" + echo "mode=$mode" echo "source_revision=$revision" echo "budget_seconds=$((budget * 60))" echo "timeout_minutes=$((budget + 15))" } >> "$GITHUB_OUTPUT" loop: - name: Bun crash loop (${{ matrix.bun_version }}) + name: Bun crash loop (${{ matrix.bun_version }}, ${{ needs.plan.outputs.mode }}) needs: plan strategy: fail-fast: false matrix: - bun_version: ${{ fromJSON(needs.plan.outputs.bun_versions) }} + include: ${{ fromJSON(needs.plan.outputs.runs) }} runs-on: blacksmith-6vcpu-macos-15 timeout-minutes: ${{ fromJSON(needs.plan.outputs.timeout_minutes) }} permissions: @@ -144,15 +178,19 @@ jobs: else echo "Source revision has no test preflight" fi - - name: Loop the owner test and the full suite within the budget + - name: Loop the selected tests within the budget env: BUN_ENABLE_CRASH_REPORTING: "0" EXPECTED_BUN_VERSION: ${{ matrix.bun_version }} SOURCE_REVISION: ${{ needs.plan.outputs.source_revision }} BUDGET_SECONDS: ${{ needs.plan.outputs.budget_seconds }} + LOOP_MODE: ${{ needs.plan.outputs.mode }} + LOOP_ROLE: ${{ matrix.role }} run: >- bun "$RUNNER_TEMP/bun-crash-instrument/scripts/bun-crash-loop.ts" --budget-seconds="$BUDGET_SECONDS" + --mode="$LOOP_MODE" + ${LOOP_ROLE:+"--role=$LOOP_ROLE"} --expect-bun-version="$EXPECTED_BUN_VERSION" --source-revision="$SOURCE_REVISION" --out-dir="$RUNNER_TEMP/bun-crash-loop" diff --git a/scripts/bun-crash-loop.ts b/scripts/bun-crash-loop.ts index 220aa1d21..a8e9a1581 100644 --- a/scripts/bun-crash-loop.ts +++ b/scripts/bun-crash-loop.ts @@ -2,14 +2,26 @@ /** * Fixed-budget Bun crash loop for the gated `bun-crash-experiment` workflow. * - * It alternates `bun test tests/native-https-owner.test.ts` with the full - * `bun test`, classifies every run with the shared classifier, keeps each run's - * log, and writes a closed, value-free summary. + * Modes: + * - `focused` (default) loops `bun test tests/native-https-owner.test.ts`, the file + * in which both original segfaults happened; + * - `full` loops the full `bun test`; + * - `alternate` alternates the two. + * Every run is classified with the shared classifier and keeps its own log, and + * the loop writes a closed, value-free summary with per-command counts and a + * one-line verdict. * - * Budget invariant: an iteration starts only when its estimated duration (the - * larger of its initial estimate and its longest observed run) ends at or before - * the deadline. An iteration that is still running at the deadline is stopped and - * recorded as `overrun`, never as a crash, unless its log carries the crash banner. + * Budget invariant: an iteration starts only when its estimated duration ends at + * or before the deadline. Until a command has completed a run, its estimate is a + * conservative prior; afterwards it is the longest completed run of that command + * times {@link ESTIMATE_HEADROOM}. A command whose estimate does not fit is + * recorded once as skipped for budget and leaves the rotation: its estimate cannot + * change without a run and the remaining budget only shrinks, so it could never + * fit again. The loop ends when every command has been skipped. + * + * A run still going at the deadline (an estimate that was too small) is stopped + * and recorded as `overrun`, never as a crash, unless its log carries the crash + * banner. With honest estimates this does not happen; the stop is a safety net. * * The loop never opens or fetches crash-report links. It sets * `BUN_ENABLE_CRASH_REPORTING=0` so Bun does not upload a report itself; the logs @@ -39,6 +51,19 @@ import { export type LoopCommandId = "focused" | "full"; +/** Which commands the loop runs: one file, the full suite, or both in turn. */ +export type LoopMode = "focused" | "full" | "alternate"; + +export const LOOP_MODES: readonly LoopMode[] = ["focused", "full", "alternate"]; +export const DEFAULT_LOOP_MODE: LoopMode = "focused"; + +/** Closed labels for the matched pair; a dispatched run has none. */ +export type LoopRole = "baseline" | "candidate"; +const LOOP_ROLES: readonly LoopRole[] = ["baseline", "candidate"]; + +/** The test file in which both original segfaults happened. */ +export const FOCUSED_TEST_FILE = "tests/native-https-owner.test.ts"; + export interface LoopCommand { readonly id: LoopCommandId; readonly argv: readonly string[]; @@ -66,15 +91,35 @@ export interface IterationRecord { readonly durationMs: number; } +/** A command the loop did not start because its estimate exceeded the remaining budget. */ +export interface SkippedForBudget { + readonly command: LoopCommandId; + readonly reason: "budget"; + /** Loop time at which the skip was decided. */ + readonly atMs: number; + readonly remainingMs: number; + readonly estimateMs: number; +} + export interface LoopResult { readonly budgetMs: number; readonly elapsedMs: number; + /** The configured commands, in rotation order. */ + readonly commands: readonly LoopCommandId[]; readonly iterations: readonly IterationRecord[]; - readonly estimatesMs: Readonly>; + readonly skippedForBudget: readonly SkippedForBudget[]; + /** The estimate each command had when the loop ended. */ + readonly estimatesMs: Readonly>>; } export const DEFAULT_FOCUSED_ESTIMATE_SECONDS = 120; -export const DEFAULT_FULL_ESTIMATE_SECONDS = 600; +/** + * The full CLI suite takes about 30 to 35 minutes on hosted macOS, so the prior + * is 40 minutes. The workflow's full-mode budget floor mirrors this value. + */ +export const DEFAULT_FULL_ESTIMATE_SECONDS = 2400; +/** Multiplier on the longest completed run, to absorb run-to-run variance. */ +export const ESTIMATE_HEADROOM = 1.2; const MIN_BUDGET_SECONDS = 60; const MAX_BUDGET_SECONDS = 6 * 60 * 60; const KILL_GRACE_MS = 10_000; @@ -85,50 +130,75 @@ export class LoopError extends Error {} export function loopCommands(opts: { readonly bunExecutable: string; + readonly mode: LoopMode; readonly focusedEstimateSeconds: number; readonly fullEstimateSeconds: number; }): readonly LoopCommand[] { - return [ - { - id: "focused", - argv: [opts.bunExecutable, "test", "tests/native-https-owner.test.ts"], - initialEstimateMs: opts.focusedEstimateSeconds * 1000, - }, - { - id: "full", - argv: [opts.bunExecutable, "test"], - initialEstimateMs: opts.fullEstimateSeconds * 1000, - }, - ]; + const focused: LoopCommand = { + id: "focused", + argv: [opts.bunExecutable, "test", FOCUSED_TEST_FILE], + initialEstimateMs: opts.focusedEstimateSeconds * 1000, + }; + const full: LoopCommand = { + id: "full", + argv: [opts.bunExecutable, "test"], + initialEstimateMs: opts.fullEstimateSeconds * 1000, + }; + switch (opts.mode) { + case "focused": + return [focused]; + case "full": + return [full]; + case "alternate": + return [focused, full]; + default: + throw new LoopError("Invalid mode"); + } +} + +/** + * The estimate for the next start of a command: its conservative prior until a + * run completes, then the longest completed run times {@link ESTIMATE_HEADROOM}. + */ +export function nextEstimateMs(opts: { + readonly priorMs: number; + readonly longestCompletedMs: number | null; +}): number { + return opts.longestCompletedMs === null + ? opts.priorMs + : Math.ceil(opts.longestCompletedMs * ESTIMATE_HEADROOM); } /** - * Picks the next command, or null when none can finish by the deadline. - * The command at `cursor` is preferred; when it does not fit, the next command in - * sequence order that fits runs instead. Equality with the deadline fits. + * Decides the next step for the active rotation: run the command at `cursor` + * when it ends at or before the deadline, otherwise skip it for budget. Equality + * with the deadline fits; nothing fits once the deadline is reached. + * Returns null when no command is left. */ -export function planNextCommand(opts: { +export function planNextIteration(opts: { readonly nowMs: number; readonly deadlineMs: number; - readonly commands: readonly LoopCommand[]; + readonly active: readonly LoopCommand[]; readonly cursor: number; - readonly estimatesMs: Readonly>; -}): { readonly command: LoopCommand; readonly cursor: number } | null { - const count = opts.commands.length; - if (count === 0 || opts.nowMs >= opts.deadlineMs) { + readonly estimatesMs: Readonly>>; +}): { + readonly action: "run" | "skip"; + readonly command: LoopCommand; + readonly position: number; +} | null { + const count = opts.active.length; + if (count === 0) { return null; } - const preferred = ((opts.cursor % count) + count) % count; - for (let offset = 0; offset < count; offset += 1) { - const command = opts.commands[(preferred + offset) % count]; - if ( - command !== undefined && - opts.nowMs + opts.estimatesMs[command.id] <= opts.deadlineMs - ) { - return { command, cursor: preferred + 1 }; - } + const position = ((opts.cursor % count) + count) % count; + const command = opts.active[position]; + if (command === undefined) { + return null; } - return null; + const estimate = opts.estimatesMs[command.id] ?? command.initialEstimateMs; + const fits = + opts.nowMs < opts.deadlineMs && opts.nowMs + estimate <= opts.deadlineMs; + return { action: fits ? "run" : "skip", command, position }; } function classifyIteration(run: ExecutedRun): { @@ -143,7 +213,7 @@ function classifyIteration(run: ExecutedRun): { return classifyBunRun(run); } -/** Runs commands until none can finish within the budget. */ +/** Runs commands until every command has been skipped for budget. */ export async function runBunCrashLoop(opts: { readonly budgetMs: number; readonly commands: readonly LoopCommand[]; @@ -153,36 +223,62 @@ export async function runBunCrashLoop(opts: { context: { readonly index: number; readonly deadlineMs: number } ) => Promise; readonly onIteration?: (record: IterationRecord) => void; + readonly onSkip?: (skip: SkippedForBudget) => void; }): Promise { const startMs = opts.now(); const deadlineMs = startMs + opts.budgetMs; - const estimatesMs = {} as Record; + const estimatesMs: Partial> = {}; + const longestCompletedMs: Partial> = {}; for (const command of opts.commands) { estimatesMs[command.id] = command.initialEstimateMs; } + const active = [...opts.commands]; const iterations: IterationRecord[] = []; + const skippedForBudget: SkippedForBudget[] = []; let cursor = 0; for (;;) { - const next = planNextCommand({ - nowMs: opts.now(), + const nowMs = opts.now(); + const next = planNextIteration({ + nowMs, deadlineMs, - commands: opts.commands, + active, cursor, estimatesMs, }); if (next === null) { break; } - cursor = next.cursor; + if (next.action === "skip") { + const skip: SkippedForBudget = { + command: next.command.id, + reason: "budget", + atMs: Math.max(0, nowMs - startMs), + remainingMs: Math.max(0, deadlineMs - nowMs), + estimateMs: estimatesMs[next.command.id] ?? 0, + }; + skippedForBudget.push(skip); + opts.onSkip?.(skip); + // The command after the skipped one moves into its position. + active.splice(next.position, 1); + cursor = next.position; + continue; + } + cursor = next.position + 1; const index = iterations.length + 1; - const started = opts.now(); const run = await opts.execute(next.command, { index, deadlineMs }); - const durationMs = Math.max(0, opts.now() - started); - estimatesMs[next.command.id] = Math.max( - estimatesMs[next.command.id], - durationMs - ); + const durationMs = Math.max(0, opts.now() - nowMs); const { outcome, reason } = classifyIteration(run); + if (!run.stoppedAtDeadline) { + const longest = Math.max( + longestCompletedMs[next.command.id] ?? 0, + durationMs + ); + longestCompletedMs[next.command.id] = longest; + estimatesMs[next.command.id] = nextEstimateMs({ + priorMs: next.command.initialEstimateMs, + longestCompletedMs: longest, + }); + } const record: IterationRecord = { index, command: next.command.id, @@ -201,32 +297,114 @@ export async function runBunCrashLoop(opts: { return { budgetMs: opts.budgetMs, elapsedMs: Math.max(0, opts.now() - startMs), + commands: opts.commands.map((command) => command.id), iterations, + skippedForBudget, estimatesMs, }; } -const OUTCOMES: readonly IterationOutcome[] = [ - "pass", - "failure", - "crash", - "overrun", -]; - -export function summarizeIterations( - iterations: readonly IterationRecord[] -): Record> { - const empty = (): Record => ({ - pass: 0, - failure: 0, - crash: 0, - overrun: 0, +/** Per-command counts. `failures` are non-crash failures. */ +export interface CommandCounts { + iterations: number; + passes: number; + crashes: number; + failures: number; + overruns: number; + skippedForBudget: number; +} + +/** Counts for each configured command, in rotation order. */ +export function countIterations( + result: Pick +): Partial> { + const counts: Partial> = {}; + for (const id of result.commands) { + counts[id] = { + iterations: 0, + passes: 0, + crashes: 0, + failures: 0, + overruns: 0, + skippedForBudget: 0, + }; + } + const outcomeField = { + pass: "passes", + crash: "crashes", + failure: "failures", + overrun: "overruns", + } as const satisfies Record; + for (const record of result.iterations) { + const row = counts[record.command]; + if (row !== undefined) { + row.iterations += 1; + row[outcomeField[record.outcome]] += 1; + } + } + for (const skip of result.skippedForBudget) { + const row = counts[skip.command]; + if (row !== undefined) { + row.skippedForBudget += 1; + } + } + return counts; +} + +/** + * One closed line, for example `baseline 0/97 focused crashes` or + * `Bun 1.4.2 0/40 focused crashes, 0/1 full crashes`. + */ +export function renderVerdict(opts: { + readonly result: Pick< + LoopResult, + "commands" | "iterations" | "skippedForBudget" + >; + readonly role: LoopRole | null; + readonly bunVersion: string | null; +}): string { + const counts = countIterations(opts.result); + const subject = opts.role ?? `Bun ${opts.bunVersion ?? "unknown"}`; + const parts = opts.result.commands.map((id) => { + const row = counts[id]; + return `${row?.crashes ?? 0}/${row?.iterations ?? 0} ${id} crashes`; }); - const totals = { focused: empty(), full: empty() }; - for (const record of iterations) { - totals[record.command][record.outcome] += 1; + const overruns = opts.result.commands.reduce( + (sum, id) => sum + (counts[id]?.overruns ?? 0), + 0 + ); + let verdict = `${subject} ${parts.join(", ")}`; + if (opts.result.iterations.length === 0) { + verdict += "; no iteration fit the budget"; + } else if (overruns > 0) { + verdict += `; ${overruns} overrun`; } - return totals; + return verdict; +} + +/** The closed, value-free `summary.json` document. */ +export function buildLoopSummary(opts: { + readonly result: LoopResult; + readonly mode: LoopMode; + readonly role: LoopRole | null; + readonly bunVersion: string | null; + readonly sourceRevision: string | null; + readonly diagnosticReports: number; +}) { + return { + verdict: renderVerdict(opts), + bunVersion: opts.bunVersion, + role: opts.role, + mode: opts.mode, + sourceRevision: opts.sourceRevision, + diagnosticReports: opts.diagnosticReports, + budgetMs: opts.result.budgetMs, + elapsedMs: opts.result.elapsedMs, + counts: countIterations(opts.result), + estimatesMs: opts.result.estimatesMs, + skippedForBudget: opts.result.skippedForBudget, + iterations: opts.result.iterations, + }; } const seconds = (ms: number): string => (ms / 1000).toFixed(1); @@ -234,31 +412,36 @@ const seconds = (ms: number): string => (ms / 1000).toFixed(1); /** Markdown for `$GITHUB_STEP_SUMMARY`. Only closed tokens, numbers and validated identifiers. */ export function renderLoopSummary(opts: { readonly result: LoopResult; + readonly mode: LoopMode; + readonly role: LoopRole | null; readonly bunVersion: string | null; readonly sourceRevision: string | null; readonly diagnosticReports: number; }): string { - const totals = summarizeIterations(opts.result.iterations); + const counts = countIterations(opts.result); const lines = [ "## Bun crash experiment", "", + `Verdict: ${renderVerdict(opts)}`, + "", "| Field | Value |", "| --- | --- |", `| Bun | ${opts.bunVersion ?? "unknown"} |`, + `| Role | ${opts.role ?? "-"} |`, + `| Mode | ${opts.mode} |`, `| Source revision | ${opts.sourceRevision ?? "unknown"} |`, `| Budget seconds | ${seconds(opts.result.budgetMs)} |`, `| Loop seconds | ${seconds(opts.result.elapsedMs)} |`, `| Iterations | ${opts.result.iterations.length} |`, `| Crash reports (bun*.ips) | ${opts.diagnosticReports} |`, "", - "| Command | Runs | Pass | Failure | Crash | Overrun | Estimate seconds |", - "| --- | --- | --- | --- | --- | --- | --- |", + "| Command | Runs | Pass | Crash | Other failure | Overrun | Skipped for budget | Estimate seconds |", + "| --- | --- | --- | --- | --- | --- | --- | --- |", ]; - for (const id of ["focused", "full"] as const) { - const row = totals[id]; - const runs = OUTCOMES.reduce((sum, outcome) => sum + row[outcome], 0); + for (const id of opts.result.commands) { + const row = counts[id]; lines.push( - `| ${id} | ${runs} | ${row.pass} | ${row.failure} | ${row.crash} | ${row.overrun} | ${seconds(opts.result.estimatesMs[id] ?? 0)} |` + `| ${id} | ${row?.iterations ?? 0} | ${row?.passes ?? 0} | ${row?.crashes ?? 0} | ${row?.failures ?? 0} | ${row?.overruns ?? 0} | ${row?.skippedForBudget ?? 0} | ${seconds(opts.result.estimatesMs[id] ?? 0)} |` ); } lines.push( @@ -271,6 +454,11 @@ export function renderLoopSummary(opts: { `| ${record.index} | ${record.command} | ${record.outcome} | ${record.reason} | ${record.exitCode ?? "-"} | ${record.signal ?? "-"} | ${seconds(record.durationMs)} |` ); } + for (const skip of opts.result.skippedForBudget) { + lines.push( + `| - | ${skip.command} | skipped | budget | - | - | ${seconds(skip.estimateMs)} estimated, ${seconds(skip.remainingMs)} left |` + ); + } lines.push( "", "Logs and crash reports are in the run artifact. Crash-report links are never opened.", @@ -366,6 +554,8 @@ export function executeWithLog(opts: { export interface LoopArguments { readonly budgetSeconds: number; readonly outDir: string; + readonly mode: LoopMode; + readonly role: LoopRole | null; readonly expectBunVersion: string | null; readonly sourceRevision: string | null; readonly focusedEstimateSeconds: number; @@ -383,28 +573,49 @@ function parseSeconds(name: string, value: string, min: number, max: number) { return parsed; } -export function parseLoopArguments(argv: readonly string[]): LoopArguments { +function parseClosed( + name: string, + value: string, + allowed: readonly T[] +): T { + const match = allowed.find((candidate) => candidate === value); + if (match === undefined) { + throw new LoopError(`Invalid ${name}`); + } + return match; +} + +const KNOWN_FLAGS: ReadonlySet = new Set([ + "budget-seconds", + "out-dir", + "mode", + "role", + "expect-bun-version", + "source-revision", + "focused-estimate-seconds", + "full-estimate-seconds", +]); + +/** `--name=value` flags; unknown, malformed or repeated flags are rejected. */ +function readFlags(argv: readonly string[]): ReadonlyMap { const values = new Map(); for (const arg of argv) { const match = /^--([a-z-]+)=(.*)$/.exec(arg); - if (!match?.[1] || match[2] === undefined || values.has(match[1])) { + if ( + !match?.[1] || + match[2] === undefined || + values.has(match[1]) || + !KNOWN_FLAGS.has(match[1]) + ) { throw new LoopError("Invalid argument"); } values.set(match[1], match[2]); } - const known = new Set([ - "budget-seconds", - "out-dir", - "expect-bun-version", - "source-revision", - "focused-estimate-seconds", - "full-estimate-seconds", - ]); - for (const key of values.keys()) { - if (!known.has(key)) { - throw new LoopError("Invalid argument"); - } - } + return values; +} + +export function parseLoopArguments(argv: readonly string[]): LoopArguments { + const values = readFlags(argv); const budget = values.get("budget-seconds"); if (budget === undefined) { throw new LoopError("Missing --budget-seconds"); @@ -421,6 +632,7 @@ export function parseLoopArguments(argv: readonly string[]): LoopArguments { if (outDir !== undefined && outDir.length === 0) { throw new LoopError("Invalid --out-dir"); } + const role = values.get("role"); return { budgetSeconds: parseSeconds( "--budget-seconds", @@ -429,6 +641,12 @@ export function parseLoopArguments(argv: readonly string[]): LoopArguments { MAX_BUDGET_SECONDS ), outDir: resolve(outDir ?? join(tmpdir(), "bun-crash-loop")), + mode: parseClosed( + "--mode", + values.get("mode") ?? DEFAULT_LOOP_MODE, + LOOP_MODES + ), + role: role === undefined ? null : parseClosed("--role", role, LOOP_ROLES), expectBunVersion: expect ?? null, sourceRevision: revision ?? null, focusedEstimateSeconds: parseSeconds( @@ -461,6 +679,7 @@ async function main(): Promise { budgetMs: args.budgetSeconds * 1000, commands: loopCommands({ bunExecutable: process.execPath, + mode: args.mode, focusedEstimateSeconds: args.focusedEstimateSeconds, fullEstimateSeconds: args.fullEstimateSeconds, }), @@ -471,37 +690,37 @@ async function main(): Promise { `iteration=${record.index} command=${record.command} outcome=${record.outcome} reason=${record.reason} seconds=${seconds(record.durationMs)}\n` ); }, + onSkip: (skip) => { + process.stdout.write( + `skipped command=${skip.command} reason=${skip.reason} estimate_seconds=${seconds(skip.estimateMs)} remaining_seconds=${seconds(skip.remainingMs)}\n` + ); + }, }); const diagnosticReports = await collectDiagnosticReports({ source: join(homedir(), "Library", "Logs", "DiagnosticReports"), destination: join(args.outDir, "diagnostic-reports"), sinceMs: loopStartMs, }); - const summary = renderLoopSummary({ + const report = { result, + mode: args.mode, + role: args.role, bunVersion, sourceRevision: args.sourceRevision, diagnosticReports, - }); + }; await Bun.write( join(args.outDir, "summary.json"), - `${JSON.stringify( - { - bunVersion, - sourceRevision: args.sourceRevision, - diagnosticReports, - ...result, - }, - null, - 2 - )}\n` + `${JSON.stringify(buildLoopSummary(report), null, 2)}\n` ); + const summary = renderLoopSummary(report); const stepSummary = process.env.GITHUB_STEP_SUMMARY; if (stepSummary) { await appendFile(stepSummary, summary); } else { process.stdout.write(summary); } + process.stdout.write(`verdict: ${renderVerdict(report)}\n`); if (result.iterations.length === 0) { throw new LoopError("No iteration fits the budget"); } diff --git a/scripts/lib/bun-crash-classify.ts b/scripts/lib/bun-crash-classify.ts index 443176811..0e4134fac 100644 --- a/scripts/lib/bun-crash-classify.ts +++ b/scripts/lib/bun-crash-classify.ts @@ -9,8 +9,11 @@ * crash-report links or values) is ever returned. */ -/** Exit statuses a shell reports for Bun's crash signals: SIGTRAP (133) and SIGSEGV (139). */ -export const BUN_CRASH_EXIT_CODES: readonly number[] = [133, 139]; +/** + * Exit statuses a shell reports for Bun's crash signals: SIGTRAP (133), SIGABRT + * (134), SIGBUS (138) and SIGSEGV (139). + */ +export const BUN_CRASH_EXIT_CODES: readonly number[] = [133, 134, 138, 139]; export type BunRunOutcome = "crash" | "failure" | "pass"; @@ -19,7 +22,7 @@ export type BunRunReason = | "banner" /** The process was ended by a signal it did not get from the harness. */ | "signal" - /** The exit status encodes a crash signal (133 or 139). */ + /** The exit status encodes a crash signal (133, 134, 138 or 139). */ | "signal_exit" /** Exit 0 and no crash banner. */ | "exit_zero" diff --git a/tests/bun-crash-loop.test.ts b/tests/bun-crash-loop.test.ts index e7a3ff683..ba3ef6ad3 100644 --- a/tests/bun-crash-loop.test.ts +++ b/tests/bun-crash-loop.test.ts @@ -3,14 +3,18 @@ import { mkdtemp, readdir, rm, utimes, writeFile } from "node:fs/promises"; import { homedir, tmpdir } from "node:os"; import { join } from "node:path"; import { + buildLoopSummary, collectDiagnosticReports, + countIterations, type ExecutedRun, executeWithLog, type LoopCommand, loopCommands, + nextEstimateMs, parseLoopArguments, - planNextCommand, + planNextIteration, renderLoopSummary, + renderVerdict, runBunCrashLoop, } from "../scripts/bun-crash-loop.ts"; import { @@ -51,8 +55,8 @@ describe("classifyBunRun", () => { ).toEqual({ outcome: "crash", reason: "banner" }); }); - test("exit 133 or 139 without a banner is a signal crash", () => { - for (const exitCode of [133, 139]) { + test("a trap, abort, bus-error or segfault exit status without a banner is a signal crash", () => { + for (const exitCode of [133, 134, 138, 139]) { expect(classifyBunRun({ exitCode, signal: null, output: "" })).toEqual({ outcome: "crash", reason: "signal_exit", @@ -148,78 +152,188 @@ const commands: readonly LoopCommand[] = [ { id: "full", argv: ["full"], initialEstimateMs: 300_000 }, ]; -describe("planNextCommand deadline arithmetic", () => { +const PASS: ExecutedRun = { + exitCode: 0, + signal: null, + output: " 3 pass\n 0 fail\n", + stoppedAtDeadline: false, +}; + +/** + * An injected executor on an injected clock. A run that would end past the + * deadline is stopped at the deadline, as the real executor does, so starting an + * iteration that cannot fit shows up as an overrun. + */ +function simulatedExecutor(opts: { + readonly clock: { now: number }; + readonly durationMs: (command: LoopCommand) => number; + readonly run?: (command: LoopCommand, index: number) => ExecutedRun; + readonly starts?: { id: string; at: number; argv: readonly string[] }[]; +}) { + return async ( + command: LoopCommand, + context: { readonly index: number; readonly deadlineMs: number } + ): Promise => { + opts.starts?.push({ + id: command.id, + at: opts.clock.now, + argv: command.argv, + }); + const end = opts.clock.now + opts.durationMs(command); + if (end > context.deadlineMs) { + opts.clock.now = context.deadlineMs; + return { + exitCode: null, + signal: "SIGTERM", + output: "", + stoppedAtDeadline: true, + }; + } + opts.clock.now = end; + return opts.run?.(command, context.index) ?? PASS; + }; +} + +describe("planNextIteration deadline arithmetic", () => { const estimatesMs = { focused: 100_000, full: 300_000 }; test("a command that ends exactly at the deadline may start", () => { expect( - planNextCommand({ + planNextIteration({ nowMs: 700_000, deadlineMs: 1_000_000, - commands, + active: commands, cursor: 1, estimatesMs, - })?.command.id - ).toBe("full"); - }); - - test("a command that would end past the deadline falls back to one that fits", () => { - const next = planNextCommand({ - nowMs: 700_001, - deadlineMs: 1_000_000, - commands, - cursor: 1, - estimatesMs, - }); - expect(next?.command.id).toBe("focused"); - expect(next?.cursor).toBe(2); + }) + ).toMatchObject({ action: "run", position: 1, command: { id: "full" } }); }); - test("nothing starts when no command fits or the deadline has passed", () => { + test("a command that would end past the deadline is skipped, not started", () => { expect( - planNextCommand({ - nowMs: 900_001, + planNextIteration({ + nowMs: 700_001, deadlineMs: 1_000_000, - commands, - cursor: 0, + active: commands, + cursor: 1, estimatesMs, }) - ).toBeNull(); + ).toMatchObject({ action: "skip", position: 1, command: { id: "full" } }); + }); + + test("nothing starts once the deadline is reached, and an empty rotation ends", () => { expect( - planNextCommand({ + planNextIteration({ nowMs: 1_000_000, deadlineMs: 1_000_000, - commands, + active: commands, cursor: 0, estimatesMs: { focused: 0, full: 0 }, + })?.action + ).toBe("skip"); + expect( + planNextIteration({ + nowMs: 0, + deadlineMs: 1_000_000, + active: [], + cursor: 0, + estimatesMs, }) ).toBeNull(); }); + + test("the estimate is the prior until a run completes, then the longest run with headroom", () => { + expect( + nextEstimateMs({ priorMs: 2_400_000, longestCompletedMs: null }) + ).toBe(2_400_000); + expect( + nextEstimateMs({ priorMs: 120_000, longestCompletedMs: 16_500 }) + ).toBe(19_800); + }); }); describe("runBunCrashLoop", () => { - test("no iteration starts past the budget and estimates grow to observed durations", async () => { - let clock = 0; - const durations = { focused: 90_000, full: 500_000 }; - const starts: { id: string; at: number }[] = []; + test("a 30-minute alternating budget never starts the 35-minute suite and records the skip", async () => { + // The hosted shape: focused ~16.5 s, full suite ~35 min, 30-minute budget. + const clock = { now: 0 }; + const starts: { id: string; at: number; argv: readonly string[] }[] = []; + const budgetMs = 1_800_000; + const result = await runBunCrashLoop({ + budgetMs, + commands: loopCommands({ + bunExecutable: "/bin/bun", + mode: "alternate", + focusedEstimateSeconds: 120, + fullEstimateSeconds: 2400, + }), + now: () => clock.now, + execute: simulatedExecutor({ + clock, + starts, + durationMs: (command) => (command.id === "full" ? 2_100_000 : 16_500), + }), + }); + expect(starts.some((start) => start.id === "full")).toBe(false); + expect(starts).toHaveLength(108); + for (const start of starts) { + expect(start.at + 16_500).toBeLessThanOrEqual(budgetMs); + } + expect(result.iterations.every((record) => record.outcome === "pass")).toBe( + true + ); + expect(result.skippedForBudget).toEqual([ + { + command: "full", + reason: "budget", + atMs: 16_500, + remainingMs: 1_783_500, + estimateMs: 2_400_000, + }, + { + command: "focused", + reason: "budget", + atMs: 1_782_000, + remainingMs: 18_000, + estimateMs: 19_800, + }, + ]); + expect(countIterations(result)).toEqual({ + focused: { + iterations: 108, + passes: 108, + crashes: 0, + failures: 0, + overruns: 0, + skippedForBudget: 1, + }, + full: { + iterations: 0, + passes: 0, + crashes: 0, + failures: 0, + overruns: 0, + skippedForBudget: 1, + }, + }); + }); + + test("estimates follow measured durations and no start overruns the deadline", async () => { + const clock = { now: 0 }; + const starts: { id: string; at: number; argv: readonly string[] }[] = []; const result = await runBunCrashLoop({ budgetMs: 1_000_000, commands, - now: () => clock, - execute: async (command) => { - starts.push({ id: command.id, at: clock }); - clock += durations[command.id]; - return { - exitCode: 0, - signal: null, - output: "", - stoppedAtDeadline: false, - }; - }, + now: () => clock.now, + execute: simulatedExecutor({ + clock, + starts, + durationMs: (command) => (command.id === "full" ? 500_000 : 90_000), + }), }); - // After the full suite takes 500 s, its estimate is 500 s and it no longer - // fits; focused runs (estimate 100 s) continue until 950 s + 100 s > 1000 s. - expect(starts).toEqual([ + // The full suite fits its 300 s prior once; measured at 500 s (600 s with + // headroom) it no longer fits at 680 s and leaves the rotation. Focused runs + // (108 s estimate after the first) continue until 950 s + 108 s > 1000 s. + expect(starts.map(({ id, at }) => ({ id, at }))).toEqual([ { id: "focused", at: 0 }, { id: "full", at: 90_000 }, { id: "focused", at: 590_000 }, @@ -227,16 +341,141 @@ describe("runBunCrashLoop", () => { { id: "focused", at: 770_000 }, { id: "focused", at: 860_000 }, ]); - for (const start of starts) { - const estimate = start.id === "full" ? 300_000 : 100_000; - expect(start.at + estimate).toBeLessThanOrEqual(1_000_000); - } - expect(result.estimatesMs).toEqual({ focused: 100_000, full: 500_000 }); + expect(result.estimatesMs).toEqual({ focused: 108_000, full: 600_000 }); + expect( + result.skippedForBudget.map(({ command, atMs }) => [command, atMs]) + ).toEqual([ + ["full", 680_000], + ["focused", 950_000], + ]); expect(result.iterations.map((record) => record.outcome)).toEqual( new Array(6).fill("pass") ); }); + test("focused mode only ever runs the owner test file", async () => { + const clock = { now: 0 }; + const starts: { id: string; at: number; argv: readonly string[] }[] = []; + const result = await runBunCrashLoop({ + budgetMs: 600_000, + commands: loopCommands({ + bunExecutable: "/bin/bun", + mode: "focused", + focusedEstimateSeconds: 120, + fullEstimateSeconds: 2400, + }), + now: () => clock.now, + execute: simulatedExecutor({ clock, starts, durationMs: () => 20_000 }), + }); + expect(starts.length).toBeGreaterThan(20); + for (const start of starts) { + expect(start.argv).toEqual([ + "/bin/bun", + "test", + "tests/native-https-owner.test.ts", + ]); + } + expect(result.commands).toEqual(["focused"]); + expect(Object.keys(countIterations(result))).toEqual(["focused"]); + expect(result.skippedForBudget.map((skip) => skip.command)).toEqual([ + "focused", + ]); + }); + + test("full mode with a budget below the prior starts nothing and says so", async () => { + const clock = { now: 0 }; + const starts: { id: string; at: number; argv: readonly string[] }[] = []; + const result = await runBunCrashLoop({ + budgetMs: 1_800_000, + commands: loopCommands({ + bunExecutable: "/bin/bun", + mode: "full", + focusedEstimateSeconds: 120, + fullEstimateSeconds: 2400, + }), + now: () => clock.now, + execute: simulatedExecutor({ clock, starts, durationMs: () => 1 }), + }); + expect(starts).toEqual([]); + expect(result.skippedForBudget).toHaveLength(1); + expect( + renderVerdict({ result, role: "candidate", bunVersion: "1.4.2" }) + ).toBe("candidate 0/0 full crashes; no iteration fit the budget"); + }); + + test("injected crashes count as crashes, other non-zero exits as other failures", async () => { + const clock = { now: 0 }; + const injected: ExecutedRun[] = [ + PASS, + // Bun's own banner, whatever the exit status. + { + exitCode: 0, + signal: null, + output: CRASH_TRANSCRIPT, + stoppedAtDeadline: false, + }, + // Signal deaths as Bun.spawn reports them; no real process gets a signal. + { + exitCode: null, + signal: "SIGSEGV", + output: "", + stoppedAtDeadline: false, + }, + { + exitCode: null, + signal: "SIGBUS", + output: "", + stoppedAtDeadline: false, + }, + // Shell-style abort exit status without a banner. + { exitCode: 134, signal: null, output: "", stoppedAtDeadline: false }, + { + exitCode: 1, + signal: null, + output: "(fail) owner starts [5.00ms]\n 3 pass\n 1 fail\n", + stoppedAtDeadline: false, + }, + { exitCode: 2, signal: null, output: "", stoppedAtDeadline: false }, + ]; + const result = await runBunCrashLoop({ + budgetMs: 200_000, + commands: [commands[0] as LoopCommand], + now: () => clock.now, + execute: simulatedExecutor({ + clock, + durationMs: () => 10_000, + run: (_command, index) => injected[index - 1] ?? PASS, + }), + }); + expect( + result.iterations + .slice(0, 7) + .map((record) => [record.outcome, record.reason]) + ).toEqual([ + ["pass", "exit_zero"], + ["crash", "banner"], + ["crash", "signal"], + ["crash", "signal"], + ["crash", "signal_exit"], + ["failure", "test_failure"], + ["failure", "exit_code"], + ]); + // 10 s runs, 12 s estimate after the first: starts at 0..180 s, then 190 s + 12 s > 200 s. + expect(countIterations(result)).toEqual({ + focused: { + iterations: 19, + passes: 13, + crashes: 4, + failures: 2, + overruns: 0, + skippedForBudget: 1, + }, + }); + expect( + renderVerdict({ result, role: "baseline", bunVersion: "1.3.9" }) + ).toBe("baseline 4/19 focused crashes"); + }); + test("a run stopped at the deadline is an overrun unless it printed the crash banner", async () => { let clock = 0; const runs: ExecutedRun[] = [ @@ -254,21 +493,15 @@ describe("runBunCrashLoop", () => { }, ]; const result = await runBunCrashLoop({ - budgetMs: 1_000_000, + budgetMs: 250_000, commands: [commands[0] as LoopCommand], now: () => clock, execute: async () => { clock += 100_000; - return ( - runs.shift() ?? { - exitCode: 0, - signal: null, - output: "", - stoppedAtDeadline: false, - } - ); + return runs.shift() ?? PASS; }, }); + expect(result.iterations).toHaveLength(2); expect(result.iterations[0]).toMatchObject({ outcome: "overrun", reason: "deadline", @@ -278,6 +511,12 @@ describe("runBunCrashLoop", () => { outcome: "crash", reason: "banner", }); + // A stopped run is not a measured duration, so the prior stays in force. + expect(result.estimatesMs.focused).toBe(100_000); + expect(countIterations(result).focused?.overruns).toBe(1); + expect(renderVerdict({ result, role: null, bunVersion: "1.3.9" })).toBe( + "Bun 1.3.9 1/2 focused crashes; 1 overrun" + ); }); test("a signal death from the executor is recorded as a crash with its closed signal name", async () => { @@ -303,19 +542,12 @@ describe("runBunCrashLoop", () => { }, ]; const result = await runBunCrashLoop({ - budgetMs: 300_000, + budgetMs: 400_000, commands: [commands[0] as LoopCommand], now: () => clock, execute: async () => { clock += 100_000; - return ( - runs.shift() ?? { - exitCode: 0, - signal: null, - output: "", - stoppedAtDeadline: false, - } - ); + return runs.shift() ?? PASS; }, }); expect( @@ -332,7 +564,7 @@ describe("runBunCrashLoop", () => { ]); }); - test("the summary is closed and value-free", async () => { + test("the summaries are closed and value-free", async () => { let clock = 0; const result = await runBunCrashLoop({ budgetMs: 250_000, @@ -348,17 +580,37 @@ describe("runBunCrashLoop", () => { }; }, }); - const summary = renderLoopSummary({ + const report = { result, + mode: "alternate" as const, + role: null, bunVersion: "1.3.9", sourceRevision: "2b8057c1ae6f4311150b6e139a9a919bbc7788d4", diagnosticReports: 1, - }); - expect(summary).toContain("| focused | 2 | 0 | 0 | 2 | 0 |"); + }; + const summary = renderLoopSummary(report); + expect(summary).toContain( + "Verdict: Bun 1.3.9 2/2 focused crashes, 0/0 full crashes" + ); + expect(summary).toContain("| focused | 2 | 0 | 2 | 0 | 0 | 1 | 120.0 |"); + expect(summary).toContain("| full | 0 | 0 | 0 | 0 | 0 | 1 | 300.0 |"); expect(summary).toContain("| 1 | focused | crash | banner | 133 | - |"); - expect(summary).not.toContain("do-not-print"); - expect(summary).not.toContain("bun.report"); - expect(summary).not.toContain("native-https-owner"); + expect(summary).toContain("| - | full | skipped | budget |"); + const json = JSON.stringify(buildLoopSummary(report)); + expect(JSON.parse(json)).toMatchObject({ + verdict: "Bun 1.3.9 2/2 focused crashes, 0/0 full crashes", + mode: "alternate", + role: null, + counts: { + focused: { iterations: 2, crashes: 2, skippedForBudget: 1 }, + full: { iterations: 0, skippedForBudget: 1 }, + }, + }); + for (const text of [summary, json]) { + expect(text).not.toContain("do-not-print"); + expect(text).not.toContain("bun.report"); + expect(text).not.toContain("native-https-owner"); + } }); }); @@ -434,21 +686,27 @@ describe("executeWithLog", () => { }); }); -test("loop commands run the focused owner test and then the full suite", () => { - expect( +test("each mode selects its commands, and focused is the owner test file alone", () => { + const ids = (mode: "focused" | "full" | "alternate") => loopCommands({ bunExecutable: "/bin/bun", + mode, focusedEstimateSeconds: 60, - fullEstimateSeconds: 600, - }).map((command) => [command.id, command.argv, command.initialEstimateMs]) - ).toEqual([ - [ - "focused", - ["/bin/bun", "test", "tests/native-https-owner.test.ts"], - 60_000, - ], - ["full", ["/bin/bun", "test"], 600_000], - ]); + fullEstimateSeconds: 2400, + }).map((command): unknown[] => [ + command.id, + command.argv, + command.initialEstimateMs, + ]); + const focused = [ + "focused", + ["/bin/bun", "test", "tests/native-https-owner.test.ts"], + 60_000, + ]; + const full = ["full", ["/bin/bun", "test"], 2_400_000]; + expect(ids("focused")).toEqual([focused]); + expect(ids("full")).toEqual([full]); + expect(ids("alternate")).toEqual([focused, full]); }); test("loop arguments are validated and closed", () => { @@ -462,10 +720,19 @@ test("loop arguments are validated and closed", () => { ).toMatchObject({ budgetSeconds: 1800, outDir: "/tmp/x", + mode: "focused", + role: null, expectBunVersion: "1.3.9", focusedEstimateSeconds: 120, - fullEstimateSeconds: 600, + fullEstimateSeconds: 2400, }); + expect( + parseLoopArguments([ + "--budget-seconds=1800", + "--mode=alternate", + "--role=baseline", + ]) + ).toMatchObject({ mode: "alternate", role: "baseline" }); for (const argv of [ [], ["--budget-seconds=10"], @@ -474,6 +741,10 @@ test("loop arguments are validated and closed", () => { ["--budget-seconds=1800", "--unknown=1"], ["--budget-seconds=1800", "--expect-bun-version=latest"], ["--budget-seconds=1800", "--source-revision=main"], + ["--budget-seconds=1800", "--mode=both"], + ["--budget-seconds=1800", "--mode="], + ["--budget-seconds=1800", "--role=dispatch"], + ["--budget-seconds=1800", "--role="], ]) { expect(() => parseLoopArguments(argv)).toThrow(); }