fix(ci): retry transient dependency audit failures

Author: chenanran555

package.json

@@ -18,7 +18,7 @@
 	],
 	"scripts": {
 		"prepare": "bash scripts/setup-githooks.sh",
-		"audit": "bun audit",
+		"audit": "bun scripts/audit.ts",
 		"lint": "biome check . --diagnostic-level=warn --error-on-warnings",
 		"lint:fix": "biome check --write .",
 		"lint:changed": "bash scripts/lint-changed.sh",

scripts/audit.test.ts

@@ -0,0 +1,115 @@
+import { describe, expect, test } from "bun:test";
+import { readFileSync } from "node:fs";
+import { type AuditResult, isTransientAuditFailure, runAudit } from "./audit.ts";
+import { profileSteps } from "./verify.ts";
+
+const connectionClosed: AuditResult = {
+	exitCode: 1,
+	stdout: "",
+	stderr: "bun audit v1.3.5 (1e86cebd)\nConnectionClosed: audit request failed\n",
+};
+const success: AuditResult = { exitCode: 0, stdout: "No vulnerabilities found\n", stderr: "" };
+
+function auditSequence(results: AuditResult[]) {
+	let attempts = 0;
+	const delays: number[] = [];
+	const stdout: string[] = [];
+	const stderr: string[] = [];
+	return {
+		get attempts() {
+			return attempts;
+		},
+		delays,
+		stdout,
+		stderr,
+		dependencies: {
+			execute: () => {
+				const result = results[attempts++];
+				if (!result) throw new Error("Unexpected audit attempt");
+				return result;
+			},
+			sleep: async (delayMs: number) => {
+				delays.push(delayMs);
+			},
+			writeStdout: (output: string) => stdout.push(output),
+			writeStderr: (output: string) => stderr.push(output),
+		},
+	};
+}
+
+describe("dependency audit retries", () => {
+	test("keeps successful audits unchanged without waiting", async () => {
+		const sequence = auditSequence([success]);
+		expect(await runAudit(sequence.dependencies)).toBe(0);
+		expect(sequence.attempts).toBe(1);
+		expect(sequence.delays).toEqual([]);
+		expect(sequence.stdout).toEqual([success.stdout]);
+	});
+
+	test("retries the CI ConnectionClosed error with bounded backoff", async () => {
+		const sequence = auditSequence([connectionClosed, connectionClosed, success]);
+		expect(await runAudit(sequence.dependencies)).toBe(0);
+		expect(sequence.attempts).toBe(3);
+		expect(sequence.delays).toEqual([2_000, 5_000]);
+		expect(sequence.stderr.join("")).toContain(connectionClosed.stderr);
+		expect(sequence.stderr.join("")).toContain("retrying (3/3)");
+	});
+
+	test("still fails after three unsuccessful network attempts", async () => {
+		const sequence = auditSequence([connectionClosed, connectionClosed, connectionClosed]);
+		expect(await runAudit(sequence.dependencies)).toBe(1);
+		expect(sequence.attempts).toBe(3);
+		expect(sequence.delays).toEqual([2_000, 5_000]);
+		expect(sequence.stderr.join("")).toContain("verification remains failed");
+	});
+
+	test("does not retry vulnerability findings, even after a transient error", async () => {
+		const finding = { exitCode: 1, stdout: "1 vulnerability (1 high)\n", stderr: "" };
+		const sequence = auditSequence([connectionClosed, finding]);
+		expect(await runAudit(sequence.dependencies)).toBe(1);
+		expect(sequence.attempts).toBe(2);
+		expect(sequence.delays).toEqual([2_000]);
+		expect(sequence.stdout).toContain(finding.stdout);
+	});
+
+	test("does not retry unknown errors, auth failures, invalid lockfiles, or interrupted processes", async () => {
+		for (const result of [
+			{ exitCode: 2, stdout: "", stderr: "error: invalid lockfile\n" },
+			{ exitCode: 1, stdout: "", stderr: "HTTP 401 Unauthorized\n" },
+			{ exitCode: 1, stdout: "", stderr: "unknown audit error\n" },
+			{ ...connectionClosed, errorCode: "ENOENT" },
+			{ ...connectionClosed, signal: "SIGINT" },
+		]) {
+			const sequence = auditSequence([result]);
+			expect(await runAudit(sequence.dependencies)).toBe(result.exitCode);
+			expect(sequence.attempts).toBe(1);
+			expect(sequence.delays).toEqual([]);
+		}
+	});
+
+	test("handles ANSI banners but does not match errors embedded in vulnerability reports", () => {
+		expect(isTransientAuditFailure({ ...connectionClosed, stderr: `\x1b[1m${connectionClosed.stderr}\x1b[0m` })).toBe(
+			true,
+		);
+		expect(isTransientAuditFailure({ ...connectionClosed, stdout: "1 vulnerability (1 high)\n" })).toBe(false);
+		expect(
+			isTransientAuditFailure({ ...connectionClosed, stderr: "high: ConnectionClosed: audit request failed\n" }),
+		).toBe(false);
+	});
+
+	test("bounds retries of timed-out audit processes", async () => {
+		const timeout = { exitCode: 1, stdout: "", stderr: "", errorCode: "ETIMEDOUT", signal: "SIGTERM" };
+		const sequence = auditSequence([timeout, timeout, timeout]);
+		expect(await runAudit(sequence.dependencies)).toBe(1);
+		expect(sequence.attempts).toBe(3);
+		expect(sequence.stderr.join("")).toContain("ETIMEDOUT");
+		expect(isTransientAuditFailure({ ...timeout, stdout: "1 vulnerability (1 high)\n" })).toBe(false);
+	});
+
+	test("routes both full and release verification through the audit wrapper", () => {
+		const manifest = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"));
+		expect(manifest.scripts.audit).toBe("bun scripts/audit.ts");
+		expect(profileSteps("full")).toContain("audit");
+		expect(profileSteps("release")).toContain("audit");
+	});
+});

scripts/audit.ts

@@ -0,0 +1,79 @@
+import { spawnSync } from "node:child_process";
+import { resolve } from "node:path";
+import { stripVTControlCharacters } from "node:util";
+
+export interface AuditResult {
+	exitCode: number;
+	stdout: string;
+	stderr: string;
+	errorCode?: string;
+	signal?: string;
+}
+
+const retryDelaysMs = [2_000, 5_000] as const;
+const root = resolve(import.meta.dirname, "..");
+
+export function isTransientAuditFailure(result: AuditResult): boolean {
+	if (result.exitCode === 0) return false;
+
+	// Only retry a standalone transport error, optionally preceded by Bun's banner.
+	// A vulnerability report or any unknown error must remain an immediate failure.
+	const diagnostics = stripVTControlCharacters(`${result.stdout}\n${result.stderr}`)
+		.split(/\r?\n/)
+		.map((line) => line.trim())
+		.filter((line) => line && !/^bun audit(?: v\d+\.\d+\.\d+(?:-[\w.-]+)?(?: \([a-f0-9]+\))?)?$/.test(line));
+	const transportError =
+		diagnostics.length === 1 &&
+		/^(?:ConnectionClosed|ConnectionReset|ConnectionRefused|Timeout): audit request failed$/.test(diagnostics[0]);
+	if (result.errorCode === "ETIMEDOUT") return diagnostics.length === 0 || transportError;
+	if (result.errorCode || result.signal) return false;
+	return transportError;
+}
+
+function executeAudit(): AuditResult {
+	const result = spawnSync(process.execPath, ["audit"], {
+		cwd: root,
+		encoding: "utf8",
+		stdio: ["ignore", "pipe", "pipe"],
+		timeout: 45_000,
+		maxBuffer: 16 * 1024 * 1024,
+	});
+	return {
+		exitCode: result.status ?? 1,
+		stdout: result.stdout ?? "",
+		stderr: result.stderr ?? "",
+		errorCode: (result.error as NodeJS.ErrnoException | undefined)?.code,
+		signal: result.signal ?? undefined,
+	};
+}
+
+export async function runAudit(
+	dependencies: {
+		execute?: () => AuditResult;
+		sleep?: (delayMs: number) => Promise<void>;
+		writeStdout?: (output: string) => void;
+		writeStderr?: (output: string) => void;
+	} = {},
+): Promise<number> {
+	const execute = dependencies.execute ?? executeAudit;
+	const sleep = dependencies.sleep ?? Bun.sleep;
+	const writeStdout = dependencies.writeStdout ?? ((output) => process.stdout.write(output));
+	const writeStderr = dependencies.writeStderr ?? ((output) => process.stderr.write(output));
+
+	for (let attempt = 0; ; attempt++) {
+		const result = execute();
+		writeStdout(result.stdout);
+		writeStderr(result.stderr);
+		if (result.errorCode) writeStderr(`Audit process failed (${result.errorCode}).\n`);
+		if (!isTransientAuditFailure(result)) return result.exitCode;
+		if (attempt === retryDelaysMs.length) {
+			writeStderr("Dependency audit could not complete after 3 attempts; verification remains failed.\n");
+			return result.exitCode;
+		}
+		const delayMs = retryDelaysMs[attempt];
+		writeStderr(`Audit request failed temporarily; retrying (${attempt + 2}/3) in ${delayMs / 1000}s.\n`);
+		await sleep(delayMs);
+	}
+}
+
+if (import.meta.main) process.exit(await runAudit());