diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index d660551a..a06a2f79 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -1080,8 +1080,33 @@ export async function main( let exitCode = 0; let frameworkExit: number | undefined; let frameworkOutput = ""; + let renderedScanFailure: string | undefined; let renderedHistory: string | undefined; let renderedPublication: string | undefined; + const startedAt = performance.now(); + const scanFailure = ( + command: "scan" | "scans rerun", + format: string, + code: "SCAN_FAILED" | "SCAN_REPLAY_UNAVAILABLE", + message: string, + ) => { + const error = { code, message }; + if (format === "json" || format === "jsonl") { + // Keep failures complete and exclude framework invocation metadata. + const payload = argv.includes("--full-output") + ? { + ok: false, + error, + meta: { + command, + duration: `${Math.round(performance.now() - startedAt)}ms`, + }, + } + : error; + renderedScanFailure = `${JSON.stringify(payload, null, format === "json" ? 2 : undefined)}\n`; + } + return { ...error, exitCode }; + }; const history = async ( args: readonly string[], select: (value: JsonObject) => JsonObject | Promise = (value) => @@ -1406,36 +1431,39 @@ export async function main( .describe("Print scan diagnostics to stderr."), }), output: z.record(z.string(), z.unknown()).optional(), - async run({ args, error: incurError, options }) { - const scanId = args.scanId ?? (await latestScans())?.[0]?.scanId; - if (scanId === undefined) return; - let scanArguments: ScanArguments; + async run({ args, error: incurError, format, options }) { + let scanArguments: ScanArguments | undefined; try { - const { recipe } = await dependencies.runWorkbench([ - "get-scan-recipe", - "--scan-id", - scanId, - ]); - scanArguments = scanArgumentsFromRecipe(recipe, scanId); - scanArguments.verbose = options.verbose; + const scanId = args.scanId ?? (await latestScans())?.[0]?.scanId; + if (scanId !== undefined) { + const { recipe } = await dependencies.runWorkbench([ + "get-scan-recipe", + "--scan-id", + scanId, + ]); + scanArguments = scanArgumentsFromRecipe(recipe, scanId); + scanArguments.verbose = options.verbose; + } } catch (error) { - const message = errorMessage(error); - errorOutput.write(`codex-security: ${message}\n`); + errorOutput.write(`codex-security: ${errorMessage(error)}\n`); + } + if (scanArguments === undefined) { exitCode = 2; - return incurError({ - code: "SCAN_REPLAY_UNAVAILABLE", - message, - exitCode, - }); + return incurError( + scanFailure( + "scans rerun", + format, + "SCAN_REPLAY_UNAVAILABLE", + "The saved scan could not be replayed.", + ), + ); } const outcome = await runScan(scanArguments, errorOutput, dependencies); exitCode = outcome.exitCode; if (outcome.error !== undefined) { - return incurError({ - code: "SCAN_FAILED", - message: outcome.error, - exitCode, - }); + return incurError( + scanFailure("scans rerun", format, "SCAN_FAILED", outcome.error), + ); } return outcome.data; }, @@ -2065,11 +2093,9 @@ export async function main( ); exitCode = outcome.exitCode; if (outcome.error !== undefined) { - return incurError({ - code: "SCAN_FAILED", - message: outcome.error, - exitCode, - }); + return incurError( + scanFailure("scan", format, "SCAN_FAILED", outcome.error), + ); } if ( !options.dryRun && @@ -2665,18 +2691,23 @@ export async function main( updateController.abort(); } if (notice !== undefined) errorOutput.write(formatUpdateNotice(notice)); - if (frameworkExit !== undefined) { + if (frameworkExit !== undefined && renderedScanFailure === undefined) { if (exitCode !== 0) return exitCode; errorOutput.write( `codex-security: ${errorMessage(incurErrorMessage(frameworkOutput))}\n`, ); return 2; } - if (frameworkOutput.length === 0) return exitCode; + if (renderedScanFailure === undefined && frameworkOutput.length === 0) { + return exitCode; + } try { await writeCliOutput( output, - renderedPublication ?? renderedHistory ?? frameworkOutput, + renderedScanFailure ?? + renderedPublication ?? + renderedHistory ?? + frameworkOutput, ); return exitCode; } catch (error) { @@ -4028,15 +4059,12 @@ async function executeScan( estimated_usd: costLimitFailure?.cost.estimatedUsd, }); errorOutput.write(`${message}\n`); - if (failure instanceof ScanInterruptedError) { - return { exitCode: 2, error: message }; - } - if (scanDir !== null) { + if (!(failure instanceof ScanInterruptedError) && scanDir !== null) { errorOutput.write( `Partial output was kept at ${errorMessage(scanDir)}.\n`, ); } - return { exitCode: 2, error: message }; + return { exitCode: 2, error: structuredScanFailureMessage(failure) }; } if (preflight !== null) { const effectivePreflight: ScanPreflight = { @@ -4210,6 +4238,39 @@ function scanFailureMessage( } } +function structuredScanFailureMessage(error: unknown): string { + if (error instanceof OutputInsideProtectedRootError) { + return protectedRootErrorMessage(error, false); + } + if (error instanceof ScanCostLimitExceededError) { + return "The scan exceeded its configured cost limit."; + } + if (isLocalScanFailure(error)) { + return "The scan could not complete because a local input or filesystem operation failed."; + } + if ( + /flagged for possible cybersecurity risk|trusted access for cyber|cybersecurity policy/iu.test( + errorMessage(error), + ) + ) { + return "The scan was blocked by a cybersecurity policy. Trusted Access for Cyber may be required."; + } + switch (classifyConnectionFailure(error)) { + case "unauthorized": + return "Authentication failed. Check the selected credentials."; + case "forbidden": + return "The selected credentials cannot access the configured model."; + case "rate_limited": + return "The configured account reached its rate limit. Wait and retry."; + case "network_error": + return "The scan encountered a network or connection failure."; + case "timeout": + return "The scan timed out."; + case "unknown": + return "The scan failed. See stderr for details."; + } +} + function scanScope(arguments_: ScanArguments): string | null { if (arguments_.paths.length > 0) { const displayed = arguments_.paths.slice(0, 3).map((path) => { @@ -4340,6 +4401,7 @@ function formatTokenUsage(usage: unknown): string | null { function protectedRootErrorMessage( error: OutputInsideProtectedRootError, + includePaths = true, ): string { const description = error.pathKind === "output" @@ -4351,6 +4413,9 @@ function protectedRootErrorMessage( error.pathKind === "output" ? "Scan artifacts cannot be written inside the protected scan root." : "Temporary and runtime files cannot be created inside the protected scan root."; + if (!includePaths) { + return `${description} must be outside the scanned directory and any enclosing Git worktree. ${reason}`; + } const suggestion = suggestedOutputDirectory(error.protectedRoot); const recovery = error.pathKind === "output" diff --git a/sdk/typescript/src/knowledge-base.ts b/sdk/typescript/src/knowledge-base.ts index 064b98ef..3994efdf 100644 --- a/sdk/typescript/src/knowledge-base.ts +++ b/sdk/typescript/src/knowledge-base.ts @@ -11,6 +11,7 @@ import { import { tmpdir } from "node:os"; import { basename, extname, join, resolve } from "node:path"; import { unzipSync } from "fflate"; +import { ConfigurationError, errorMessage } from "./errors.js"; import { expandHome } from "./runtime.js"; const SUPPORTED_EXTENSIONS = new Set([ @@ -30,6 +31,18 @@ export interface PreparedKnowledgeBase { export async function prepareKnowledgeBase( paths: readonly string[], signal?: AbortSignal, +): Promise { + try { + return await stageKnowledgeBase(paths, signal); + } catch (error) { + if (signal?.aborted) throw error; + throw new ConfigurationError(errorMessage(error), { cause: error }); + } +} + +async function stageKnowledgeBase( + paths: readonly string[], + signal?: AbortSignal, ): Promise { const sources = new Set(); const documents = new Set(); diff --git a/sdk/typescript/src/runtime.ts b/sdk/typescript/src/runtime.ts index 13c41f2c..8349f878 100644 --- a/sdk/typescript/src/runtime.ts +++ b/sdk/typescript/src/runtime.ts @@ -32,6 +32,7 @@ import extractZip from "extract-zip"; import { parse } from "smol-toml"; import { CodexSecurityError, + ConfigurationError, OutputDirectoryError, PluginBootstrapError, PluginPythonUnavailableError, @@ -1813,7 +1814,12 @@ export async function extractPluginZip( } catch (error) { await rm(staging, { recursive: true, force: true }).catch(() => undefined); throwIfSignalAborted(signal); - if (error instanceof PluginBootstrapError) throw error; + if ( + error instanceof PluginBootstrapError || + error instanceof ConfigurationError + ) { + throw error; + } throw new PluginBootstrapError(`Invalid plugin ZIP: ${archivePath}`, { cause: error, }); @@ -1911,6 +1917,19 @@ export async function resolvePluginPath( pluginPath: string | undefined, workspace: string, signal?: AbortSignal, +): Promise { + try { + return await resolveLocalPluginPath(pluginPath, workspace, signal); + } catch (error) { + if (signal?.aborted || error instanceof ConfigurationError) throw error; + throw new ConfigurationError(errorMessage(error), { cause: error }); + } +} + +async function resolveLocalPluginPath( + pluginPath: string | undefined, + workspace: string, + signal?: AbortSignal, ): Promise { if (pluginPath === undefined) { return await bundledPluginRoot(); @@ -1938,6 +1957,19 @@ export async function createMarketplace( codexHome: string, pluginRoot: string, signal?: AbortSignal, +): Promise { + try { + return await stageMarketplace(codexHome, pluginRoot, signal); + } catch (error) { + if (signal?.aborted || error instanceof ConfigurationError) throw error; + throw new ConfigurationError(errorMessage(error), { cause: error }); + } +} + +async function stageMarketplace( + codexHome: string, + pluginRoot: string, + signal?: AbortSignal, ): Promise { throwIfSignalAborted(signal); const root = await realpath(pluginRoot); @@ -2049,7 +2081,7 @@ export async function bootstrapPlugin( throw error; }); if (existing !== null && !existing.isDirectory()) { - throw new PluginBootstrapError( + throw new ConfigurationError( `Codex Security plugin marketplace path must be a directory: ${marketplace}`, ); } @@ -2135,18 +2167,18 @@ export async function pluginMetadata( } manifest = JSON.parse(await readFile(manifestPath, "utf8")); } catch (error) { - throw new PluginBootstrapError(`Invalid Codex plugin directory: ${root}`, { + throw new ConfigurationError(`Invalid Codex plugin directory: ${root}`, { cause: error, }); } if (!isRecord(manifest) || manifest["name"] !== PLUGIN_NAME) { - throw new PluginBootstrapError( + throw new ConfigurationError( "Plugin manifest must have name 'codex-security'.", ); } const version = manifest["version"]; if (typeof version !== "string" || version.trim().length === 0) { - throw new PluginBootstrapError( + throw new ConfigurationError( "Plugin manifest must have a non-empty version.", ); } diff --git a/sdk/typescript/tests-ts/cli.test.ts b/sdk/typescript/tests-ts/cli.test.ts index b8401334..ce00c8d9 100644 --- a/sdk/typescript/tests-ts/cli.test.ts +++ b/sdk/typescript/tests-ts/cli.test.ts @@ -13,7 +13,8 @@ import { delimiter, join, normalize } from "node:path"; import { Writable } from "node:stream"; import { fileURLToPath } from "node:url"; import { stripVTControlCharacters } from "node:util"; -import { describe, expect, test } from "bun:test"; +import { describe, expect, spyOn, test } from "bun:test"; +import { Formatter } from "incur"; import { parse as parseToml } from "smol-toml"; import type { CodexSecurityConfig, @@ -24,10 +25,12 @@ import type { import { BUNDLED_PLUGIN_VERSION, CodexSecurityError, + ConfigurationError, DiffTarget, InvalidTargetError, OutputDirectoryError, OutputInsideProtectedRootError, + PluginBootstrapError, PluginPythonUnavailableError, ScanCostLimitExceededError, ScanInterruptedError, @@ -3405,6 +3408,9 @@ describe("CLI", () => { expect(stderr.text()).not.toContain("org-private"); expect(stderr.text()).not.toContain("SYNTHETIC_VERBOSE_KEY"); expect(stderr.text()).not.toContain("SYNTHETIC_PROVIDER_SECRET"); + expect(JSON.parse(stdout.text())).toMatchObject({ code: "SCAN_FAILED" }); + expect(stdout.text()).not.toContain("org-private"); + expect(stdout.text()).not.toContain("SYNTHETIC_PROVIDER_SECRET"); }); test("keeps unclassified provider context out of structured failure diagnostics", async () => { @@ -3443,7 +3449,12 @@ describe("CLI", () => { expect(stderr.text()).toContain("Provider failed for"); expect(stderr.text()).toContain("tenant-private"); expect(stderr.text()).toContain("req-internal"); - expect(stdout.text()).toBe(""); + expect(JSON.parse(stdout.text())).toEqual({ + code: "SCAN_FAILED", + message: "The scan failed. See stderr for details.", + }); + expect(stdout.text()).not.toContain("tenant-private"); + expect(stdout.text()).not.toContain("req-internal"); }); test("preserves provider identifier variants in scan failures", async () => { @@ -3536,10 +3547,14 @@ describe("CLI", () => { deps, ), ).toBe(2); - expect(stdout.text()).toBe(""); + expect(JSON.parse(stdout.text())).toEqual({ + code: "SCAN_FAILED", + message: "The scan failed. See stderr for details.", + }); expect(stderr.text()).toContain("Provider failed for"); for (const identifier of identifiers) { expect(stderr.text()).toContain(identifier); + expect(stdout.text()).not.toContain(identifier); } } } @@ -3756,7 +3771,12 @@ describe("CLI", () => { expect(stderr.text()).toContain("Cleanup failed for"); expect(stderr.text()).toContain("tenant-private"); expect(stderr.text()).toContain("req-internal"); - expect(stdout.text()).toBe(""); + expect(JSON.parse(stdout.text())).toEqual({ + code: "SCAN_FAILED", + message: "The scan failed. See stderr for details.", + }); + expect(stdout.text()).not.toContain("tenant-private"); + expect(stdout.text()).not.toContain("req-internal"); }); test("reports reconnect progress on stderr and keeps JSON output clean", async () => { @@ -3863,13 +3883,28 @@ describe("CLI", () => { }); test("surfaces underlying scanner errors instead of inventing a model outage", async () => { - for (const message of [ - "Could not save the Codex Security scan: UNIQUE constraint failed: scans.scan_dir", - "sandbox-exec: sandbox_apply: Operation not permitted during network setup.", - "network failure ECONNRESET while connecting to the model.", - "request timed out while reading the scanner response.", - "Local scan failed: project_directory=/tmp/project tenant_count=2 request_index=3.", - ]) { + for (const [message, safeMessage] of [ + [ + "Could not save the Codex Security scan: UNIQUE constraint failed: scans.scan_dir", + "The scan failed. See stderr for details.", + ], + [ + "sandbox-exec: sandbox_apply: Operation not permitted during network setup.", + "The scan encountered a network or connection failure.", + ], + [ + "network failure ECONNRESET while connecting to the model.", + "The scan encountered a network or connection failure.", + ], + [ + "request timed out while reading the scanner response.", + "The scan timed out.", + ], + [ + "Local scan failed: project_directory=/tmp/project tenant_count=2 request_index=3.", + "The scan failed. See stderr for details.", + ], + ] as const) { const stdout = capture(); const stderr = capture(); const deps = dependencies(); @@ -3884,7 +3919,11 @@ describe("CLI", () => { expect( await main(["scan", ".", "--json"], stdout.stream, stderr.stream, deps), ).toBe(2); - expect(stdout.text()).toBe(""); + expect(JSON.parse(stdout.text())).toEqual({ + code: "SCAN_FAILED", + message: safeMessage, + }); + expect(stdout.text()).not.toContain("/tmp/project"); expect(stderr.text()).toContain(`${message}\n`); expect(stderr.text()).not.toContain("codex-security:"); expect(stderr.text()).not.toContain("model service could not be reached"); @@ -3920,13 +3959,27 @@ describe("CLI", () => { ), ], [ - "path target naming a 403 directory", - new InvalidTargetError("Path target does not exist: src/403/client.ts"), + "path target naming policy and 403 directories", + new InvalidTargetError( + "Path target does not exist: src/cybersecurity policy/403/client.ts", + ), ], [ "git ref naming a forbidden branch", new InvalidTargetError("unknown Git ref: origin/forbidden-paths"), ], + [ + "knowledge-base parser failure", + new ConfigurationError( + "Cannot extract text from knowledge base PDF: /documents/network-security.pdf", + ), + ], + [ + "local plugin validation failure", + new ConfigurationError( + "Invalid Codex plugin directory: /plugins/network-security", + ), + ], [ "python interpreter unavailable", new PluginPythonUnavailableError( @@ -3949,12 +4002,17 @@ describe("CLI", () => { expect( await main( - ["scan", ".", "--verbose"], + ["scan", ".", "--verbose", "--json"], stdout.stream, stderr.stream, deps, ), ).toBe(2); + expect(JSON.parse(stdout.text())).toEqual({ + code: "SCAN_FAILED", + message: + "The scan could not complete because a local input or filesystem operation failed.", + }); expect(stderr.text()).toContain((failure as Error).message); expect(stderr.text()).toContain('scan.failed classification="local"'); expect(stderr.text()).not.toContain("cannot access the configured model"); @@ -3963,29 +4021,51 @@ describe("CLI", () => { } }); - test("keeps model authorization advice for genuine transport failures", async () => { + test("keeps model connection advice for genuine transport failures", async () => { // The bypass must not swallow real 401/403 handling, and the advice must // still replace upstream text that can name the organization or project. - for (const [detail, expected] of [ - ["401 invalid API key for org-private", "Authentication failed"], + for (const [failure, expected, safeMessage] of [ + [ + new CodexSecurityError("401 invalid API key for org-private"), + "Authentication failed", + "Authentication failed. Check the selected credentials.", + ], [ - "403 model access denied for org-private", + new CodexSecurityError("403 model access denied for org-private"), "cannot access the configured model", + "The selected credentials cannot access the configured model.", + ], + [ + new CodexSecurityError("429 too many requests for org-private"), + "reached its rate limit", + "The configured account reached its rate limit. Wait and retry.", + ], + [ + new PluginBootstrapError( + "Codex plugin bootstrap failed: network ECONNRESET", + ), + "network ECONNRESET", + "The scan encountered a network or connection failure.", ], ] as const) { + const stdout = capture(); const stderr = capture(); const deps = dependencies(); deps.createSecurity = () => ({ run: async () => { - throw new CodexSecurityError(detail); + throw failure; }, preflight: async () => fakePreflight(), close: async () => {}, }); expect( - await main(["scan", "."], capture().stream, stderr.stream, deps), + await main(["scan", ".", "--json"], stdout.stream, stderr.stream, deps), ).toBe(2); + expect(JSON.parse(stdout.text())).toEqual({ + code: "SCAN_FAILED", + message: safeMessage, + }); expect(stderr.text()).toContain(expected); expect(stderr.text()).not.toContain("org-private"); } @@ -4008,7 +4088,11 @@ describe("CLI", () => { expect( await main(["scan", ".", "--json"], stdout.stream, stderr.stream, deps), ).toBe(2); - expect(stdout.text()).toBe(""); + expect(JSON.parse(stdout.text())).toEqual({ + code: "SCAN_FAILED", + message: "The scan encountered a network or connection failure.", + }); + expect(stdout.text()).not.toContain("SYNTHETIC_KEY_123"); expect(stderr.text()).toContain( `network failure ECONNRESET ${SYNTHETIC_CREDENTIALS}`, ); @@ -4655,7 +4739,11 @@ describe("CLI", () => { }), ), ).toBe(2); - expect(stdout.text()).toBe(""); + expect(JSON.parse(stdout.text())).toEqual({ + code: "SCAN_FAILED", + message: "The scan exceeded its configured cost limit.", + }); + expect(stdout.text()).not.toContain("/tmp/scan"); expect(stderr.text()).toContain( "Scan stopped: estimated cost $0.00625 exceeded the $0.005 limit; partial output remains at /tmp/scan.", ); @@ -5007,6 +5095,235 @@ describe("CLI", () => { expect(stderr.text()).not.toContain("CodexSecurityError"); }); + test("emits structured policy failures for scans and reruns", async () => { + const policyMessage = + "This content was flagged for possible cybersecurity risk. Join Trusted Access for Cyber."; + for (const [arguments_, message, diagnostic] of [ + [["scan", ".", "--json"], policyMessage, policyMessage], + [ + ["scan", ".", "--format", "jsonl"], + "403 forbidden by cybersecurity policy for org-private", + "cannot access the configured model", + ], + [ + ["scans", "rerun", "scan-original", "--json"], + policyMessage, + policyMessage, + ], + ] as const) { + const stdout = capture(); + const stderr = capture(); + expect( + await main( + arguments_, + stdout.stream, + stderr.stream, + dependencies({ + onWorkbench: () => ({ + recipe: { + repository: "/original/repository", + target: { kind: "repository", paths: [] }, + mode: "standard", + config: {}, + }, + }), + onTurn: (_repository, options) => { + (options as ScanOptions).onOutputDirReady?.("/tmp/partial-scan"); + throw new CodexSecurityError(message); + }, + }), + ), + ).toBe(2); + expect(JSON.parse(stdout.text())).toEqual({ + code: "SCAN_FAILED", + message: + "The scan was blocked by a cybersecurity policy. Trusted Access for Cyber may be required.", + }); + expect(stderr.text()).toContain(diagnostic); + expect(stderr.text()).not.toContain("org-private"); + expect(stderr.text()).toContain( + "Partial output was kept at /tmp/partial-scan.", + ); + } + }); + + test("keeps unavailable saved scan failures structured and private", async () => { + const sensitiveFailure = + "Workbench failed for customer-private org-private ACCOUNT_ID=account-private request_id=req-private /private/customer/repository token=sk-proj-SYNTHETIC_REPLAY_SECRET_123"; + const cases = [ + { + arguments: ["scan-original", "--json"], + onWorkbench: () => ({}), + detail: "This scan does not have a saved launch recipe.", + }, + { + arguments: ["scan-original", "--format", "json"], + onWorkbench: () => ({ recipe: { repository: "" } }), + detail: "The saved scan recipe does not contain a repository.", + }, + { + arguments: ["scan-original", "--format", "jsonl"], + onWorkbench: () => { + throw new Error(sensitiveFailure); + }, + detail: sensitiveFailure, + }, + { + arguments: ["--json"], + onWorkbench: () => ({ scans: [] }), + detail: "No completed scans found for the current repository.", + }, + { + arguments: ["--format", "jsonl"], + onWorkbench: () => { + throw new Error(sensitiveFailure); + }, + detail: sensitiveFailure, + }, + { + arguments: [], + onWorkbench: () => ({ scans: [] }), + detail: "No completed scans found for the current repository.", + }, + ]; + + for (const scenario of cases) { + const stdout = capture(); + const stderr = capture(); + + expect( + await main( + ["scans", "rerun", ...scenario.arguments], + stdout.stream, + stderr.stream, + dependencies({ onWorkbench: scenario.onWorkbench }), + ), + ).toBe(2); + if (scenario.arguments.length === 0) { + expect(stdout.text()).toBe(""); + } else { + expect(JSON.parse(stdout.text())).toEqual({ + code: "SCAN_REPLAY_UNAVAILABLE", + message: "The saved scan could not be replayed.", + }); + } + expect(stderr.text()).toBe(`codex-security: ${scenario.detail}\n`); + } + }); + + test("keeps structured scan failure envelopes free of framework metadata", async () => { + const originalFormat = Formatter.format; + const marker = "SYNTHETIC_FRAMEWORK_METADATA"; + let decorated = 0; + const formatter = spyOn(Formatter, "format").mockImplementation( + (value, format) => { + if (typeof value !== "object" || value === null) { + return originalFormat(value, format); + } + const record = value as { + ok?: boolean; + code?: string; + error?: { code?: string }; + }; + const code = record.ok === false ? record.error?.code : record.code; + if (code !== "SCAN_FAILED" && code !== "SCAN_REPLAY_UNAVAILABLE") { + return originalFormat(value, format); + } + decorated += 1; + const cta = { description: marker, commands: [] }; + return originalFormat( + { + ...record, + ...(record.ok === false + ? { meta: { command: marker, duration: marker, cta } } + : { cta }), + frameworkOnly: marker, + }, + format, + ); + }, + ); + + try { + for (const [arguments_, command, unavailable] of [ + [["scan", "."], "scan", false], + [["scans", "rerun", "scan-original"], "scans rerun", false], + [["scans", "rerun", "scan-original"], "scans rerun", true], + ] as const) { + for (const format of ["json", "jsonl"]) { + for (const [outputOptions, fullOutput] of [ + [[], false], + [["--full-output"], true], + [["--token-count"], false], + [["--token-limit", "1"], false], + [["--full-output", "--token-offset", "1"], true], + ] as const) { + const stdout = capture(); + const stderr = capture(); + const expectedError = unavailable + ? { + code: "SCAN_REPLAY_UNAVAILABLE", + message: "The saved scan could not be replayed.", + } + : { + code: "SCAN_FAILED", + message: "The scan failed. See stderr for details.", + }; + const before = decorated; + expect( + await main( + [...arguments_, "--format", format, ...outputOptions], + stdout.stream, + stderr.stream, + dependencies({ + onWorkbench: (): JsonObject => + unavailable + ? {} + : { + recipe: { + repository: "/original/repository", + target: { kind: "repository", paths: [] }, + mode: "standard", + config: {}, + }, + }, + onRun: () => { + throw new CodexSecurityError("synthetic scan failure"); + }, + }), + ), + ).toBe(2); + expect(decorated).toBeGreaterThan(before); + expect(JSON.parse(stdout.text())).toEqual( + fullOutput + ? { + ok: false, + error: expectedError, + meta: { + command, + duration: expect.stringMatching(/^\d+ms$/u), + }, + } + : expectedError, + ); + expect(stdout.text()).not.toContain(marker); + expect(stdout.text().endsWith("\n")).toBe(true); + if (format === "jsonl") { + expect(stdout.text().trimEnd().split("\n")).toHaveLength(1); + } + expect(stderr.text()).toContain( + unavailable + ? "This scan does not have a saved launch recipe." + : "synthetic scan failure", + ); + } + } + } + } finally { + formatter.mockRestore(); + } + }); + test("does not emit a successful full-output envelope for a failed scan", async () => { const stdout = capture(); const stderr = capture(); @@ -5074,7 +5391,13 @@ describe("CLI", () => { failing, ), ).toBe(2); - expect(stdout.text()).toBe(""); + expect(JSON.parse(stdout.text())).toEqual({ + code: "SCAN_FAILED", + message: + "Scan output directory must be outside the scanned directory and any enclosing Git worktree. Scan artifacts cannot be written inside the protected scan root.", + }); + expect(stdout.text()).not.toContain(output); + expect(stdout.text()).not.toContain(worktree); expect(stderr.text()).toContain( "Scan output directory must be outside the scanned directory and any enclosing Git worktree.", ); @@ -5181,7 +5504,13 @@ describe("CLI", () => { failing, ), ).toBe(2); - expect(stdout.text()).toBe(""); + expect(JSON.parse(stdout.text())).toEqual({ + code: "SCAN_FAILED", + message: + "Scan output directory must be outside the scanned directory and any enclosing Git worktree. Scan artifacts cannot be written inside the protected scan root.", + }); + expect(stdout.text()).not.toContain("SYNTHETIC_ROOT_KEY"); + expect(stdout.text()).not.toContain("SYNTHETIC_OUTPUT_KEY"); expect(stderr.text()).toContain(`Resolved path: ${output}`); expect(stderr.text()).toContain(`Protected root: ${protectedRoot}`); }); @@ -5217,32 +5546,44 @@ describe("CLI", () => { test("preserves retained partial-output paths", async () => { const path = "/private/tmp/scan_sk-proj-SYNTHETIC_PATH_KEY_123/results"; - for (const [signal, expectedExit] of [ - [null, 2], - ["SIGINT", 130], - ["SIGTERM", 143], + for (const [signal, expectedExit, message] of [ + [null, 2, "The scan failed. See stderr for details."], + ["SIGINT", 130, "Scan canceled by Ctrl-C."], + ["SIGTERM", 143, "Scan terminated by SIGTERM."], ] as const) { - const signals = new FakeSignals(); - const stdout = capture(); - const stderr = capture(); - const deps = dependencies({ - signals, - onTurn: (_repository, options) => { - ( - options as { onOutputDirReady?: (scanDir: string) => void } - ).onOutputDirReady?.(path); - }, - onRun: () => { - if (signal !== null) signals.emit(signal); - throw new Error("runtime failed"); - }, - }); + for (const json of [false, true]) { + const signals = new FakeSignals(); + const stdout = capture(); + const stderr = capture(); + const deps = dependencies({ + signals, + onTurn: (_repository, options) => { + (options as ScanOptions).onOutputDirReady?.(path); + }, + onRun: () => { + if (signal !== null) signals.emit(signal); + throw new Error("runtime failed"); + }, + }); - expect( - await main(["scan", "."], stdout.stream, stderr.stream, deps), - ).toBe(expectedExit); - expect(stdout.text()).toBe(""); - expect(stderr.text()).toContain(`Partial output was kept at ${path}.`); + expect( + await main( + json ? ["scan", ".", "--json"] : ["scan", "."], + stdout.stream, + stderr.stream, + deps, + ), + ).toBe(expectedExit); + if (json) { + expect(JSON.parse(stdout.text())).toEqual({ + code: "SCAN_FAILED", + message, + }); + } else { + expect(stdout.text()).toBe(""); + } + expect(stderr.text()).toContain(`Partial output was kept at ${path}.`); + } } }, 30_000); @@ -5262,7 +5603,14 @@ describe("CLI", () => { }), ), ).toBe(2); - expect(stdout.text()).toBe(""); + if (json) { + expect(JSON.parse(stdout.text())).toEqual({ + code: "SCAN_FAILED", + message: "The scan failed. See stderr for details.", + }); + } else { + expect(stdout.text()).toBe(""); + } expect(stderr.text()).toContain("SYNTHETIC_AUTH_HOME_CLEANUP_FAILED"); expect(stderr.text()).toContain("Partial output was kept at /tmp/scan."); } diff --git a/sdk/typescript/tests-ts/knowledge-base.test.ts b/sdk/typescript/tests-ts/knowledge-base.test.ts index ffe54734..699aad18 100644 --- a/sdk/typescript/tests-ts/knowledge-base.test.ts +++ b/sdk/typescript/tests-ts/knowledge-base.test.ts @@ -16,6 +16,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, spyOn, test } from "bun:test"; import { strToU8, zipSync } from "fflate"; +import { ConfigurationError } from "../src/errors.js"; import { prepareKnowledgeBase } from "../src/knowledge-base.js"; import { expandHome } from "../src/runtime.js"; @@ -238,6 +239,28 @@ describe("scan knowledge bases", () => { expect(documents).toContain("SSRF & IDOR\n"); }); + test("preserves the local origin and cause of document parser failures", async () => { + const root = await temporaryDirectory(); + const source = join(root, "network-security.pdf"); + await writeFile(source, pdf("Network design")); + const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs"); + const cause = new Error("Synthetic parser failure."); + const parser = spyOn(pdfjs, "getDocument").mockImplementation(() => { + throw cause; + }); + + try { + const prepared = prepareKnowledgeBase([source]); + await expect(prepared).rejects.toBeInstanceOf(ConfigurationError); + await expect(prepared).rejects.toMatchObject({ + message: `Cannot extract text from knowledge base PDF: ${source}`, + cause: { cause }, + }); + } finally { + parser.mockRestore(); + } + }); + test("cleans up documents and rediscovers directory contents on later runs", async () => { const root = await temporaryDirectory(); const source = join(root, "scope.md"); diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index 16280e70..365eb13c 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -30,17 +30,19 @@ import { import { fileURLToPath, pathToFileURL } from "node:url"; import { promisify } from "node:util"; import { brotliDecompressSync } from "node:zlib"; -import { afterEach, describe, expect, mock, test } from "bun:test"; +import { afterEach, describe, expect, mock, spyOn, test } from "bun:test"; import { strToU8, zipSync } from "fflate"; import { BUNDLED_PLUGIN_VERSION, bootstrapPlugin, bundledPluginRoot, + ConfigurationError, createIsolatedHome, createMarketplace, extractPluginZip, importAmbientAuth, pluginExecutionEnvironment, + pluginMetadata, PluginBootstrapError, PluginPythonUnavailableError, prepareOutputDir, @@ -827,17 +829,46 @@ describe("plugin runtime preparation", () => { } }); + test("preserves the local origin of plugin selection and manifest failures", async () => { + const root = await temporaryDirectory(); + const workspace = join(root, "workspace"); + const source = await plugin(root); + await mkdir(workspace); + await expect( + resolvePluginPath(join(root, "network-plugin"), workspace), + ).rejects.toBeInstanceOf(ConfigurationError); + + const cause = new Error("Synthetic manifest read failure."); + const manifestRead = spyOn(fsPromises, "readFile").mockRejectedValue(cause); + try { + for (const operation of [ + () => pluginMetadata(source), + () => resolvePluginPath(source, workspace), + ]) { + const result = operation(); + await expect(result).rejects.toBeInstanceOf(ConfigurationError); + await expect(result).rejects.toMatchObject({ + message: `Invalid Codex plugin directory: ${source}`, + cause, + }); + } + } finally { + manifestRead.mockRestore(); + } + }); + test("honors cancellation while staging a configured plugin directory", async () => { const root = await temporaryDirectory(); const workspace = join(root, "bootstrap"); await mkdir(workspace); const source = await plugin(root); const controller = new AbortController(); - controller.abort(new DOMException("canceled", "AbortError")); + const reason = new DOMException("canceled", "AbortError"); + controller.abort(reason); await expect( resolvePluginPath(source, workspace, controller.signal), - ).rejects.toMatchObject({ name: "AbortError" }); + ).rejects.toBe(reason); expect(existsSync(join(workspace, "selected-plugin"))).toBe(false); }); @@ -866,6 +897,53 @@ describe("plugin runtime preparation", () => { ).toBeDefined(); }); + test("keeps local marketplace failures separate from installer failures", async () => { + const root = await temporaryDirectory(); + const selected = await plugin(root); + const home = join(root, "home"); + const marketplace = join(home, "sdk-marketplace"); + const installationFailure = new PluginBootstrapError( + "Codex plugin bootstrap failed: network ECONNRESET", + ); + let installerCalls = 0; + const options = { + codexCommand: { command: join(root, "codex") }, + environment: {}, + runCodex: async () => { + installerCalls += 1; + throw installationFailure; + }, + }; + + await mkdir(home); + await writeFile(marketplace, "local fixture"); + await expect( + bootstrapPlugin(home, selected, options), + ).rejects.toBeInstanceOf(ConfigurationError); + await rm(marketplace); + + const cause = new PluginBootstrapError( + "Plugin projection failed for a local network directory.", + ); + const copy = spyOn(fsPromises, "cp").mockRejectedValue(cause); + try { + const result = bootstrapPlugin(home, selected, options); + await expect(result).rejects.toBeInstanceOf(ConfigurationError); + await expect(result).rejects.toMatchObject({ + message: cause.message, + cause, + }); + } finally { + copy.mockRestore(); + } + expect(installerCalls).toBe(0); + + await expect(bootstrapPlugin(home, selected, options)).rejects.toBe( + installationFailure, + ); + expect(installerCalls).toBe(1); + }); + test("copies configured plugins with more than 4,096 entries", async () => { const root = await temporaryDirectory(); const source = await plugin(root); @@ -970,7 +1048,7 @@ describe("plugin runtime preparation", () => { await expect( createMarketplace(join(root, "home"), selected), - ).rejects.toThrow(PluginBootstrapError); + ).rejects.toThrow(ConfigurationError); expect(existsSync(destination)).toBe(false); expect(await readFile(outside, "utf8")).toBe("OUTSIDE_SECRET"); }, @@ -1004,7 +1082,7 @@ describe("plugin runtime preparation", () => { await expect( createMarketplace(join(root, "home"), selected), - ).rejects.toThrow(PluginBootstrapError); + ).rejects.toThrow(ConfigurationError); expect(existsSync(destination)).toBe(false); expect(await readFile(outside, "utf8")).toBe("OUTSIDE_SECRET"); }, @@ -1052,7 +1130,7 @@ describe("plugin runtime preparation", () => { try { await expect( createMarketplace(join(root, "home"), selected), - ).rejects.toThrow(PluginBootstrapError); + ).rejects.toThrow(ConfigurationError); expect(swapped).toBe(true); expect(existsSync(destination)).toBe(false); expect(await readFile(join(outsideScripts, "helper.py"), "utf8")).toBe( @@ -1089,7 +1167,7 @@ describe("plugin runtime preparation", () => { } await expect(resolvePluginPath(source, workspace)).rejects.toThrow( - PluginBootstrapError, + ConfigurationError, ); } }, @@ -1137,7 +1215,7 @@ describe("plugin runtime preparation", () => { ).toBe(false); }); - test("extracts a plugin in one top-level directory", async () => { + test("extracts a plugin in one top-level directory and preserves manifest failures", async () => { const root = await temporaryDirectory(); const archive = join(root, "plugin.zip"); await writeFile( @@ -1150,6 +1228,35 @@ describe("plugin runtime preparation", () => { ); const extracted = await extractPluginZip(archive, join(root, "extracted")); expect(extracted).toBe(join(root, "extracted", "release")); + + const cause = new Error("Synthetic manifest read failure."); + const originalReadFile = fsPromises.readFile; + const manifestRead = spyOn(fsPromises, "readFile").mockImplementation((( + ...args: Parameters + ) => { + if ( + args[1] === "utf8" && + String(args[0]).endsWith(join(".codex-plugin", "plugin.json")) + ) { + return Promise.reject(cause); + } + return Reflect.apply(originalReadFile, fsPromises, args); + }) as typeof originalReadFile); + try { + for (const operation of [ + () => extractPluginZip(archive, join(root, "failed-extract")), + () => resolvePluginPath(archive, join(root, "bootstrap")), + ]) { + const result = operation(); + await expect(result).rejects.toBeInstanceOf(ConfigurationError); + await expect(result).rejects.toMatchObject({ + message: expect.stringContaining("Invalid Codex plugin directory:"), + cause, + }); + } + } finally { + manifestRead.mockRestore(); + } }); test("decodes flag-clear ZIP filenames with the legacy CP437 encoding", async () => {