diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index 459a38b5..6f44b805 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -261,11 +261,12 @@ overrides, without starting Codex or contacting the network. findings or failed scans. Set `--fail-on-severity` to change the threshold. `--path` scopes a scan to one or more paths, `--diff` scans committed changes, -and `--working-tree` scans staged and unstaged changes. Deep scans support -repository and path targets. The output directory must be outside the scanned -directory and any enclosing Git worktree. When SARIF is produced, it is written -to -`/exports/results.sarif`. +and `--working-tree` scans staged and unstaged changes. These target selectors +are mutually exclusive. `--head` requires `--diff`; `--base` requires +`--working-tree`. Deep scans support repository and path targets, and deep-scan +settings require `--mode deep`. The output directory must be outside the +scanned directory and any enclosing Git worktree. When SARIF is produced, it +is written to `/exports/results.sarif`. Working-tree snapshots include files from untracked nested Git repositories. Initialized submodules must be clean and checked out at the commit recorded by @@ -666,9 +667,10 @@ const directPublication = await publishScan("/path/to/completed-scan", { ### Scan history and reruns `scans` or `scans list` lists scans for the current repository. Pass a repository -path to inspect another checkout, or `--scan-root DIR` to list scans whose -artifacts are under a particular root. `scans show` opens the latest completed -scan for the current repository. Pass `SCAN_ID` to inspect another scan. Scan +path to inspect another checkout, or `--scan-root DIR` to filter indexed scans +whose artifacts are under a particular root. It does not import report +directories. `scans show` opens the latest completed scan for the current +repository. Pass `SCAN_ID` to inspect another scan. Scan details include the configuration, results, coverage, and artifact locations. Add `--show-linked-findings` to include finding links from previous scans. @@ -714,11 +716,18 @@ treated as resolved when the later scan is incomplete or does not cover their original scope. The CLI uses [Incur](https://github.com/wevm/incur) for agent-friendly discovery -and structured output. Inspect the command manifest with `--llms`, inspect a +and structured output. Use `--llms` for a command index or `--llms-full` for +the full Markdown reference, including accepted flags, values, and the operating +guide from this README. Add `--format json` to read the original structured +manifest, or scope either manifest to a command or group, such as +`scans --llms-full`. The full operating guide appears only in the root +manifest. Inspect a command schema with `scan --schema --format json`, register the CLI as an MCP server with `mcp add`, sync agent skills with `skills add`, or generate shell completions with `completions bash|zsh|fish`. Scan results support -`--format toon|json|yaml|jsonl` and `--full-output`. +`--format toon|json|yaml|jsonl` and `--full-output`, but not Markdown or +`--filter-output`. Structured manifest property names are parsed option keys; +command-line flags use kebab-case. Use `info --json` for SDK and bundled-plugin metadata. MCP exposes only this read-only metadata command; scans, bulk repository scans, authentication, exports, validation, and patching remain CLI-only because the @@ -737,10 +746,22 @@ npx @openai/codex-security scan . \ --fail-on-severity high > "$SCAN_ROOT/findings.json" ``` +Use `scan --json` for machine-readable results on stdout; progress and +diagnostics go to stderr. Completed-result fields are `manifest`, `findings`, +`coverage`, `repositoryFindings`, `scanDir`, `reportPath`, `artifactsDir`, +`sarifPath`, `threadId`, `cost`, and `turn`. `sarifPath` and `cost` may be null; +`repositoryFindings` may be absent. `findings` describes this scan; +`repositoryFindings`, when available, includes open findings across scans. +If the target changes during execution, the result also includes a `warnings` +array of strings and the CLI exits with code `2`. Those results do not describe +the current checkout. + JSON scans never use interactive terminal controls, even when stderr is a TTY. -The `validate`, `patch`, `login`, and `logout` commands reject `--json` because -they do not produce structured CLI output. Sign-in commands remain interactive. -CSV exports cannot be written to stdout while JSON output is requested. +The `validate`, `patch`, `login`, and `logout` commands reject JSON and JSONL +result output because they do not produce structured CLI output. Sign-in +commands remain interactive. CSV exports cannot be written to stdout while +JSON output is requested. These result-format restrictions do not apply to +`--llms`, `--llms-full`, or `--schema` discovery. Use `export` to create CSV, JSON, or SARIF from a completed, sealed scan without starting Codex or loading credentials. Without a scan directory, it exports the @@ -761,8 +782,9 @@ the model with `--codex 'model="gpt-5.6-sol"'` and the reasoning effort with `--effort high` or `--codex 'model_reasoning_effort="high"'`. Exit codes are `0` for a completed report-only scan or a passing policy, `1` -for a completed policy violation, `2` for invalid input, incomplete coverage, or -a runtime/export error, `130` for interruption, and `143` for termination. +for a completed policy violation, `2` for invalid input, incomplete coverage, a +changed target, or a runtime/export error, `130` for interruption, and `143` +for termination. Use `--dry-run` or `await security.preflight(...)` to validate the repository, target, mode, output location, and Codex overrides without initializing the diff --git a/sdk/typescript/scripts/check-package.mjs b/sdk/typescript/scripts/check-package.mjs index 1d5f0ffd..2da51b34 100644 --- a/sdk/typescript/scripts/check-package.mjs +++ b/sdk/typescript/scripts/check-package.mjs @@ -164,6 +164,7 @@ const distFiles = new Set( "auth", "bulk-scan-discovery", "cli", + "cli-manifest", "config", "contract", "cost", diff --git a/sdk/typescript/scripts/smoke-package.mjs b/sdk/typescript/scripts/smoke-package.mjs index 9c7307b6..c0bc9870 100644 --- a/sdk/typescript/scripts/smoke-package.mjs +++ b/sdk/typescript/scripts/smoke-package.mjs @@ -493,6 +493,12 @@ try { /lin_api_|security@example\.test/u, ); + const manifest = runInstalledCli("--llms-full"); + assert.match(manifest, /^# codex-security$/mu); + assert.match(manifest, /^## Authentication$/mu); + assert.match(manifest, /\| `--working-tree` \|/u); + assert.doesNotMatch(manifest, /--[a-z][a-z0-9-]*[A-Z][A-Za-z0-9-]*/u); + await smokeNestedDeepScanWorker(installedRoot, consumer); console.log( diff --git a/sdk/typescript/src/cli-manifest.ts b/sdk/typescript/src/cli-manifest.ts new file mode 100644 index 00000000..bff031f6 --- /dev/null +++ b/sdk/typescript/src/cli-manifest.ts @@ -0,0 +1,477 @@ +import { readFileSync } from "node:fs"; +import { Cli, Help, Skill, z } from "incur"; +import { DEFAULT_CODEX_CONFIG, scanModelConfiguration } from "./config.js"; +import { + BUNDLED_PLUGIN_VERSION, + CODEX_EXECUTABLE_VERSION, + CODEX_SDK_VERSION, + VERSION, +} from "./version.js"; + +interface Manifest { + commands: { name: string }[]; +} + +const INCUR_FLAGS = new Set([ + "--full-output", + "--llms", + "--llms-full", + "--mcp", + "--help", + "-h", + "--version", + "--schema", + "--token-count", +]); +export const INCUR_VALUE_OPTIONS = new Set([ + "--format", + "--filter-output", + "--token-limit", + "--token-offset", +]); + +export const INFO_OUTPUT_SCHEMA = z.object({ + sdkVersion: z.string().describe("Codex Security package version."), + bundledPluginVersion: z.string().describe("Bundled security plugin version."), + scanMcp: z + .literal(false) + .describe("Whether scans are available over MCP; always false."), + cancellationNote: z.string().describe("Why scans are CLI-only."), + cliVersion: z.string().describe("Codex Security CLI version."), + codexVersion: z.string().describe("Bundled Codex executable version."), + codexSdkVersion: z.string().describe("Bundled Codex SDK version."), + model: z.string().describe("Default scan model."), + reasoningEffort: z.string().describe("Default scan reasoning effort."), + nextStep: z.string().describe("Suggested first local preflight command."), +}); + +const INFO_METADATA_FIELDS = new Set(Object.keys(INFO_OUTPUT_SCHEMA.shape)); +export const SCAN_MARKDOWN_RESULT_RESTRICTION = + "Markdown output is not supported for scan results."; + +interface CommandResultRule { + commands: readonly string[]; + message(command: string): string; + rejects(argv: readonly string[]): boolean; +} + +function hasOptionValue( + argv: readonly string[], + option: string, + value: string, +): boolean { + return argv.some( + (argument, index) => + argument === `${option}=${value}` || + (argument === option && argv[index + 1] === value), + ); +} + +function structuredOutputRequested(argv: readonly string[]): boolean { + return ( + argv.includes("--json") || + hasOptionValue(argv, "--format", "json") || + hasOptionValue(argv, "--format", "jsonl") + ); +} + +const COMMAND_RESULT_RULES: readonly CommandResultRule[] = [ + { + commands: ["validate", "patch", "login", "logout"], + message: (command) => + `${command} does not support noninteractive JSON output; run it without --json, --format json, or --format jsonl.`, + rejects: structuredOutputRequested, + }, + { + commands: ["export"], + message: () => + "CSV stdout cannot be combined with JSON output; write CSV to a file or omit --json.", + rejects: (argv) => + structuredOutputRequested(argv) && + hasOptionValue(argv, "--output", "-") && + hasOptionValue(argv, "--export-format", "csv"), + }, + { + commands: ["scan"], + message: () => "--filter-output is not supported for scan results.", + rejects: (argv) => + argv.some( + (argument) => + argument === "--filter-output" || + argument.startsWith("--filter-output="), + ), + }, + { + commands: ["scan"], + message: () => SCAN_MARKDOWN_RESULT_RESTRICTION, + rejects: (argv) => hasOptionValue(argv, "--format", "md"), + }, + { + commands: ["info"], + message: () => "--filter-output must select an info metadata field.", + rejects: (argv) => + argv.some((argument, index) => { + if ( + argument !== "--filter-output" && + !argument.startsWith("--filter-output=") + ) { + return false; + } + const selector = argument.includes("=") + ? argument.slice(argument.indexOf("=") + 1) + : argv[index + 1]; + return ( + selector !== undefined && + !selector.split(",").every((field) => INFO_METADATA_FIELDS.has(field)) + ); + }), + }, +]; + +function commandResultRules(command: string): CommandResultRule[] { + const root = command.split(" ", 1)[0]!; + return COMMAND_RESULT_RULES.filter((rule) => rule.commands.includes(root)); +} + +export function commandResultRestrictions(command: string): string[] { + const root = command.split(" ", 1)[0]!; + return commandResultRules(command).map((rule) => rule.message(root)); +} + +export function validateCommandResultOptions( + command: string, + argv: readonly string[], +): string | undefined { + const root = command.split(" ", 1)[0]!; + return commandResultRules(command) + .find((rule) => rule.rejects(argv)) + ?.message(root); +} + +/** Keep command lookup aligned with Incur's built-in option consumption. */ +export function parseIncurArguments(argv: readonly string[]): { + commandArguments: string[]; + commandIndex: number; + format: string | undefined; +} { + let format: string | undefined; + let commandIndex = -1; + const commandArguments: string[] = []; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]!; + if (argument === "--json") format = "json"; + else if (argument === "--format") format = argv[++index]; + else if (INCUR_FLAGS.has(argument)) continue; + else if ( + INCUR_VALUE_OPTIONS.has(argument) && + argv[index + 1] !== undefined + ) { + index += 1; + } else { + if (commandIndex < 0) commandIndex = index; + commandArguments.push(argument); + } + } + return { commandArguments, commandIndex, format }; +} + +export function fullMarkdownManifestArguments( + argv: readonly string[], +): string[] | undefined { + if (!argv.includes("--llms-full") || argv.includes("--mcp")) return undefined; + // Incur 0.4.13 omits the requested path from its structured manifest. + const { commandArguments, format } = parseIncurArguments(argv); + return format === undefined || format === "md" ? commandArguments : undefined; +} + +function commandScope( + commands: readonly Skill.CommandInfo[], + commandArguments: readonly string[], +): string { + let scope = ""; + for (const argument of commandArguments) { + const next = scope ? `${scope} ${argument}` : argument; + if ( + !commands.some( + ({ name }) => name === next || name?.startsWith(`${next} `), + ) + ) { + break; + } + scope = next; + if (commands.some(({ name }) => name === scope)) break; + } + return scope; +} + +/** Reconstruct only schema-owned guidance from Incur's human validation block. */ +export function humanValidationMessage( + cli: Cli.Cli, + commandArguments: readonly string[], + output: string, +): string | undefined { + const lines = output.split("\n"); + const usage = lines.indexOf("See below for usage."); + if (usage <= 0) return undefined; + const commands = Cli.collectSkillCommands( + Cli.toCommands.get(cli)!, + [], + new Map(), + ); + const scope = commandScope(commands, commandArguments); + const command = commands.find(({ name }) => name === scope); + if (command?.options === undefined) return undefined; + const input = z.toJSONSchema(command.options, { + io: "input", + unrepresentable: "any", + }); + const required = new Set(input.required); + const messages: string[] = []; + for (const line of lines.slice(0, usage)) { + const field = Object.keys(command.options.shape).find((name) => { + const flag = `--${optionName(name)}`; + return ( + line.startsWith(`Error: invalid value for ${flag}: `) || + (required.has(name) && + line === `Error: missing required option ${flag}`) + ); + }); + if (field === undefined) return undefined; + const property = input.properties?.[field]; + if (typeof property !== "object" || property === null) return undefined; + const constraints = staticInputConstraints(property); + if (constraints === undefined) return undefined; + const flag = `--${optionName(field)}`; + const problem = line.startsWith("Error: missing required option ") + ? "Missing required option" + : "Invalid value for"; + messages.push( + `${problem} ${flag}. ${describeConstraints(constraints, plainValue, true).join(" ")}`, + ); + } + return messages.join("\n"); +} + +/** Render a documentation-only view; keep Incur's parsed schemas unchanged. */ +export function renderFullMarkdownManifest( + cli: Cli.Cli, + manifest: Manifest, + commandArguments: readonly string[] = [], +): string { + const selected = new Set(manifest.commands.map(({ name }) => name)); + const groups = new Map(); + const allCommands = Cli.collectSkillCommands( + Cli.toCommands.get(cli)!, + [], + groups, + ); + const scope = commandScope(allCommands, commandArguments); + const commands = allCommands + .filter((command) => selected.has(command.name!)) + .map((command) => ({ + ...command, + args: documentInputs(command.args, false), + options: documentInputs(command.options, true), + })); + const groupRows = [...groups] + .filter(([name]) => + commands.some((command) => command.name?.startsWith(`${name} `)), + ) + .map( + ([name, description]) => `| \`${cli.name} ${name}\` | ${description} |`, + ); + const defaults = scanModelConfiguration(DEFAULT_CODEX_CONFIG); + const scopedName = scope ? `${cli.name} ${scope}` : cli.name; + const description = scope + ? groups.get(scope) ?? + allCommands.find(({ name }) => name === scope)?.description + : cli.description; + + return [ + Skill.index(cli.name, commands, description).replace( + /^# [^\n]+/u, + `# ${scopedName}`, + ), + `CLI/SDK version: ${VERSION}. Bundled plugin: ${BUNDLED_PLUGIN_VERSION}. ` + + `Codex runtime: ${CODEX_EXECUTABLE_VERSION}. Codex SDK: ${CODEX_SDK_VERSION}. ` + + `Default model: ${defaults.model}; reasoning effort: ${defaults.reasoningEffort}.`, + scope + ? `Run \`${cli.name} --llms-full\` for the operating guide.` + : readOperatingGuide(), + "## Global options and integrations", + "`--format` and `--json` select the output format for `--llms`, `--llms-full`, and `--schema`. `--filter-output` applies only to command results, which follow the restrictions in each command reference.", + "```text\n" + Help.formatRoot(cli.name, { root: true }) + "\n```", + ...(groupRows.length === 0 + ? [] + : [ + "## Command groups", + [ + "| Group | Description |", + "|-------|-------------|", + ...groupRows, + ].join("\n"), + ]), + "## Command reference", + ...commands.map((command) => { + const restrictions = commandResultRestrictions(command.name ?? ""); + return [ + Skill.generate(cli.name, [command]).replace(/^#/gmu, "###"), + ...(restrictions.length === 0 + ? [] + : [ + "#### Command result restrictions", + restrictions.map((restriction) => `- ${restriction}`).join("\n"), + ]), + ].join("\n\n"); + }), + "", + ].join("\n\n"); +} + +function readOperatingGuide(): string { + return readFileSync(new URL("../README.md", import.meta.url), "utf8") + .replace(/\r\n/gu, "\n") + .split(/(?=^## )/mu) + .filter((section) => + /^## (?:Install|Authentication|CLI|Local security model)\n/u.test( + section, + ), + ) + .join("") + .trim(); +} + +function documentInputs( + schema: z.ZodObject | undefined, + options: boolean, +): z.ZodObject | undefined { + if (schema === undefined) return undefined; + // Incur 0.4.13 renders schema keys as flags and omits input constraints. + // Adapt only the Markdown view, not the parser or machine-readable schema. + const input = z.toJSONSchema(schema, { + io: "input", + unrepresentable: "any", + }); + const required = new Set(input.required); + return z.object( + Object.fromEntries( + Object.entries(schema.shape).map(([name, field]) => { + const property = (input.properties?.[name] ?? + {}) as z.core.JSONSchema.JSONSchema; + const details = [field.description ?? ""]; + if (options && required.has(name)) details.push("Required."); + details.push(...describeConstraints(property)); + if (options && property.type === "array") { + details.push("Repeat this flag for multiple values."); + if (Array.isArray(property.default)) { + details.push(`Default: ${codeValue(property.default)}.`); + } + } + if ( + property.type === "array" && + typeof property.items === "object" && + !Array.isArray(property.items) + ) { + const itemDetails = describeConstraints(property.items); + if (itemDetails.length > 0) { + details.push(`Each value: ${itemDetails.join(" ")}`); + } + } + const key = options ? optionName(name) : name; + return [key, field.describe(details.join(" "))]; + }), + ), + ); +} + +const CONSTRAINT_LABELS = [ + ["minimum", "Minimum"], + ["exclusiveMinimum", "Must be greater than"], + ["maximum", "Maximum"], + ["exclusiveMaximum", "Must be less than"], + ["minLength", "Minimum length"], + ["maxLength", "Maximum length"], +] as const; + +function staticInputConstraints( + property: z.core.JSONSchema.JSONSchema, +): z.core.JSONSchema.JSONSchema | undefined { + if ( + typeof property.type !== "string" || + !["string", "number", "integer", "boolean", "null"].includes( + property.type, + ) || + property.$ref !== undefined || + property.anyOf !== undefined || + property.oneOf !== undefined || + property.allOf !== undefined || + property.not !== undefined || + property.if !== undefined + ) { + return undefined; + } + const values = + property.enum ?? + (property.const === undefined ? undefined : [property.const]); + if ( + values !== undefined && + !values.every( + (value) => + value === null || + typeof value === "string" || + typeof value === "boolean" || + (typeof value === "number" && Number.isFinite(value)), + ) + ) { + return undefined; + } + const constraints: z.core.JSONSchema.JSONSchema = { + type: property.type, + ...(values === undefined ? {} : { enum: values }), + }; + for (const [key] of CONSTRAINT_LABELS) { + const value = property[key]; + if (value === undefined) continue; + if (typeof value !== "number" || !Number.isFinite(value)) return undefined; + constraints[key] = value; + } + return constraints; +} + +function describeConstraints( + property: z.core.JSONSchema.JSONSchema, + renderValue: (value: unknown) => string = codeValue, + includeType = false, +): string[] { + const details: string[] = []; + if (includeType) details.push(`Expected type: ${property.type}.`); + const values = + property.enum ?? + (property.const === undefined ? undefined : [property.const]); + if (values !== undefined) { + details.push(`Allowed values: ${values.map(renderValue).join(", ")}.`); + } + if (property.type === "integer" && !includeType) { + details.push("Must be an integer."); + } + for (const [key, label] of CONSTRAINT_LABELS) { + if (property[key] !== undefined) { + details.push(`${label}: ${property[key]}.`); + } + } + return details; +} + +function optionName(name: string): string { + return name.replace(/[A-Z]/gu, (letter) => `-${letter.toLowerCase()}`); +} + +function plainValue(value: unknown): string { + return typeof value === "string" + ? value + : JSON.stringify(value) ?? String(value); +} + +function codeValue(value: unknown): string { + return `\`${plainValue(value)}\``; +} diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index d660551a..14229d82 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -50,6 +50,16 @@ import { type ScanPreflight, } from "./api.js"; import { accountStatus } from "./auth.js"; +import { + fullMarkdownManifestArguments, + humanValidationMessage, + INCUR_VALUE_OPTIONS, + INFO_OUTPUT_SCHEMA, + parseIncurArguments, + renderFullMarkdownManifest, + SCAN_MARKDOWN_RESULT_RESTRICTION, + validateCommandResultOptions, +} from "./cli-manifest.js"; import { createBulkScanDiscoveryDependencies, runBulkScanWizard, @@ -143,6 +153,7 @@ const CHILD_TERMINATION_GRACE_MS = 1_000; const PUBLICATION_GRAPHEME_SEGMENTER = new Intl.Segmenter(undefined, { granularity: "grapheme", }); +const DOCUMENTATION_FLAGS = new Set(["--help", "-h", "--llms", "--llms-full"]); type Writable = Pick & { on?(event: "error", listener: (error: Error) => void): unknown; @@ -187,7 +198,7 @@ const EXPORT_DEFAULT_OUTPUTS = { json: "findings.json", sarif: "results.sarif", } as const; -const VALUE_OPTIONS = new Set([ +const COMMAND_VALUE_OPTIONS = new Set([ "--auth", "--path", "--knowledge-base", @@ -215,10 +226,6 @@ const VALUE_OPTIONS = new Set([ "--export-format", "--output", "--source-root", - "--format", - "--filter-output", - "--token-limit", - "--token-offset", "--scan-root", "--reason", "--to", @@ -230,7 +237,9 @@ const VALUE_OPTIONS = new Set([ const PROVIDER_OPTION = z .enum(["openai", "openrouter", "fireworks", "amazon-bedrock"]) .default("openai") - .describe("Inference provider for scans."); + .describe( + "Inference provider; non-OpenAI providers require an explicit model.", + ); function optionValue(flag: string) { return z.string().min(1, `${flag} must not be empty.`); @@ -525,31 +534,39 @@ const DEEP_SCAN_OPTION_SCHEMAS = { .int() .positive() .optional() - .describe("Maximum concurrent deep-scan discovery workers."), + .describe( + "Maximum concurrent deep-scan discovery workers (bundled default: 4).", + ), subagents: z .number() .int() .nonnegative() .optional() - .describe("Subagents available to each deep-scan worker."), + .describe( + "Subagents available to each deep-scan worker (bundled default: 3).", + ), stopAfterNoNew: z .number() .int() .positive() .optional() - .describe("Stop after this many runs find no new issues."), + .describe( + "Stop after this many runs find no new issues (bundled default: 4).", + ), maxDiscoveryRuns: z .number() .int() .positive() .optional() - .describe("Maximum deep-scan discovery runs."), + .describe("Maximum deep-scan discovery runs (bundled default: 40)."), maxTimeHours: z .number() .positive() .max(96) .optional() - .describe("Maximum deep-scan discovery hours (default: 96; maximum: 96)."), + .describe( + "Maximum deep-scan discovery hours (bundled default: 96; maximum: 96).", + ), }; async function readPromptFiles( @@ -1049,15 +1066,29 @@ export async function main( errorOutput: Writable = process.stderr, dependencies: CliDependencies = DEFAULT_DEPENDENCIES, ): Promise { - argv = defaultListCommand(argv); + const completing = Boolean(process.env["COMPLETE"]); + if (!completing) { + argv = defaultListCommand( + argv.flatMap((argument) => { + const equals = argument.indexOf("="); + const option = argument.slice(0, equals); + return equals >= 0 && INCUR_VALUE_OPTIONS.has(option) + ? [option, argument.slice(equals + 1)] + : [argument]; + }), + ); + } const positionals: string[] = []; - const argumentError = validateCliArguments(argv, positionals); + const argumentError = completing + ? undefined + : validateCliArguments(argv, positionals); if (argumentError !== undefined) { errorOutput.write(`codex-security: ${argumentError}\n`); return 2; } const updateController = new AbortController(); const pendingUpdate = + !completing && errorOutput.isTTY === true && argv.length > 0 && argv[0] !== "completions" && @@ -1275,7 +1306,7 @@ export async function main( scanRoot: z .string() .optional() - .describe("Include scans whose output is under ROOT."), + .describe("Include indexed scans whose output is under ROOT."), }), output: z.record(z.string(), z.unknown()).optional(), async run({ args, format, options }) { @@ -1466,6 +1497,7 @@ export async function main( .default(false) .describe("Recompute an existing semantic finding comparison."), }), + hint: "Provide two scan identifiers, or use --all without identifiers.", output: z.record(z.string(), z.unknown()).optional(), async run({ args, format, options }) { try { @@ -1921,7 +1953,7 @@ export async function main( model: optionValue("--model") .optional() .describe( - `OpenAI model to use (default: ${DEFAULT_SCAN_MODEL_CONFIGURATION.model}).`, + `Model identifier for the selected provider (OpenAI default: ${DEFAULT_SCAN_MODEL_CONFIGURATION.model}).`, ), effort: effortOption(), provider: PROVIDER_OPTION, @@ -2018,11 +2050,15 @@ export async function main( }, }, ], + hint: + "--path, --diff, and --working-tree are mutually exclusive. " + + "Deep-scan settings require --mode deep. Use --json for scan results; " + + "--dry-run checks local inputs without verifying authentication or model access.", output: z.record(z.string(), z.unknown()).optional(), async run({ args, error: incurError, format, options }) { if (format === "md") { errorOutput.write( - "codex-security: Markdown output is not supported for scan results.\n", + `codex-security: ${SCAN_MARKDOWN_RESULT_RESTRICTION}\n`, ); exitCode = 2; return; @@ -2099,8 +2135,10 @@ export async function main( }), output: z .object({ - hook: z.string(), - failOnSeverity: z.enum(REPORTABLE_SEVERITIES), + hook: z.string().describe("Installed pre-commit hook path."), + failOnSeverity: z + .enum(REPORTABLE_SEVERITIES) + .describe("Finding severity threshold that blocks commits."), }) .optional(), async run({ args, options }) { @@ -2200,7 +2238,7 @@ export async function main( model: optionValue("--model") .optional() .describe( - `OpenAI model for each repository (default: ${DEFAULT_SCAN_MODEL_CONFIGURATION.model}).`, + `Model identifier for each repository's provider (OpenAI default: ${DEFAULT_SCAN_MODEL_CONFIGURATION.model}).`, ), effort: effortOption(), provider: PROVIDER_OPTION, @@ -2235,10 +2273,10 @@ export async function main( }, ], hint: - "CSV example:\n" + - " codex-security bulk-scan repositories.csv " + + "A repository CSV requires --output-dir. For example: " + + "`codex-security bulk-scan repositories.csv " + "--output-dir /path/outside/repositories/results " + - "--workers 4 --max-attempts 3", + "--workers 4 --max-attempts 3`.", output: z.record(z.string(), z.unknown()).optional(), async run({ args, options }) { const controller = new AbortController(); @@ -2363,7 +2401,7 @@ export async function main( sourceRoot: optionValue("--source-root") .optional() .describe( - "Repository checkout used for SARIF source-line fingerprints.", + "Repository checkout used for source-line fingerprints; requires --export-format sarif.", ), python: optionValue("--python") .optional() @@ -2613,18 +2651,7 @@ export async function main( openWorldHint: false, }, }, - output: z.object({ - sdkVersion: z.string(), - bundledPluginVersion: z.string(), - scanMcp: z.literal(false), - cancellationNote: z.string(), - cliVersion: z.string(), - codexVersion: z.string(), - codexSdkVersion: z.string(), - model: z.string(), - reasoningEffort: z.string(), - nextStep: z.string(), - }), + output: INFO_OUTPUT_SCHEMA, run() { return { sdkVersion: VERSION, @@ -2642,13 +2669,13 @@ export async function main( }); let notice: UpdateNotice | undefined; + const manifestArguments = completing + ? undefined + : fullMarkdownManifestArguments(argv); + const markdownManifest = manifestArguments !== undefined; try { await cli.serve( - argv.flatMap((argument) => - argument.startsWith("--format=") - ? ["--format", argument.slice("--format=".length)] - : [argument], - ), + [...argv, ...(markdownManifest ? ["--format", "json"] : [])], { stdout: (value) => { frameworkOutput += value; @@ -2668,12 +2695,19 @@ export async function main( if (frameworkExit !== undefined) { if (exitCode !== 0) return exitCode; errorOutput.write( - `codex-security: ${errorMessage(incurErrorMessage(frameworkOutput))}\n`, + `codex-security: ${safeIncurErrorMessage(frameworkOutput, cli, argv)}\n`, ); return 2; } if (frameworkOutput.length === 0) return exitCode; try { + if (markdownManifest) { + frameworkOutput = renderFullMarkdownManifest( + cli, + JSON.parse(frameworkOutput), + manifestArguments, + ); + } await writeCliOutput( output, renderedPublication ?? renderedHistory ?? frameworkOutput, @@ -2686,19 +2720,15 @@ export async function main( } function defaultListCommand(argv: readonly string[]): readonly string[] { - const commandIndex = argv.findIndex((value, index) => { - if (value.startsWith("-")) return false; - return index === 0 || !VALUE_OPTIONS.has(argv[index - 1]!); - }); + const { commandArguments, commandIndex } = parseIncurArguments(argv); if ( commandIndex < 0 || - !["scans", "findings"].includes(argv[commandIndex]!) || - argv.includes("--help") || - argv.includes("-h") + !["scans", "findings"].includes(commandArguments[0]!) || + argv.some((argument) => DOCUMENTATION_FLAGS.has(argument)) ) { return argv; } - const following = argv[commandIndex + 1]; + const following = commandArguments[1]; if (following !== undefined && !following.startsWith("-")) return argv; return [ ...argv.slice(0, commandIndex + 1), @@ -2851,13 +2881,42 @@ function scanArgumentsFromRecipe( }; } +function hasFlagValue(argv: readonly string[], index: number): boolean { + const next = argv[index + 1]; + return next !== undefined && !next.startsWith("--") && next !== "-h"; +} + function validateCliArguments( argv: readonly string[], positionals: string[], ): string | undefined { - if (argv.includes("--help") || argv.includes("-h")) return undefined; - const commandIndex = argv.findIndex((value) => - [ + for (let index = 0; index < argv.length; index += 1) { + const option = argv[index]!; + if (!INCUR_VALUE_OPTIONS.has(option)) continue; + const value = argv[index + 1]; + if (value === undefined || value === "" || !hasFlagValue(argv, index)) { + return `Missing value for flag: ${option}`; + } + // Keep Incur's accepted numeric values without echoing rejected operands. + if ( + (option === "--token-limit" || option === "--token-offset") && + (!Number.isFinite(Number(value)) || value.trim() === "") + ) { + return `Invalid value for ${option}: expected a finite number.`; + } + index += 1; + } + if ( + argv.includes("--schema") || + argv.some((argument) => DOCUMENTATION_FLAGS.has(argument)) + ) { + return undefined; + } + const commandArguments = parseIncurArguments(argv).commandArguments; + const command = commandArguments[0]; + if ( + command === undefined || + ![ "scan", "install-hook", "bulk-scan", @@ -2870,121 +2929,34 @@ function validateCliArguments( "login", "logout", "info", - ].includes(value), - ); - if (commandIndex < 0) return undefined; - const command = argv[commandIndex]!; - const structuredOutput = argv.some( - (value, index) => - value === "--json" || - ((value === "--format" || - value === "--format=json" || - value === "--format=jsonl") && - (value.endsWith("=json") || - value.endsWith("=jsonl") || - argv[index + 1] === "json" || - argv[index + 1] === "jsonl")), - ); - if ( - structuredOutput && - ["validate", "patch", "login", "logout"].includes(command) && - !argv.includes("--schema") - ) { - return `${command} does not support noninteractive JSON output; run it without --json, --format json, or --format jsonl.`; - } - if ( - command === "export" && - structuredOutput && - argv.some( - (value, index) => - value === "--output=-" || - (value === "--output" && argv[index + 1] === "-"), - ) && - argv.some( - (value, index) => - value === "--export-format=csv" || - (value === "--export-format" && argv[index + 1] === "csv"), - ) + ].includes(command) ) { - return "CSV stdout cannot be combined with JSON output; write CSV to a file or omit --json."; - } - if (command === "scan" && !argv.includes("--schema")) { - if ( - argv.some( - (value) => - value === "--filter-output" || value.startsWith("--filter-output="), - ) - ) { - return "--filter-output is not supported for scan results."; - } - if ( - argv.some( - (value, index) => - value === "--format=md" || - (value === "--format" && argv[index + 1] === "md"), - ) - ) { - return "Markdown output is not supported for scan results."; - } + return undefined; } + const resultOptionError = validateCommandResultOptions(command, argv); + if (resultOptionError !== undefined) return resultOptionError; const nestedCommand = command === "scans" || command === "findings" || command === "publish"; - const subcommand = nestedCommand ? argv[commandIndex + 1] : undefined; - if (command === "info") { - const metadataFields = new Set([ - "sdkVersion", - "bundledPluginVersion", - "scanMcp", - "cancellationNote", - "cliVersion", - "codexVersion", - "codexSdkVersion", - "model", - "reasoningEffort", - "nextStep", - ]); - for (let index = 0; index < argv.length; index += 1) { - const argument = argv[index]!; - if ( - argument !== "--filter-output" && - !argument.startsWith("--filter-output=") - ) { - continue; - } - const selector = argument.includes("=") - ? argument.slice(argument.indexOf("=") + 1) - : argv[index + 1]; - if ( - selector !== undefined && - !selector.split(",").every((field) => metadataFields.has(field)) - ) { - return "--filter-output must select an info metadata field."; - } - } - } + const subcommand = nestedCommand ? commandArguments[1] : undefined; for ( - let index = commandIndex + (nestedCommand ? 2 : 1); - index < argv.length; + let index = nestedCommand ? 2 : 1; + index < commandArguments.length; index += 1 ) { - const value = argv[index]!; + const value = commandArguments[index]!; if (!value.startsWith("-")) { positionals.push(value); continue; } const equals = value.indexOf("="); const option = equals < 0 ? value : value.slice(0, equals); - if (equals >= 0 || !VALUE_OPTIONS.has(option)) continue; - const next = argv[index + 1]; - if (next === undefined || next.startsWith("--") || next === "-h") { + if (equals >= 0 || !COMMAND_VALUE_OPTIONS.has(option)) continue; + if (!hasFlagValue(commandArguments, index)) { return `Missing value for flag: ${option}`; } index += 1; } - if ( - subcommand === "match" && - !argv.some((value) => ["--schema", "--llms", "--llms-full"].includes(value)) - ) { + if (subcommand === "match") { if (argv.includes("--all") && positionals.length > 0) { return "scans match --all does not accept scan identifiers."; } @@ -3319,18 +3291,30 @@ export function skillCommandFailure( return `${command} failed with exit code ${status}.`; } -function incurErrorMessage(output: string): string { +export function safeIncurErrorMessage( + output: string, + cli: Cli.Cli, + argv: readonly string[], +): string { const message = output .split("\n") .find((line) => line.startsWith("message: ")) ?.slice("message: ".length); - if (message === undefined) return output.trim(); - try { - const parsed: unknown = JSON.parse(message); - return typeof parsed === "string" ? parsed : message; - } catch { - return message; + let detail = message ?? output.trim(); + if (message !== undefined) { + try { + const parsed: unknown = JSON.parse(message); + if (typeof parsed === "string") detail = parsed; + } catch {} } + const safe = safeErrorMessage(detail); + return safe === "[redacted]" + ? humanValidationMessage( + cli, + parseIncurArguments(argv).commandArguments, + output, + ) ?? safe + : safe; } function isOutsidePath(path: string): boolean { diff --git a/sdk/typescript/tests-ts/cli-manifest.test.ts b/sdk/typescript/tests-ts/cli-manifest.test.ts new file mode 100644 index 00000000..21f18c75 --- /dev/null +++ b/sdk/typescript/tests-ts/cli-manifest.test.ts @@ -0,0 +1,549 @@ +import { describe, expect, test } from "bun:test"; +import { readFile } from "node:fs/promises"; +import { Cli, Schema, z } from "incur"; +import { main } from "../src/cli.js"; +import { + commandResultRestrictions, + fullMarkdownManifestArguments, + INCUR_VALUE_OPTIONS, + parseIncurArguments, + renderFullMarkdownManifest, + validateCommandResultOptions, +} from "../src/cli-manifest.js"; +import { DEFAULT_CODEX_CONFIG, scanModelConfiguration } from "../src/config.js"; +import { + BUNDLED_PLUGIN_VERSION, + CODEX_EXECUTABLE_VERSION, + CODEX_SDK_VERSION, + VERSION, +} from "../src/version.js"; +import { capture, dependencies, fakeResult } from "./cli-fixtures.js"; + +interface ObjectSchema { + properties?: Record; + required?: string[]; +} + +interface Command { + name: string; + schema?: { + args?: ObjectSchema; + options?: ObjectSchema; + output?: ObjectSchema; + }; + examples?: { command: string }[]; +} + +interface Manifest { + version: string; + commands: Command[]; +} + +function documentationDependencies() { + const unexpected = (): never => { + throw new Error("Documentation must not run a command or access state."); + }; + const deps = dependencies({ + environment: { + OPENAI_API_KEY: "SYNTHETIC_MANIFEST_KEY", + CODEX_SECURITY_STATE_DIR: "/synthetic/private-state", + }, + }); + deps.createSecurity = unexpected; + deps.prepareAuthenticationHome = unexpected; + deps.hasStoredChatGPTSignIn = unexpected; + deps.currentDirectory = unexpected; + deps.runCodex = unexpected; + deps.runWorkbench = unexpected; + deps.matchFindings = unexpected; + deps.exportFindings = unexpected; + deps.publishScan = unexpected; + deps.checkForUpdate = unexpected; + return deps; +} + +async function invoke(args: readonly string[]): Promise { + const stdout = capture(); + const stderr = capture(true); + expect( + await main(args, stdout.stream, stderr.stream, documentationDependencies()), + ).toBe(0); + expect(stderr.text()).toBe(""); + expect(stdout.text()).not.toContain("SYNTHETIC_MANIFEST_KEY"); + expect(stdout.text()).not.toContain("/synthetic/private-state"); + return stdout.text(); +} + +async function readManifest(args: readonly string[] = []): Promise { + return JSON.parse( + await invoke([...args, "--llms-full", "--format", "json"]), + ) as Manifest; +} + +function commandSections(markdown: string): Map { + return new Map( + markdown + .split(/^### codex-security /mu) + .slice(1) + .map((section) => { + const newline = section.indexOf("\n"); + return [section.slice(0, newline), section.slice(newline + 1)]; + }), + ); +} + +function flag(name: string): string { + return `--${name.replace(/[A-Z]/gu, (letter) => `-${letter.toLowerCase()}`)}`; +} + +describe("full CLI manifest", () => { + test("documents every live command, argument, option, and allowed value", async () => { + const manifest = await readManifest(); + const markdown = await invoke(["--llms-full"]); + const sections = commandSections(markdown); + + expect(manifest.version).toBe("incur.v1"); + expect([...sections.keys()]).toEqual( + manifest.commands.map(({ name }) => name), + ); + expect(markdown).not.toMatch(/--[a-z][a-z0-9-]*[A-Z][A-Za-z0-9-]*/u); + + for (const command of manifest.commands) { + const section = sections.get(command.name)!; + for (const [name, field] of Object.entries( + command.schema?.args?.properties ?? {}, + )) { + expect(section).toContain(`| \`${name}\` |`); + expect(section).toContain(field.description!); + } + for (const [name, field] of Object.entries( + command.schema?.options?.properties ?? {}, + )) { + const row = section + .split("\n") + .find((line) => line.startsWith(`| \`${flag(name)}\` |`)); + expect(row).toBeDefined(); + expect(row).toContain(field.description!); + const details = row!.slice( + row!.indexOf(field.description!) + field.description!.length, + ); + const schemas = [field]; + if (typeof field.items === "object" && !Array.isArray(field.items)) { + schemas.push(field.items); + } + for (const schema of schemas) { + for (const value of schema.enum ?? + (schema.const === undefined ? [] : [schema.const])) { + expect(details).toContain(`\`${String(value)}\``); + } + for (const constraint of [ + "minimum", + "exclusiveMinimum", + "maximum", + "exclusiveMaximum", + "minLength", + "maxLength", + ] as const) { + if (typeof schema[constraint] === "number") { + expect(details).toContain(String(schema[constraint])); + } + } + } + const required = + command.schema?.options?.required?.includes(name) === true && + field.default === undefined; + expect(/\brequired\b/iu.test(details)).toBe(required); + } + for (const example of command.examples ?? []) { + expect(section).toContain(`codex-security ${example.command}`); + } + } + + expect(sections.get("info")).toContain("| `sdkVersion` |"); + expect(sections.get("bulk-scan")).toContain( + "--output-dir /path/outside/repositories/results", + ); + }); + + test("includes current metadata and the packaged operating guide", async () => { + const markdown = await invoke(["--llms-full"]); + const readme = ( + await readFile(new URL("../README.md", import.meta.url), "utf8") + ).replace(/\r\n/gu, "\n"); + for (const title of [ + "Install", + "Authentication", + "CLI", + "Local security model", + ]) { + const heading = `## ${title}\n`; + const start = readme.indexOf(heading); + expect(start).toBeGreaterThanOrEqual(0); + const end = readme.indexOf("\n## ", start + heading.length); + expect(markdown).toContain( + readme.slice(start, end < 0 ? undefined : end).trim(), + ); + } + for (const title of [ + "Run a scan from TypeScript", + "Containerized bulk scans", + ]) { + expect(markdown).not.toContain(`\n## ${title}\n`); + } + const defaults = scanModelConfiguration(DEFAULT_CODEX_CONFIG); + for (const value of [ + VERSION, + BUNDLED_PLUGIN_VERSION, + CODEX_EXECUTABLE_VERSION, + CODEX_SDK_VERSION, + defaults.model, + defaults.reasoningEffort, + "--schema", + "--mcp", + "completions", + "## Global options and integrations", + "## Command groups", + "## Command reference", + ]) { + expect(markdown).toContain(value); + } + for (const key of [...Object.keys(fakeResult().toJSON()), "warnings"]) { + expect(markdown).toContain(`\`${key}\``); + } + }); + + test("preserves descriptions for selected command groups", () => { + const descriptions = { + first: "First group metadata.", + nested: "Nested group metadata.", + second: "Second group metadata.", + }; + const cli = Cli.create("sample") + .command( + Cli.create("first", { description: descriptions.first }) + .command("show", { run() {} }) + .command( + Cli.create("nested", { description: descriptions.nested }).command( + "show", + { run() {} }, + ), + ), + ) + .command( + Cli.create("second", { description: descriptions.second }).command( + "show", + { run() {} }, + ), + ); + const commands = ["first show", "first nested show", "second show"].map( + (name) => ({ name }), + ); + const full = renderFullMarkdownManifest(cli, { commands }); + for (const description of Object.values(descriptions)) { + expect(full).toContain(description); + } + const scoped = renderFullMarkdownManifest( + cli, + { + commands: [{ name: "first nested show" }], + }, + ["first", "nested"], + ); + expect(scoped).toContain(descriptions.first); + expect(scoped).toContain(descriptions.nested); + expect(scoped).not.toContain(descriptions.second); + }); + + test("preserves group and leaf discovery without executing handlers", async () => { + const root = await readManifest(); + const groups = new Set( + root.commands.flatMap(({ name }) => + name.includes(" ") ? [name.split(" ")[0]!] : [], + ), + ); + for (const group of groups) { + const expected = root.commands.filter(({ name }) => + name.startsWith(`${group} `), + ); + expect((await readManifest([group])).commands).toEqual(expected); + const short = JSON.parse( + await invoke([group, "--llms", "--json"]), + ) as Manifest; + expect(short.commands.map(({ name }) => name)).toEqual( + expected.map(({ name }) => name), + ); + const markdown = await invoke([group, "--llms-full"]); + expect(markdown).toStartWith(`# codex-security ${group}\n`); + expect(markdown).not.toContain("\n## Authentication\n"); + expect([...commandSections(markdown).keys()]).toEqual( + expected.map(({ name }) => name), + ); + } + for (const command of root.commands) { + const args = command.name.split(" "); + expect((await readManifest(args)).commands).toEqual([command]); + const markdown = await invoke([...args, "--llms-full", "--format=md"]); + expect(markdown).toStartWith(`# codex-security ${command.name}\n`); + expect(markdown).not.toContain("\n## Authentication\n"); + expect([...commandSections(markdown).keys()]).toEqual([command.name]); + } + }); + + test("keeps built-in operands out of the shared command-argument view", () => { + for (const [option, value] of [ + ["--format", "md"], + ["--filter-output", "scan"], + ["--token-limit", "100000"], + ["--token-offset", "0"], + ] as const) { + expect( + parseIncurArguments([option, value, "scans", "show", "--llms-full"]), + ).toEqual({ + commandArguments: ["scans", "show"], + commandIndex: 2, + format: option === "--format" ? value : undefined, + }); + } + expect( + parseIncurArguments([ + "scans", + "compare", + "before", + "after", + "--filter-output", + "summary", + ]), + ).toEqual({ + commandArguments: ["scans", "compare", "before", "after"], + commandIndex: 0, + format: undefined, + }); + }); + + test("preserves scoped paths when global discovery flags come first or between commands", async () => { + for (const args of [ + ["--llms-full", "--format", "md", "scans", "show"], + ["scans", "--llms-full", "--token-count", "show"], + ["--filter-output", "scan", "scans", "show", "--llms-full"], + ["--filter-output=scan", "scans", "show", "--llms-full"], + ["--token-limit=100000", "scans", "show", "--llms-full"], + ["scans", "--token-offset=0", "show", "--llms-full"], + ]) { + const markdown = await invoke(args); + expect(markdown).toStartWith("# codex-security scans show\n"); + expect([...commandSections(markdown).keys()]).toEqual(["scans show"]); + expect(markdown).not.toContain("\n## Authentication\n"); + } + expect( + await invoke(["--filter-output", "scan", "--llms-full"]), + ).toStartWith("# codex-security\n"); + }); + + test("keeps schema discovery separate from execution-only format checks", async () => { + const schema = JSON.parse( + await invoke(["export", "--schema", "--format", "json"]), + ); + expect( + JSON.parse( + await invoke([ + "export", + "--schema", + "--format", + "json", + "--output", + "-", + "--export-format", + "csv", + ]), + ), + ).toEqual(schema); + }); + + test("rejects missing or empty global values before discovery", async () => { + const rejectsMissingValue = async ( + args: readonly string[], + option: string, + ) => { + const stdout = capture(); + const stderr = capture(); + expect( + await main( + args, + stdout.stream, + stderr.stream, + documentationDependencies(), + ), + ).toBe(2); + expect(stdout.text()).toBe(""); + expect(stderr.text()).toContain(`Missing value for flag: ${option}`); + }; + for (const option of INCUR_VALUE_OPTIONS) { + await rejectsMissingValue([option], option); + await rejectsMissingValue([option, "", "scans", "--llms-full"], option); + } + await rejectsMissingValue( + ["scans", "--llms-full", "--format="], + "--format", + ); + for (const command of ["scan", "logout"]) { + for (const discovery of ["--llms", "--llms-full", "--schema"]) { + await rejectsMissingValue( + [command, "--filter-output", discovery], + "--filter-output", + ); + } + } + }); + + test("delegates partial shell-completion words without command validation", async () => { + const original = { + COMPLETE: process.env["COMPLETE"], + _COMPLETE_INDEX: process.env["_COMPLETE_INDEX"], + }; + try { + process.env["COMPLETE"] = "bash"; + for (const option of INCUR_VALUE_OPTIONS) { + for (const suffix of [[option], [option, ""]]) { + const words = ["codex-security", ...suffix]; + process.env["_COMPLETE_INDEX"] = String(words.length - 1); + await invoke(["--", ...words]); + } + } + process.env["_COMPLETE_INDEX"] = "3"; + const completions = await invoke([ + "--", + "codex-security", + "--format=json", + "scan", + "--mo", + ]); + expect(completions).toContain("--model"); + expect(completions).toContain("--mode"); + } finally { + for (const [name, value] of Object.entries(original)) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + } + }); + + test("honors explicit output formats without rewriting structured manifests", async () => { + const markdown = await invoke(["scan", "--llms-full"]); + for (const format of [ + ["--format", "md"], + ["--format=md"], + ["--json", "--format", "md"], + ]) { + expect(await invoke(["scan", "--llms-full", ...format])).toBe(markdown); + } + const manifest = await readManifest(["scan"]); + expect( + JSON.parse( + await invoke(["scan", "--llms-full", "--format", "md", "--json"]), + ), + ).toEqual(manifest); + expect(manifest.commands[0]?.schema?.options?.properties).toHaveProperty( + "outputDir", + ); + expect( + manifest.commands[0]?.schema?.options?.properties, + ).not.toHaveProperty("output-dir"); + expect( + fullMarkdownManifestArguments(["--llms-full", "--mcp"]), + ).toBeUndefined(); + }); + + test("does not mutate the schemas used by the command parser", () => { + const options = z.object({ + requiredValue: z.string().min(1), + defaultValue: z.enum(["one", "two"]).default("one"), + }); + const before = Schema.toJsonSchema(options); + const cli = Cli.create("sample").command("show", { + options, + run() {}, + }); + const markdown = renderFullMarkdownManifest(cli, { + commands: [{ name: "show" }], + }); + expect(markdown).toContain("--required-value"); + expect(markdown).toContain("--default-value"); + expect(Schema.toJsonSchema(options)).toEqual(before); + expect(options.parse({ requiredValue: "value" })).toEqual({ + requiredValue: "value", + defaultValue: "one", + }); + }); + + test("keeps unsupported result formats rejected", async () => { + for (const args of [ + ["scan", "--format", "md"], + ["scan", "--filter-output", "findings"], + ["validate", "--json"], + ["patch", "--format", "jsonl"], + ["login", "--json"], + ["logout", "--json"], + ]) { + const stdout = capture(); + const stderr = capture(); + expect( + await main( + args, + stdout.stream, + stderr.stream, + documentationDependencies(), + ), + ).toBe(2); + expect(stdout.text()).toBe(""); + expect(stderr.text()).not.toBe(""); + } + }); + + test("documents the same result restrictions that the wrapper enforces", () => { + const names = [ + "scan", + "validate", + "patch", + "login", + "logout", + "export", + "info", + ]; + const cli = Cli.create("codex-security"); + for (const name of names) { + cli.command(name, { + run() { + throw new Error("Manifest rendering must not run a command."); + }, + }); + } + const root = renderFullMarkdownManifest(cli, { + commands: names.map((name) => ({ name })), + }); + const sections = commandSections(root); + for (const [command, args] of [ + ["scan", ["scan", "--format", "md"]], + ["scan", ["scan", "--filter-output", "findings"]], + ["validate", ["validate", "--json"]], + ["patch", ["patch", "--format", "jsonl"]], + ["login", ["login", "--json"]], + ["logout", ["logout", "--json"]], + [ + "export", + ["export", "--json", "--output", "-", "--export-format", "csv"], + ], + ["info", ["info", "--filter-output", "findings"]], + ] as const) { + const restriction = validateCommandResultOptions(command, args); + expect(restriction).toBeDefined(); + expect(commandResultRestrictions(command)).toContain(restriction!); + expect(sections.get(command)).toContain(restriction!); + const scoped = renderFullMarkdownManifest( + cli, + { commands: [{ name: command }] }, + [command], + ); + expect(scoped).toContain(restriction!); + } + }); +}); diff --git a/sdk/typescript/tests-ts/cli-workbench.test.ts b/sdk/typescript/tests-ts/cli-workbench.test.ts index ef869db2..33fbb915 100644 --- a/sdk/typescript/tests-ts/cli-workbench.test.ts +++ b/sdk/typescript/tests-ts/cli-workbench.test.ts @@ -99,6 +99,10 @@ describe("CLI workbench", () => { ["scans", "list", "--scan-root", "/tmp/history"], ["list-scans", "--scan-root", resolve("/tmp/history")], ], + [ + ["scans", "list", "--scan-root", ""], + ["list-scans", "--scan-root", repository], + ], ]; for (const [argv, expected] of cases) { let invocation: readonly string[] | undefined; diff --git a/sdk/typescript/tests-ts/cli.test.ts b/sdk/typescript/tests-ts/cli.test.ts index b8401334..b6e9bb25 100644 --- a/sdk/typescript/tests-ts/cli.test.ts +++ b/sdk/typescript/tests-ts/cli.test.ts @@ -14,6 +14,7 @@ import { Writable } from "node:stream"; import { fileURLToPath } from "node:url"; import { stripVTControlCharacters } from "node:util"; import { describe, expect, test } from "bun:test"; +import { Cli, z } from "incur"; import { parse as parseToml } from "smol-toml"; import type { CodexSecurityConfig, @@ -33,7 +34,12 @@ import { ScanInterruptedError, VERSION, } from "../src/index.js"; -import { main, parseCodexOverrides, Progress } from "../src/cli.js"; +import { + main, + parseCodexOverrides, + Progress, + safeIncurErrorMessage, +} from "../src/cli.js"; import { scanPreflightCodexConfig } from "../src/api.js"; import { CODEX_EXECUTABLE_VERSION, CODEX_SDK_VERSION } from "../src/version.js"; import { @@ -412,6 +418,28 @@ describe("CLI", () => { maxTimeHours: number; }; expect(defaults.workers).toBe(4); + const schema = capture(); + expect( + await main( + ["scan", "--schema", "--format", "json"], + schema.stream, + capture().stream, + dependencies(), + ), + ).toBe(0); + const scanOptions = JSON.parse(schema.text()).options + .properties as Record; + for (const name of [ + "workers", + "subagents", + "stopAfterNoNew", + "maxDiscoveryRuns", + "maxTimeHours", + ] as const) { + expect(scanOptions[name]?.description).toContain( + `bundled default: ${defaults[name]}`, + ); + } const documentedDeepScan = documentedConfigs.find( (config) => typeof config["deep_scan"] === "object" && @@ -2189,7 +2217,7 @@ describe("CLI", () => { "--provider ", ); expect(help.text()).toContain( - `OpenAI model to use (default: ${DEFAULT_SCAN_MODEL_CONFIGURATION.model}).`, + `OpenAI default: ${DEFAULT_SCAN_MODEL_CONFIGURATION.model}`, ); expect(help.text()).toContain( "--effort ", @@ -2227,7 +2255,7 @@ describe("CLI", () => { ).toBe(0); expect(help.text()).toContain("--model "); expect(help.text()).toContain( - `OpenAI model for each repository (default: ${DEFAULT_SCAN_MODEL_CONFIGURATION.model}).`, + `OpenAI default: ${DEFAULT_SCAN_MODEL_CONFIGURATION.model}`, ); expect(help.text()).toContain( "--effort ", @@ -2733,6 +2761,65 @@ describe("CLI", () => { } }); + test("rebuilds value-free guidance from the selected command schema", () => { + const unexpected = (): never => { + throw new Error("Formatting must not run a command."); + }; + const cli = Cli.create("sample") + .command("scan", { + options: z.object({ auth: z.enum(["auto", "chatgpt", "api-key"]) }), + run: unexpected, + }) + .command( + Cli.create("publish").command("scan", { + options: z.object({ + linearApiKey: z + .string() + .min(1) + .default("documentation-only") + .describe("Description must not become a diagnostic."), + }), + run: unexpected, + }), + ) + .command("opaque", { + options: z.object({ apiKey: z.unknown() }), + run: unexpected, + }); + const humanError = (flag: string) => + [ + `Error: invalid value for ${flag}: Discard this formatter detail`, + "See below for usage.", + "", + "Usage: sample ", + ].join("\n"); + + const auth = safeIncurErrorMessage(humanError("--auth"), cli, ["scan"]); + expect(auth).toContain("Invalid value for --auth."); + expect(auth).toContain("Allowed values: auto, chatgpt, api-key."); + const key = safeIncurErrorMessage(humanError("--linear-api-key"), cli, [ + "publish", + "scan", + ]); + expect(key).toContain("Invalid value for --linear-api-key."); + expect(key).toContain("Expected type: string."); + expect(key).toContain("Minimum length: 1."); + for (const message of [auth, key]) { + expect(message).not.toContain("formatter detail"); + expect(message).not.toContain("documentation-only"); + expect(message).not.toContain("Description must not"); + } + for (const [output, command] of [ + [humanError("--linear-api-key"), ["scan"]], + [humanError("--linearApiKey"), ["publish", "scan"]], + [humanError("--access-token"), ["publish", "scan"]], + [humanError("--api-key"), ["opaque"]], + ["Error: invalid value for --auth: Invalid option", ["scan"]], + ] as const) { + expect(safeIncurErrorMessage(output, cli, command)).toBe("[redacted]"); + } + }); + test("honors Incur help before command validation", async () => { const stdout = capture(); const stderr = capture();