From 47467f4e1a11de934ada58b4c2265328973753b6 Mon Sep 17 00:00:00 2001 From: kunaldhongade <74715881+kunaldhongade@users.noreply.github.com> Date: Wed, 1 Jul 2026 00:44:12 +0530 Subject: [PATCH] feat(loop): gate merge safety on evidence depth --- docs/loop.md | 16 +- packages/cli/src/benchmark/corpus.ts | 53 +++++ packages/cli/src/commands/loop.ts | 191 +++++++++++++++++- .../src/docs/command-docs/orchestration.ts | 4 +- packages/cli/src/parsers/loop.ts | 23 ++- packages/cli/src/types/loop.ts | 1 + packages/cli/test/benchmark-corpus.test.ts | 8 +- packages/cli/test/benchmark.test.ts | 20 +- packages/cli/test/loop-e2e.test.ts | 154 ++++++++++++++ packages/cli/test/loop.test.ts | 67 +++++- packages/core/src/types/security.ts | 2 +- packages/harness/src/index.ts | 8 +- packages/harness/src/loop/controller.ts | 167 +++++++++++++-- packages/harness/src/loop/index.ts | 8 +- packages/harness/src/loop/render.ts | 29 +++ packages/harness/src/loop/types.ts | 79 +++++++- packages/harness/test/loop.test.ts | 126 +++++++++++- packages/matchers/src/defaults.ts | 36 +++- packages/matchers/src/utils.ts | 42 ++++ packages/matchers/test/matchers.test.ts | 46 +++++ 20 files changed, 1027 insertions(+), 53 deletions(-) create mode 100644 packages/cli/test/loop-e2e.test.ts diff --git a/docs/loop.md b/docs/loop.md index 9a20665..a86a0dd 100644 --- a/docs/loop.md +++ b/docs/loop.md @@ -30,17 +30,26 @@ After each agent action, CodeDecay re-runs deterministic analysis and configured ## Safety Rules -`codedecay loop` only reports `merge-safe` when all of these are true: +`codedecay loop` never prints an unqualified "safe" verdict. Clean outcomes are always qualified by evidence depth. + +The loop can only report a `merge-safe-*` verdict when all of these are true: - final risk is at or below the configured safe threshold, `low` by default - weak-test findings are zero +- security score is at or below the configured threshold, `0` by default +- no high-severity findings remain in deterministic analysis - configured checks exist and pass -If no checks are configured, the best possible terminal status is `unverified`, not `merge-safe`. +If no checks are configured, the best possible terminal status is `unverified`, not a `merge-safe-*` verdict. + +`merge-safe-verified` means configured checks passed, deterministic security matchers were clean, Semgrep was enabled and clean, and coverage/mutation evidence was available if configured. + +`merge-safe-shallow` means the gates passed, but one or more deeper evidence streams were missing. Treat it as heuristic clean, not as deep verification. Run `codedecay doctor` to configure OSS adapters such as Semgrep, coverage, and StrykerJS. Terminal statuses: -- `merge-safe`: deterministic risk is low enough, weak tests are gone, and configured checks passed +- `merge-safe-verified`: configured and enabled checks found nothing at the selected thresholds, including available security/coverage/mutation depth +- `merge-safe-shallow`: risk, weak-test, security-score, and configured-check gates passed, but depth evidence such as Semgrep, coverage, or mutation testing is missing - `unverified`: risk and weak-test evidence are clean, but no configured checks proved the result - `plan-only`: no agent command was configured - `stuck`: the agent made no progress for two rounds @@ -54,6 +63,7 @@ codedecay loop \ --cwd ../my-repo \ --agent-cmd "codex exec --apply" \ --max-rounds 3 \ + --max-security-score 0 \ --format markdown ``` diff --git a/packages/cli/src/benchmark/corpus.ts b/packages/cli/src/benchmark/corpus.ts index 45d84fd..b484e7f 100644 --- a/packages/cli/src/benchmark/corpus.ts +++ b/packages/cli/src/benchmark/corpus.ts @@ -74,6 +74,12 @@ export function createDefaultBenchmarkCorpus(): BenchmarkCorpus { setup: createOneHopSqlInjectionRepo, expectedRuleIds: ["security-sql-injection"] }, + { + id: "indirect-dynamic-sqli", + kind: "positive", + setup: createIndirectDynamicSqlInjectionRepo, + expectedRuleIds: ["security-sql-injection"] + }, { id: "plain-exported-destructive-missing-auth", kind: "positive", @@ -115,6 +121,12 @@ export function createDefaultBenchmarkCorpus(): BenchmarkCorpus { kind: "decoy", setup: createGuardedDestructiveAuthDecoyRepo, expectedRuleIds: [] + }, + { + id: "dynamic-sql-local-decoy", + kind: "decoy", + setup: createDynamicSqlLocalDecoyRepo, + expectedRuleIds: [] } ], cleanup: cleanupBenchmarkCorpus @@ -329,6 +341,26 @@ export function createOneHopSqlInjectionRepo(): string { return repo; } +export function createIndirectDynamicSqlInjectionRepo(): string { + const repo = createRepo({ + "src/reports/search.ts": "export function buildSearchQuery(status) { return 'select 1'; }\n" + }); + + writeFile( + repo, + "src/reports/search.ts", + [ + "export function buildSearchQuery(status) {", + " const sql = `select * from invoices where status = '${status}'`;", + " return sql;", + "}", + "" + ].join("\n") + ); + + return repo; +} + export function createPlainExportedDestructiveMissingAuthRepo(): string { const repo = createRepo({ "src/services/users.ts": "export function listUsers() { return []; }\n" @@ -485,6 +517,27 @@ export function createGuardedDestructiveAuthDecoyRepo(): string { return repo; } +export function createDynamicSqlLocalDecoyRepo(): string { + const repo = createRepo({ + "src/reports/static.ts": "export function buildStaticReportQuery() { return 'select 1'; }\n" + }); + + writeFile( + repo, + "src/reports/static.ts", + [ + "export function buildStaticReportQuery() {", + " const status = 'paid';", + " const sql = 'select * from invoices where status = ' + status;", + " return sql;", + "}", + "" + ].join("\n") + ); + + return repo; +} + function createBranchingFunction(name: string, ifCount: number): string { return [ `export function ${name}(input) {`, diff --git a/packages/cli/src/commands/loop.ts b/packages/cli/src/commands/loop.ts index 98797a7..26ca766 100644 --- a/packages/cli/src/commands/loop.ts +++ b/packages/cli/src/commands/loop.ts @@ -2,11 +2,31 @@ import { resolve } from "node:path"; import { createAgentTaskBundle, renderAgentTaskBundle } from "@submuxhq/codedecay-agent"; import { loadCodeDecayConfig, type CodeDecayConfig, type LoadedCodeDecayConfig } from "@submuxhq/codedecay-config"; import { getGitChangedFiles } from "@submuxhq/codedecay-git"; -import { renderLoopReport, runCodeDecayLoop, type LoopCheckSnapshot, type LoopReport } from "@submuxhq/codedecay-harness"; +import { + renderLoopReport, + runCodeDecayLoop, + type Evidence, + type LoopCheckSnapshot, + type LoopCheckStatus, + type LoopCoverageSnapshot, + type LoopMutationSnapshot, + type LoopReport, + type LoopSecurityToolSnapshot +} from "@submuxhq/codedecay-harness"; +import type { ConfiguredToolAdapterKind } from "@submuxhq/codedecay-tool-adapters"; import type { RedteamReport } from "@submuxhq/codedecay-redteam"; import { CliExit } from "../errors"; import { parseLoopArgs } from "../parsers/args"; -import type { AgentOptions, AnalyzeOptions, CliAnalysisContext, CliCommandContext, CliRuntime, RedteamOptions } from "../types"; +import type { + AgentOptions, + AnalyzeOptions, + CliAnalysisContext, + CliCommandContext, + CliRuntime, + ExecutionReport, + ExecutionToolAdapterResult, + RedteamOptions +} from "../types"; import { createExecutionReport } from "./execute/report"; import type { RunExecuteCommandDependencies } from "./execute/types"; import { createRedteamReportForCli, type RedteamReportDependencies } from "./redteam-report"; @@ -41,6 +61,7 @@ export async function runLoopCommand( maxRounds: options.maxRounds, agentCommand: options.agentCommand, safeRiskLevel: options.safeRiskLevel, + securityScoreThreshold: options.securityScoreThreshold, agentTimeoutMs: loadedConfig.config.safety.commandTimeoutMs, commandSafety: { allowCommands: loadedConfig.config.safety.allowCommands @@ -85,11 +106,15 @@ async function createLoopCheckSnapshot( timedOut: 0, errors: 0, durationMs: 0, + semgrep: emptySecurityToolSnapshot(), + coverage: emptyCoverageSnapshot(), + mutation: emptyMutationSnapshot(), note: "No configured commands, probes, or tool adapters were found." }; } const report = await createExecutionReport(rootDir, loadedConfig, dependencies); + const adapterEvidence = createAdapterEvidenceSnapshot(report); return { configured: true, status: report.summary.status, @@ -99,7 +124,10 @@ async function createLoopCheckSnapshot( skipped: report.summary.skipped, timedOut: report.summary.timedOut, errors: report.summary.errors, - durationMs: report.summary.durationMs + durationMs: report.summary.durationMs, + semgrep: adapterEvidence.semgrep, + coverage: adapterEvidence.coverage, + mutation: adapterEvidence.mutation }; } @@ -119,3 +147,160 @@ function shouldFail(report: LoopReport): boolean { report.status === "needs-human" || report.status === "agent-error"; } + +function createAdapterEvidenceSnapshot(report: ExecutionReport): { + semgrep: LoopSecurityToolSnapshot; + coverage: LoopCoverageSnapshot; + mutation: LoopMutationSnapshot; +} { + return { + semgrep: summarizeSemgrepAdapter(report.toolAdapters.find((adapter) => adapter.kind === "semgrep")), + coverage: summarizeCoverageAdapter(report.toolAdapters.find((adapter) => adapter.kind === "coverage")), + mutation: summarizeMutationAdapter(report.toolAdapters.find((adapter) => adapter.kind === "stryker")) + }; +} + +function summarizeSemgrepAdapter(adapter: ExecutionToolAdapterResult | undefined): LoopSecurityToolSnapshot { + if (!adapter) { + return emptySecurityToolSnapshot(); + } + + const evidence = evidenceForAdapter(adapter, "semgrep"); + const findingCount = firstFiniteMetadataNumber(evidence, "findingCount") ?? evidence.filter((item) => item.file).length; + const highFindingCount = evidence.filter((item) => item.file && item.severity === "high").length; + const maxSeverity = maxRiskSeverity(evidence); + return { + configured: true, + ran: adapterRan(adapter.status), + status: adapterStatusToLoopStatus(adapter.status), + findingCount, + highFindingCount, + maxSeverity + }; +} + +function summarizeCoverageAdapter(adapter: ExecutionToolAdapterResult | undefined): LoopCoverageSnapshot { + if (!adapter) { + return emptyCoverageSnapshot(); + } + + const evidence = evidenceForAdapter(adapter, "coverage"); + const measuredLines = firstFiniteMetadataNumber(evidence, "measuredLines"); + const coveredLines = firstFiniteMetadataNumber(evidence, "coveredLines"); + const uncoveredLines = firstFiniteMetadataNumber(evidence, "uncoveredLines"); + const percent = measuredLines && measuredLines > 0 && coveredLines !== undefined + ? roundPercent((coveredLines / measuredLines) * 100) + : undefined; + return { + configured: true, + present: measuredLines !== undefined, + status: adapterStatusToLoopStatus(adapter.status), + percent, + measuredLines, + coveredLines, + uncoveredLines + }; +} + +function summarizeMutationAdapter(adapter: ExecutionToolAdapterResult | undefined): LoopMutationSnapshot { + if (!adapter) { + return emptyMutationSnapshot(); + } + + const evidence = evidenceForAdapter(adapter, "stryker"); + const totalMutants = firstFiniteMetadataNumber(evidence, "totalMutants"); + const survivedMutants = firstFiniteMetadataNumber(evidence, "survivedMutants") ?? 0; + const noCoverageMutants = firstFiniteMetadataNumber(evidence, "noCoverageMutants") ?? 0; + const mutationScore = firstFiniteMetadataNumber(evidence, "mutationScore"); + const weakMutants = totalMutants === undefined ? undefined : survivedMutants + noCoverageMutants; + return { + configured: true, + present: totalMutants !== undefined || mutationScore !== undefined, + status: adapterStatusToLoopStatus(adapter.status), + mutationScore, + totalMutants, + weakMutants + }; +} + +function emptySecurityToolSnapshot(): LoopSecurityToolSnapshot { + return { + configured: false, + ran: false, + status: "not-configured", + findingCount: 0, + highFindingCount: 0 + }; +} + +function emptyCoverageSnapshot(): LoopCoverageSnapshot { + return { + configured: false, + present: false, + status: "not-configured" + }; +} + +function emptyMutationSnapshot(): LoopMutationSnapshot { + return { + configured: false, + present: false, + status: "not-configured" + }; +} + +function evidenceForAdapter(adapter: ExecutionToolAdapterResult, id: ConfiguredToolAdapterKind | "semgrep" | "coverage" | "stryker"): Evidence[] { + return adapter.evidence.filter((item) => item.source.id === id || item.kind === evidenceKindForAdapter(id)); +} + +function evidenceKindForAdapter(id: ConfiguredToolAdapterKind | "semgrep" | "coverage" | "stryker"): Evidence["kind"] { + if (id === "coverage") { + return "coverage"; + } + + if (id === "stryker") { + return "mutation"; + } + + return "static-analysis"; +} + +function firstFiniteMetadataNumber(evidence: Evidence[], key: string): number | undefined { + for (const item of evidence) { + const value = item.metadata?.[key]; + if (typeof value === "number" && Number.isFinite(value)) { + return value; + } + } + + return undefined; +} + +function maxRiskSeverity(evidence: Evidence[]): LoopSecurityToolSnapshot["maxSeverity"] { + const severities = evidence.map((item) => item.severity).filter((severity) => severity !== "info"); + if (severities.includes("high")) { + return "high"; + } + + if (severities.includes("medium")) { + return "medium"; + } + + if (severities.includes("low")) { + return "low"; + } + + return undefined; +} + +function adapterRan(status: ExecutionToolAdapterResult["status"]): boolean { + return status !== "skipped"; +} + +function adapterStatusToLoopStatus(status: ExecutionToolAdapterResult["status"]): LoopCheckStatus { + return status; +} + +function roundPercent(value: number): number { + return Math.round(value * 100) / 100; +} diff --git a/packages/cli/src/docs/command-docs/orchestration.ts b/packages/cli/src/docs/command-docs/orchestration.ts index 7074ea9..afcac97 100644 --- a/packages/cli/src/docs/command-docs/orchestration.ts +++ b/packages/cli/src/docs/command-docs/orchestration.ts @@ -127,6 +127,7 @@ export const ORCHESTRATION_COMMAND_DOCS: Record = { { flag: "--max-rounds ", description: "Maximum fix/recheck rounds (default: 4)" }, { flag: "--agent-cmd ", description: "Explicit user-owned agent command that reads the task bundle on stdin and may edit the working tree" }, { flag: "--safe-risk ", description: "Maximum acceptable risk level: low, medium, or high (default: low)" }, + { flag: "--max-security-score ", description: "Maximum acceptable security score from deterministic analysis, 0-100 (default: 0)" }, { flag: "--format ", description: "json or markdown (default: markdown)" }, { flag: "--output ", description: "Write loop report to a file instead of stdout" } ], @@ -139,7 +140,8 @@ export const ORCHESTRATION_COMMAND_DOCS: Record = { "CodeDecay does not embed a model. The agent command is user-owned and explicit.", "The loop never auto-commits or auto-pushes. It leaves edits in the working tree for human review.", "Agent output is untrusted. CodeDecay re-runs deterministic analysis and configured checks after each agent action.", - "Exit codes: 0 for merge-safe or plan-only report generation, 1 for unverified, needs-human, stuck, or agent-error, and 2 for CLI/internal errors." + "Terminal clean verdicts are always qualified: merge-safe-verified has configured checks plus security/coverage/mutation depth, while merge-safe-shallow passed gates but is missing deeper evidence.", + "Exit codes: 0 for merge-safe-verified, merge-safe-shallow, or plan-only report generation; 1 for unverified, needs-human, stuck, or agent-error; and 2 for CLI/internal errors." ] }, doctor: { diff --git a/packages/cli/src/parsers/loop.ts b/packages/cli/src/parsers/loop.ts index 70e09c0..f7acbeb 100644 --- a/packages/cli/src/parsers/loop.ts +++ b/packages/cli/src/parsers/loop.ts @@ -7,7 +7,8 @@ export function parseLoopArgs(args: string[]): LoopOptions { const options: LoopOptions = { maxRounds: 4, format: "markdown", - safeRiskLevel: "low" + safeRiskLevel: "low", + securityScoreThreshold: 0 }; for (let index = 0; index < args.length; index += 1) { @@ -108,6 +109,17 @@ export function parseLoopArgs(args: string[]): LoopOptions { continue; } + if (arg.startsWith("--max-security-score=")) { + options.securityScoreThreshold = parseSecurityScoreThreshold(arg.slice("--max-security-score=".length)); + continue; + } + + if (arg === "--max-security-score") { + options.securityScoreThreshold = parseSecurityScoreThreshold(requireValue(args, index, arg)); + index += 1; + continue; + } + throwUnknownOption(arg, "loop"); } @@ -130,3 +142,12 @@ function parseMaxRounds(value: string): number { throw new Error(`Invalid --max-rounds "${value}". Expected a positive integer.`); } + +function parseSecurityScoreThreshold(value: string): number { + const parsed = Number(value); + if (Number.isFinite(parsed) && parsed >= 0 && parsed <= 100) { + return parsed; + } + + throw new Error(`Invalid --max-security-score "${value}". Expected a number from 0 to 100.`); +} diff --git a/packages/cli/src/types/loop.ts b/packages/cli/src/types/loop.ts index cbec887..179f79a 100644 --- a/packages/cli/src/types/loop.ts +++ b/packages/cli/src/types/loop.ts @@ -10,4 +10,5 @@ export interface LoopOptions { format: LoopFormat; output?: string | undefined; safeRiskLevel: RiskLevel; + securityScoreThreshold: number; } diff --git a/packages/cli/test/benchmark-corpus.test.ts b/packages/cli/test/benchmark-corpus.test.ts index 9c5019f..a4c8303 100644 --- a/packages/cli/test/benchmark-corpus.test.ts +++ b/packages/cli/test/benchmark-corpus.test.ts @@ -247,16 +247,18 @@ describe("unified harness planted issue corpus", () => { const corpus = createDefaultBenchmarkCorpus(); expect(corpus.rules.map((rule) => rule.ruleId)).toEqual(DEFAULT_BENCHMARK_RULES.map((rule) => rule.ruleId)); - expect(corpus.scenarios.filter((scenario) => scenario.kind === "positive")).toHaveLength(6); - expect(corpus.scenarios.filter((scenario) => scenario.kind === "decoy")).toHaveLength(4); + expect(corpus.scenarios.filter((scenario) => scenario.kind === "positive")).toHaveLength(7); + expect(corpus.scenarios.filter((scenario) => scenario.kind === "decoy")).toHaveLength(5); expect(corpus.scenarios.map((scenario) => scenario.id)).toEqual( expect.arrayContaining([ "one-hop-sqli", + "indirect-dynamic-sqli", "plain-exported-destructive-missing-auth", "auth-comment-destructive-missing-auth", "one-hop-path-join-traversal", "request-name-collision-decoy", - "guarded-destructive-auth-decoy" + "guarded-destructive-auth-decoy", + "dynamic-sql-local-decoy" ]) ); }); diff --git a/packages/cli/test/benchmark.test.ts b/packages/cli/test/benchmark.test.ts index d9f8ac6..b01b87b 100644 --- a/packages/cli/test/benchmark.test.ts +++ b/packages/cli/test/benchmark.test.ts @@ -28,11 +28,11 @@ describe("codedecay benchmark CLI contract", () => { expect(result.stderr).toBe(""); expect(report.corpus).toBe("default"); expect(report.summary).toMatchObject({ - totalExpected: 22, - totalMatched: 22, + totalExpected: 23, + totalMatched: 23, overallRecall: 1, falsePositives: 2, - falsePositiveRate: 0.0278, + falsePositiveRate: 0.0222, costUsd: 0, llmCalled: false, telemetrySent: false @@ -40,13 +40,13 @@ describe("codedecay benchmark CLI contract", () => { expect(report.summary.falsePositiveRate).toBeLessThan(0.1); expect(report.summary.durationMs).toBeGreaterThanOrEqual(0); expect(report.metrics.byArea).toEqual([ - expect.objectContaining({ area: "security", expected: 12, matched: 12, recall: 1, falsePositives: 0 }), + expect.objectContaining({ area: "security", expected: 13, matched: 13, recall: 1, falsePositives: 0 }), expect.objectContaining({ area: "regression", expected: 5, matched: 5, recall: 1, falsePositives: 2 }), expect.objectContaining({ area: "quality", expected: 5, matched: 5, recall: 1, falsePositives: 0 }) ]); expect(report.metrics.byRuleId).toEqual( expect.arrayContaining([ - expect.objectContaining({ ruleId: "security-sql-injection", expected: 2, matched: 2 }), + expect.objectContaining({ ruleId: "security-sql-injection", expected: 3, matched: 3 }), expect.objectContaining({ ruleId: "security-missing-auth-entrypoint", expected: 3, matched: 3 }), expect.objectContaining({ ruleId: "security-path-traversal", expected: 2, matched: 2 }), expect.objectContaining({ ruleId: "happy-path-only-test", expected: 1, matched: 1 }), @@ -59,6 +59,10 @@ describe("codedecay benchmark CLI contract", () => { id: "one-hop-sqli", matchedRuleIds: ["security-sql-injection"] }), + expect.objectContaining({ + id: "indirect-dynamic-sqli", + matchedRuleIds: ["security-sql-injection"] + }), expect.objectContaining({ id: "plain-exported-destructive-missing-auth", matchedRuleIds: ["security-missing-auth-entrypoint"] @@ -78,6 +82,10 @@ describe("codedecay benchmark CLI contract", () => { expect.objectContaining({ id: "guarded-destructive-auth-decoy", falsePositiveRuleIds: [] + }), + expect.objectContaining({ + id: "dynamic-sql-local-decoy", + falsePositiveRuleIds: [] }) ]) ); @@ -94,7 +102,7 @@ describe("codedecay benchmark CLI contract", () => { expect(result.stderr).toBe(""); expect(rendered).toContain("## CodeDecay Benchmark"); expect(rendered).toContain("| Overall recall | 100% |"); - expect(rendered).toContain("| False-positive rate | 2.78% |"); + expect(rendered).toContain("| False-positive rate | 2.22% |"); expect(rendered).toContain("- LLM/model called: no"); expect(rendered).toContain("- Telemetry sent: no"); }); diff --git a/packages/cli/test/loop-e2e.test.ts b/packages/cli/test/loop-e2e.test.ts new file mode 100644 index 0000000..91907d1 --- /dev/null +++ b/packages/cli/test/loop-e2e.test.ts @@ -0,0 +1,154 @@ +import { existsSync, readFileSync, rmSync, writeFileSync, mkdirSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { run } from "./helpers"; + +const tempRoots: string[] = []; +const describeLoopE2e = process.env.CODEDECAY_LOOP_E2E === "1" ? describe : describe.skip; + +afterEach(() => { + for (const root of tempRoots.splice(0)) { + rmSync(root, { recursive: true, force: true }); + } +}); + +describeLoopE2e("codedecay loop real edit convergence", () => { + it("drives a deterministic agent script from weak test to merge-safe-*", async () => { + const repo = createLoopConvergenceRepo(); + + const result = await run([ + "loop", + "--format", + "json", + "--max-rounds", + "3", + "--agent-cmd", + "node scripts/fix-test.mjs" + ], repo); + const report = JSON.parse(result.stdout) as { + status: string; + rounds: Array<{ mergeRiskScore: number; weakTestFindings: number; agent?: { madeChanges: boolean } }>; + safety: { llmCalled: boolean; telemetrySent: boolean; autoCommitted: boolean; autoPushed: boolean }; + }; + + expect(result.stderr).toBe(""); + expect(result.exitCode).toBe(0); + expect(report.status.startsWith("merge-safe-")).toBe(true); + expect(report.rounds.length).toBeGreaterThanOrEqual(2); + expect(report.rounds[0]?.weakTestFindings).toBeGreaterThan(report.rounds.at(-1)?.weakTestFindings ?? 0); + expect(report.rounds[0]?.mergeRiskScore).toBeGreaterThan(report.rounds.at(-1)?.mergeRiskScore ?? 0); + expect(report.rounds[0]?.agent?.madeChanges).toBe(true); + expect(readFileSync(join(repo, "test/checkout.test.js"), "utf8")).toContain("rejects negative totals"); + expect(existsSync(join(repo, ".git"))).toBe(true); + expect(report.safety).toMatchObject({ + llmCalled: false, + telemetrySent: false, + autoCommitted: false, + autoPushed: false + }); + }); +}); + +function createLoopConvergenceRepo(): string { + const repo = join(tmpdir(), `codedecay-loop-e2e-${Math.random().toString(16).slice(2)}`); + mkdirSync(repo, { recursive: true }); + tempRoots.push(repo); + git(repo, ["init", "-b", "main"]); + git(repo, ["config", "user.email", "codedecay@example.com"]); + git(repo, ["config", "user.name", "CodeDecay Test"]); + writeFile(repo, "package.json", JSON.stringify({ type: "module" }, null, 2)); + writeFile(repo, "src/checkout.js", "export function calculateTotal(amount, tax) { return amount + tax; }\n"); + writeFile( + repo, + "test/checkout.test.js", + [ + "import { test } from 'node:test';", + "import { strictEqual } from 'node:assert/strict';", + "import { calculateTotal } from '../src/checkout.js';", + "", + "test('calculates total', () => {", + " strictEqual(calculateTotal(100, 8), 108);", + "});", + "" + ].join("\n") + ); + writeFile( + repo, + ".codedecay/config.yml", + [ + "version: 1", + "commands:", + " test:", + " - node --test test/checkout.test.js", + "safety:", + " commandTimeoutMs: 1000", + " allowCommands: true", + "" + ].join("\n") + ); + writeFile( + repo, + "scripts/fix-test.mjs", + [ + "import { writeFileSync } from 'node:fs';", + "writeFileSync('test/checkout.test.js', [", + " \"import { test } from 'node:test';\",", + " \"import { strictEqual, throws } from 'node:assert/strict';\",", + " \"import { calculateTotal } from '../src/checkout.js';\",", + " \"\",", + " \"test('calculates total with tax', () => {\",", + " \" strictEqual(calculateTotal(100, 8), 108);\",", + " \"});\",", + " \"\",", + " \"test('rejects negative totals', () => {\",", + " \" throws(() => calculateTotal(-1, 0), /amount/);\",", + " \"});\",", + " \"\"", + "].join('\\n'));", + "" + ].join("\n") + ); + git(repo, ["add", "."]); + git(repo, ["commit", "-m", "initial"]); + + writeFile( + repo, + "src/checkout.js", + [ + "export function calculateTotal(amount, tax) {", + " if (amount < 0) throw new Error('amount must be positive');", + " return amount + tax;", + "}", + "" + ].join("\n") + ); + writeFile( + repo, + "test/checkout.test.js", + [ + "import { test } from 'node:test';", + "import { calculateTotal } from '../src/checkout.js';", + "", + "test('calculates total', () => {", + " calculateTotal(100, 8);", + "});", + "" + ].join("\n") + ); + + return repo; +} + +function writeFile(root: string, path: string, contents: string): void { + const fullPath = join(root, path); + mkdirSync(dirname(fullPath), { recursive: true }); + writeFileSync(fullPath, contents, "utf8"); +} + +function git(repo: string, args: string[]): void { + execFileSync("git", ["-C", repo, ...args], { + stdio: "ignore" + }); +} diff --git a/packages/cli/test/loop.test.ts b/packages/cli/test/loop.test.ts index ef082f3..a8ed902 100644 --- a/packages/cli/test/loop.test.ts +++ b/packages/cli/test/loop.test.ts @@ -12,7 +12,7 @@ import { } from "./helpers"; describe("codedecay loop CLI contract", () => { - it("reports merge-safe with low risk and passing configured checks", async () => { + it("reports merge-safe-shallow with low risk and passing configured checks when depth evidence is missing", async () => { const repo = createLowRiskRepoWithPassingCheck(); const result = await run(["loop", "--format", "json"], repo); @@ -20,12 +20,31 @@ describe("codedecay loop CLI contract", () => { expect(result.exitCode).toBe(0); expect(result.stderr).toBe(""); - expect(report.status).toBe("merge-safe"); + expect(report.status).toBe("merge-safe-shallow"); expect(report.roundsRun).toBe(1); expect(report.finalCheckStatus).toBe("passed"); + expect(report.verdict.missingDepth).toEqual( + expect.arrayContaining(["no Semgrep adapter configured", "no coverage adapter configured", "no mutation adapter configured"]) + ); expect(report.safety.commandsExecuted).toBe(true); }); + it("carries Semgrep, coverage, and mutation evidence into a merge-safe-verified verdict", async () => { + const repo = createLowRiskRepoWithVerifiedChecks(); + + const result = await run(["loop", "--format", "json"], repo); + const report = JSON.parse(result.stdout); + + expect(result.exitCode).toBe(0); + expect(result.stderr).toBe(""); + expect(report.status).toBe("merge-safe-verified"); + expect(report.rounds[0].checkStatus).toBe("passed"); + expect(report.verdict.verifiedBy).toEqual( + expect.arrayContaining(["Semgrep (0 findings)", "coverage evidence (100%)", "mutation evidence (100%)"]) + ); + expect(report.verdict.missingDepth).toEqual([]); + }); + it("runs plan-only without an agent command and writes output relative to --cwd", async () => { const repo = createHighRiskRepo(); const outside = createTempDir(); @@ -107,7 +126,8 @@ describe("codedecay loop CLI contract", () => { expect(result.exitCode).toBe(0); expect(result.stdout).toContain("## CodeDecay Loop Report"); - expect(result.stdout).toContain("**Status:** merge safe"); + expect(result.stdout).toContain("**Status:** merge safe shallow"); + expect(result.stdout).toContain("### Verdict Evidence"); }); }); @@ -128,3 +148,44 @@ function createLowRiskRepoWithPassingCheck(): string { writeFile(repo, "README.md", "# Project\nDocs change.\n"); return repo; } + +function createLowRiskRepoWithVerifiedChecks(): string { + const repo = createRepo({ + "README.md": "# Project\n", + ".codedecay/config.yml": [ + "version: 1", + "commands:", + " test:", + " - node -e \"process.exit(0)\"", + "toolAdapters:", + " semgrep:", + " command: node semgrep-pass.js", + " reportPath: reports/semgrep.json", + " coverage:", + " reportPaths:", + " - coverage/coverage-final.json", + " failOn: none", + " stryker:", + " command: node stryker-pass.js", + " reportPath: reports/mutation/mutation.json", + "safety:", + " commandTimeoutMs: 1000", + " allowCommands: true", + "" + ].join("\n"), + "semgrep-pass.js": "console.log('semgrep done');\n", + "stryker-pass.js": "console.log('stryker done');\n", + "reports/semgrep.json": JSON.stringify({ results: [] }, null, 2), + "coverage/coverage-final.json": JSON.stringify({ "src/index.ts": { l: { "1": 1, "2": 1 } } }, null, 2), + "reports/mutation/mutation.json": JSON.stringify({ + thresholds: { mutationScore: 100 }, + files: { + "src/index.ts": { + mutants: [{ id: "1", status: "Killed", mutatorName: "StringLiteral" }] + } + } + }, null, 2) + }); + writeFile(repo, "README.md", "# Project\nDocs change.\n"); + return repo; +} diff --git a/packages/core/src/types/security.ts b/packages/core/src/types/security.ts index ca1fd44..3a84b63 100644 --- a/packages/core/src/types/security.ts +++ b/packages/core/src/types/security.ts @@ -1,6 +1,6 @@ import type { RiskLevel } from "../risk"; -export type SecurityCandidateConfidence = "direct" | "heuristic" | "entry-point"; +export type SecurityCandidateConfidence = "direct" | "heuristic" | "entry-point" | "indirect"; export interface SecurityCandidate { ruleId: string; diff --git a/packages/harness/src/index.ts b/packages/harness/src/index.ts index 6ae5463..976cc00 100644 --- a/packages/harness/src/index.ts +++ b/packages/harness/src/index.ts @@ -4,6 +4,8 @@ export { changedFilePaths, createChangedFilesFingerprint, driveAgent, + classifySafeStatus, + createLoopVerdictEvidence, renderLoopMarkdown, renderLoopReport, runCodeDecayLoop @@ -15,12 +17,16 @@ export type { LoopAgentResult, LoopCheckSnapshot, LoopCheckStatus, + LoopCoverageSnapshot, LoopFixTask, LoopFormat, + LoopMutationSnapshot, LoopRedteamReport, LoopReport, LoopRoundSnapshot, - LoopStatus + LoopSecurityToolSnapshot, + LoopStatus, + LoopVerdictEvidence } from "./loop"; export type { CodeDecayHarness, diff --git a/packages/harness/src/loop/controller.ts b/packages/harness/src/loop/controller.ts index 7adf318..1208b0c 100644 --- a/packages/harness/src/loop/controller.ts +++ b/packages/harness/src/loop/controller.ts @@ -8,7 +8,8 @@ import type { LoopRedteamReport, LoopReport, LoopRoundSnapshot, - LoopStatus + LoopStatus, + LoopVerdictEvidence } from "./types"; interface PreviousAgentRound { @@ -20,6 +21,7 @@ interface PreviousAgentRound { export async function runCodeDecayLoop(input: CodeDecayLoopInput): Promise { const maxRounds = normalizeMaxRounds(input.maxRounds); const safeRiskLevel = input.safeRiskLevel ?? "low"; + const securityScoreThreshold = normalizeSecurityScoreThreshold(input.securityScoreThreshold); const rounds: LoopRoundSnapshot[] = []; let status: LoopStatus = "needs-human"; let noProgressCount = 0; @@ -57,7 +59,7 @@ export async function runCodeDecayLoop(input: CodeDecayLoopInput): Promise 100) { + throw new Error("--max-security-score must be a number from 0 to 100."); + } + + return value; +} + +export function classifySafeStatus( report: LoopRedteamReport, checks: LoopCheckSnapshot, - safeRiskLevel: LoopRedteamReport["summary"]["riskLevel"] -): "merge-safe" | "unverified" | undefined { - const riskAllowed = riskRank(report.summary.riskLevel) <= riskRank(safeRiskLevel); - const noWeakTests = report.summary.weakTestFindings === 0; - if (!riskAllowed || !noWeakTests) { + safeRiskLevel: LoopRedteamReport["summary"]["riskLevel"], + securityScoreThreshold = 0 +): "merge-safe-verified" | "merge-safe-shallow" | "unverified" | undefined { + const evidence = createLoopVerdictEvidence(report, checks, safeRiskLevel, securityScoreThreshold, "needs-human"); + if (!evidence.riskAllowed || !evidence.weakTestsClear || !evidence.securityScoreAllowed || evidence.highFindingCount > 0) { + return undefined; + } + + if (!checks.configured || checks.total === 0) { + return "unverified"; + } + + if (!evidence.checksPassed || evidence.blockingReasons.length > 0) { return undefined; } - return checks.configured && checks.total > 0 && checks.status === "passed" ? "merge-safe" : "unverified"; + return evidence.missingDepth.length === 0 ? "merge-safe-verified" : "merge-safe-shallow"; +} + +export function createLoopVerdictEvidence( + report: LoopRedteamReport, + checks: LoopCheckSnapshot, + safeRiskLevel: LoopRedteamReport["summary"]["riskLevel"], + securityScoreThreshold: number, + status: LoopStatus +): LoopVerdictEvidence { + const highFindings = report.analysis.findings.filter((finding) => finding.severity === "high"); + const highSecurityFindings = highFindings.filter((finding) => finding.category === "security"); + const securityMatcherFindings = report.analysis.securityCandidates?.length ?? report.analysis.securityAnalysis?.candidateCount ?? 0; + const securityMatcherHighFindings = (report.analysis.securityCandidates ?? []).filter( + (candidate) => candidate.severity === "high" + ).length; + const evidence: LoopVerdictEvidence = { + status, + riskAllowed: riskRank(report.summary.riskLevel) <= riskRank(safeRiskLevel), + weakTestsClear: report.summary.weakTestFindings === 0, + checksPassed: checks.configured && checks.total > 0 && checks.status === "passed", + checksConfigured: checks.configured && checks.total > 0, + securityScoreAllowed: report.summary.securityScore <= securityScoreThreshold, + securityScore: report.summary.securityScore, + securityScoreThreshold, + highFindingCount: highFindings.length, + highSecurityFindingCount: Math.max(highSecurityFindings.length, securityMatcherHighFindings), + securityMatchersRan: Boolean(report.analysis.securityAnalysis), + securityMatcherFindings, + securityMatcherHighFindings, + verifiedBy: [], + missingDepth: [], + blockingReasons: [] + }; + + if (evidence.checksPassed) { + evidence.verifiedBy.push("configured checks (passed)"); + } else if (!evidence.checksConfigured) { + evidence.blockingReasons.push("No configured checks ran."); + } else { + evidence.blockingReasons.push(`Configured checks ended with status ${checks.status}.`); + } + + if (evidence.securityMatchersRan) { + evidence.verifiedBy.push(`security matchers (${evidence.securityMatcherFindings} finding(s))`); + } else { + evidence.missingDepth.push("security matchers did not scan changed source"); + } + + if (checks.semgrep.configured && checks.semgrep.ran && checks.semgrep.status === "passed" && checks.semgrep.findingCount === 0) { + evidence.verifiedBy.push("Semgrep (0 findings)"); + } else if (!checks.semgrep.configured) { + evidence.missingDepth.push("no Semgrep adapter configured"); + } else if (!checks.semgrep.ran) { + evidence.missingDepth.push(`Semgrep adapter configured but ${checks.semgrep.status}`); + } else { + evidence.blockingReasons.push(`Semgrep reported ${checks.semgrep.findingCount} finding(s).`); + } + + if (checks.coverage.configured && checks.coverage.present && checks.coverage.status === "passed") { + const percent = checks.coverage.percent === undefined ? "unknown" : `${checks.coverage.percent}%`; + evidence.verifiedBy.push(`coverage evidence (${percent})`); + } else if (!checks.coverage.configured) { + evidence.missingDepth.push("no coverage adapter configured"); + } else if (!checks.coverage.present) { + evidence.missingDepth.push(`coverage adapter configured but no coverage evidence was present (${checks.coverage.status})`); + } else { + evidence.blockingReasons.push(`Coverage adapter ended with status ${checks.coverage.status}.`); + } + + if (checks.mutation.configured && checks.mutation.present && checks.mutation.status === "passed" && (checks.mutation.weakMutants ?? 0) === 0) { + const score = checks.mutation.mutationScore === undefined ? "unknown" : `${checks.mutation.mutationScore}%`; + evidence.verifiedBy.push(`mutation evidence (${score})`); + } else if (!checks.mutation.configured) { + evidence.missingDepth.push("no mutation adapter configured"); + } else if (!checks.mutation.present) { + evidence.missingDepth.push(`mutation adapter configured but no mutation evidence was present (${checks.mutation.status})`); + } else { + evidence.blockingReasons.push(`Mutation adapter reported ${checks.mutation.weakMutants ?? "unknown"} surviving/no-coverage mutant(s).`); + } + + if (!evidence.riskAllowed) { + evidence.blockingReasons.push(`Risk level ${report.summary.riskLevel} exceeds safe threshold ${safeRiskLevel}.`); + } + + if (!evidence.weakTestsClear) { + evidence.blockingReasons.push(`${report.summary.weakTestFindings} weak-test finding(s) remain.`); + } + + if (!evidence.securityScoreAllowed) { + evidence.blockingReasons.push(`Security score ${report.summary.securityScore}/100 exceeds threshold ${securityScoreThreshold}/100.`); + } + + if (evidence.highFindingCount > 0) { + evidence.blockingReasons.push(`${evidence.highFindingCount} high-severity finding(s) remain.`); + } + + return evidence; } function didRiskReduce(previous: PreviousAgentRound, current: LoopRedteamReport): boolean { @@ -208,19 +330,25 @@ function didAgentExecuteCommand(status: LoopAgentResult["status"]): boolean { return status === "passed" || status === "failed" || status === "timed_out" || status === "error"; } -function nextStepsForStatus(status: LoopStatus): string[] { +function nextStepsForStatus(status: LoopStatus, verdict: LoopVerdictEvidence): string[] { switch (status) { - case "merge-safe": + case "merge-safe-verified": return [ "Review the working tree diff.", "Commit the verified changes yourself when ready.", - "Do not skip human review for business-critical flows." + "Treat this as configured-check clean, not a guarantee of production safety." + ]; + case "merge-safe-shallow": + return [ + "Review the working tree diff and the missing-depth list before merge.", + "Enable Semgrep, coverage, and StrykerJS adapters to upgrade this verdict to merge-safe-verified.", + "Treat this as shallow configured-check clean, not a guarantee of production safety." ]; case "unverified": return [ "Add or enable configured checks in .codedecay/config.yml.", "Run codedecay loop again after tests/build/probes can execute.", - "Do not treat this PR as merge-safe until real checks pass." + "Do not treat this PR as merge-safe-* until real checks pass." ]; case "plan-only": return [ @@ -244,7 +372,16 @@ function nextStepsForStatus(status: LoopStatus): string[] { return [ "Max rounds were reached before CodeDecay could prove merge safety.", "Review remaining fix tasks and check failures manually.", - "Increase --max-rounds only if the agent is making measurable progress." + "Increase --max-rounds only if the agent is making measurable progress.", + ...missingDepthNextSteps(verdict) ]; } } + +function missingDepthNextSteps(verdict: LoopVerdictEvidence): string[] { + if (verdict.missingDepth.length === 0) { + return []; + } + + return ["Run codedecay doctor and enable missing OSS adapters such as Semgrep, coverage, or StrykerJS for deeper evidence."]; +} diff --git a/packages/harness/src/loop/index.ts b/packages/harness/src/loop/index.ts index 80ccd73..62b9ae1 100644 --- a/packages/harness/src/loop/index.ts +++ b/packages/harness/src/loop/index.ts @@ -1,5 +1,5 @@ export { driveAgent } from "./agent"; -export { runCodeDecayLoop } from "./controller"; +export { classifySafeStatus, createLoopVerdictEvidence, runCodeDecayLoop } from "./controller"; export { createChangedFilesFingerprint, changedFilePaths } from "./fingerprint"; export { renderLoopMarkdown, renderLoopReport } from "./render"; export type { @@ -8,10 +8,14 @@ export type { LoopAgentResult, LoopCheckSnapshot, LoopCheckStatus, + LoopCoverageSnapshot, LoopFixTask, LoopFormat, + LoopMutationSnapshot, LoopRedteamReport, LoopReport, LoopRoundSnapshot, - LoopStatus + LoopSecurityToolSnapshot, + LoopStatus, + LoopVerdictEvidence } from "./types"; diff --git a/packages/harness/src/loop/render.ts b/packages/harness/src/loop/render.ts index bdc93ae..5ba538d 100644 --- a/packages/harness/src/loop/render.ts +++ b/packages/harness/src/loop/render.ts @@ -19,9 +19,30 @@ export function renderLoopMarkdown(report: LoopReport): string { `| Rounds run | ${report.roundsRun} / ${report.maxRounds} |`, `| Final risk | ${report.finalRiskLevel} |`, `| Final merge risk | ${report.finalMergeRiskScore}/100 |`, + `| Final security risk | ${report.finalSecurityScore}/100 |`, `| Final weak-test findings | ${report.finalWeakTestFindings} |`, `| Final check status | ${report.finalCheckStatus} |`, "", + "### Verdict Evidence", + "", + "CodeDecay never guarantees a safe merge. This verdict means the configured and enabled checks below found no blocking evidence.", + "", + "| Evidence | Value |", + "| --- | --- |", + `| Security score threshold | ${report.verdict.securityScoreThreshold}/100 |`, + `| High findings remaining | ${report.verdict.highFindingCount} |`, + `| High security findings remaining | ${report.verdict.highSecurityFindingCount} |`, + `| Security matchers | ${report.verdict.securityMatchersRan ? `${report.verdict.securityMatcherFindings} finding(s)` : "not available"} |`, + "", + "**Verified by:**", + ...bulletLines(report.verdict.verifiedBy, "nothing yet"), + "", + "**Missing depth:**", + ...bulletLines(report.verdict.missingDepth, "none"), + "", + "**Blocking reasons:**", + ...bulletLines(report.verdict.blockingReasons, "none"), + "", "### Rounds", "", "| Round | Risk | Merge | Weak tests | Checks | Agent |", @@ -100,6 +121,14 @@ function statusLabel(status: LoopReport["status"]): string { return status.replaceAll("-", " "); } +function bulletLines(values: string[], emptyText: string): string[] { + if (values.length === 0) { + return [`- ${emptyText}`]; + } + + return values.map((value) => `- ${value}`); +} + function singleLine(value: string): string { return value.trim().replace(/\s+/g, " ").slice(0, 500); } diff --git a/packages/harness/src/loop/types.ts b/packages/harness/src/loop/types.ts index 64afed0..87c122c 100644 --- a/packages/harness/src/loop/types.ts +++ b/packages/harness/src/loop/types.ts @@ -2,7 +2,8 @@ import type { CommandExecutionResult, SafeCommandPolicy } from "@submuxhq/codede import type { FileChange, RiskLevel } from "@submuxhq/codedecay-core"; export type LoopStatus = - | "merge-safe" + | "merge-safe-verified" + | "merge-safe-shallow" | "unverified" | "stuck" | "needs-human" @@ -25,9 +26,32 @@ export interface LoopRedteamReport { summary: { riskLevel: RiskLevel; mergeRiskScore: number; + securityScore: number; weakTestFindings: number; fixTasks: number; }; + analysis: { + findings: Array<{ + ruleId: string; + title: string; + severity: RiskLevel; + category: string; + file?: string | undefined; + line?: number | undefined; + }>; + securityAnalysis?: { + scannedFiles: string[]; + candidateCount: number; + } | undefined; + securityCandidates?: Array<{ + ruleId: string; + title: string; + severity: RiskLevel; + confidence: string; + file: string; + line?: number | undefined; + }> | undefined; + }; fixTasks: LoopFixTask[]; safety: { commandsExecuted: false; @@ -56,9 +80,40 @@ export interface LoopCheckSnapshot { timedOut: number; errors: number; durationMs: number; + semgrep: LoopSecurityToolSnapshot; + coverage: LoopCoverageSnapshot; + mutation: LoopMutationSnapshot; note?: string | undefined; } +export interface LoopSecurityToolSnapshot { + configured: boolean; + ran: boolean; + status: LoopCheckStatus; + findingCount: number; + highFindingCount: number; + maxSeverity?: RiskLevel | undefined; +} + +export interface LoopCoverageSnapshot { + configured: boolean; + present: boolean; + status: LoopCheckStatus; + percent?: number | undefined; + measuredLines?: number | undefined; + coveredLines?: number | undefined; + uncoveredLines?: number | undefined; +} + +export interface LoopMutationSnapshot { + configured: boolean; + present: boolean; + status: LoopCheckStatus; + mutationScore?: number | undefined; + totalMutants?: number | undefined; + weakMutants?: number | undefined; +} + export interface LoopAgentResult { command: string; status: CommandExecutionResult["status"]; @@ -99,8 +154,10 @@ export interface LoopReport { planOnly: boolean; finalRiskLevel: RiskLevel; finalMergeRiskScore: number; + finalSecurityScore: number; finalWeakTestFindings: number; finalCheckStatus: LoopCheckStatus; + verdict: LoopVerdictEvidence; finalFixTasks: LoopFixTask[]; rounds: LoopRoundSnapshot[]; nextSteps: string[]; @@ -115,6 +172,25 @@ export interface LoopReport { }; } +export interface LoopVerdictEvidence { + status: LoopStatus; + riskAllowed: boolean; + weakTestsClear: boolean; + checksPassed: boolean; + checksConfigured: boolean; + securityScoreAllowed: boolean; + securityScore: number; + securityScoreThreshold: number; + highFindingCount: number; + highSecurityFindingCount: number; + securityMatchersRan: boolean; + securityMatcherFindings: number; + securityMatcherHighFindings: number; + verifiedBy: string[]; + missingDepth: string[]; + blockingReasons: string[]; +} + export interface CodeDecayLoopInput { cwd: string; base?: string | undefined; @@ -122,6 +198,7 @@ export interface CodeDecayLoopInput { maxRounds?: number | undefined; agentCommand?: string | undefined; safeRiskLevel?: RiskLevel | undefined; + securityScoreThreshold?: number | undefined; agentTimeoutMs: number; commandSafety: SafeCommandPolicy; createRedteamReport(): Promise; diff --git a/packages/harness/test/loop.test.ts b/packages/harness/test/loop.test.ts index 96b8c28..e75d2ba 100644 --- a/packages/harness/test/loop.test.ts +++ b/packages/harness/test/loop.test.ts @@ -4,7 +4,12 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { getGitChangedFiles } from "@submuxhq/codedecay-git"; -import { runCodeDecayLoop, type LoopCheckSnapshot, type LoopRedteamReport } from "../src/index"; +import { + classifySafeStatus, + runCodeDecayLoop, + type LoopCheckSnapshot, + type LoopRedteamReport +} from "../src/index"; const tempRoots: string[] = []; @@ -15,7 +20,7 @@ afterEach(() => { }); describe("CodeDecay loop controller", () => { - it("reports merge-safe when risk is low, weak tests are gone, and checks pass", async () => { + it("reports merge-safe-shallow when gates pass but depth evidence is missing", async () => { const repo = createRepo(); const report = await runCodeDecayLoop({ ...baseInput(repo), @@ -23,9 +28,32 @@ describe("CodeDecay loop controller", () => { runConfiguredChecks: async () => checkSnapshot("passed", true) }); - expect(report.status).toBe("merge-safe"); + expect(report.status).toBe("merge-safe-shallow"); expect(report.roundsRun).toBe(1); expect(report.safety.commandsExecuted).toBe(true); + expect(report.verdict.missingDepth).toEqual( + expect.arrayContaining(["no Semgrep adapter configured", "no coverage adapter configured", "no mutation adapter configured"]) + ); + }); + + it("reports merge-safe-verified when security, coverage, mutation, and configured checks pass", async () => { + const repo = createRepo(); + const report = await runCodeDecayLoop({ + ...baseInput(repo), + createRedteamReport: async () => + redteamReport({ + riskLevel: "low", + mergeRiskScore: 10, + weakTestFindings: 0, + securityAnalysis: { scannedFiles: ["src/index.ts"], candidateCount: 0 } + }), + runConfiguredChecks: async () => verifiedCheckSnapshot() + }); + + expect(report.status).toBe("merge-safe-verified"); + expect(report.verdict.verifiedBy).toEqual( + expect.arrayContaining(["Semgrep (0 findings)", "coverage evidence (100%)", "mutation evidence (100%)"]) + ); }); it("runs plan-only without an agent command", async () => { @@ -106,6 +134,41 @@ describe("CodeDecay loop controller", () => { }); }); +describe("classifySafeStatus", () => { + it("does not return merge-safe when a high security finding remains", () => { + const status = classifySafeStatus( + redteamReport({ + riskLevel: "low", + mergeRiskScore: 10, + weakTestFindings: 0, + securityScore: 0, + findings: [{ + ruleId: "security-sql-injection", + title: "SQL injection candidate", + severity: "high", + category: "security", + file: "src/api/users.ts" + }], + securityAnalysis: { scannedFiles: ["src/api/users.ts"], candidateCount: 1 } + }), + verifiedCheckSnapshot(), + "low" + ); + + expect(status).toBeUndefined(); + }); + + it("returns merge-safe-shallow when gates pass without scanner, coverage, or mutation depth", () => { + const status = classifySafeStatus( + redteamReport({ riskLevel: "low", mergeRiskScore: 10, weakTestFindings: 0 }), + checkSnapshot("passed", true), + "low" + ); + + expect(status).toBe("merge-safe-shallow"); + }); +}); + function baseInput(repo: string) { return { cwd: repo, @@ -121,15 +184,23 @@ function redteamReport(input: { riskLevel: LoopRedteamReport["summary"]["riskLevel"]; mergeRiskScore: number; weakTestFindings: number; + securityScore?: number | undefined; + findings?: LoopRedteamReport["analysis"]["findings"] | undefined; + securityAnalysis?: LoopRedteamReport["analysis"]["securityAnalysis"] | undefined; }): LoopRedteamReport { return { version: "0.3.3", summary: { riskLevel: input.riskLevel, mergeRiskScore: input.mergeRiskScore, + securityScore: input.securityScore ?? 0, weakTestFindings: input.weakTestFindings, fixTasks: input.riskLevel === "low" && input.weakTestFindings === 0 ? 0 : 1 }, + analysis: { + findings: input.findings ?? [], + securityAnalysis: input.securityAnalysis + }, fixTasks: input.riskLevel === "low" && input.weakTestFindings === 0 ? [] : [{ @@ -157,7 +228,54 @@ function checkSnapshot(status: LoopCheckSnapshot["status"], configured: boolean) skipped: status === "skipped" ? 1 : 0, timedOut: status === "timed_out" ? 1 : 0, errors: status === "error" ? 1 : 0, - durationMs: 0 + durationMs: 0, + semgrep: { + configured: false, + ran: false, + status: "not-configured", + findingCount: 0, + highFindingCount: 0 + }, + coverage: { + configured: false, + present: false, + status: "not-configured" + }, + mutation: { + configured: false, + present: false, + status: "not-configured" + } + }; +} + +function verifiedCheckSnapshot(): LoopCheckSnapshot { + return { + ...checkSnapshot("passed", true), + semgrep: { + configured: true, + ran: true, + status: "passed", + findingCount: 0, + highFindingCount: 0 + }, + coverage: { + configured: true, + present: true, + status: "passed", + percent: 100, + measuredLines: 2, + coveredLines: 2, + uncoveredLines: 0 + }, + mutation: { + configured: true, + present: true, + status: "passed", + mutationScore: 100, + totalMutants: 1, + weakMutants: 0 + } }; } diff --git a/packages/matchers/src/defaults.ts b/packages/matchers/src/defaults.ts index e4c4448..1834058 100644 --- a/packages/matchers/src/defaults.ts +++ b/packages/matchers/src/defaults.ts @@ -3,6 +3,7 @@ import { containsAny, containsAnySinkMarker, createCandidate, + findIndirectSqlConstructionLines, findParameterTaintedSinkLines, hasRouteEntryPoint, hasTemplateUserInputExpression, @@ -30,6 +31,8 @@ export const sqlInjectionMatcher: SecurityMatcher = { } ], match(context) { + // Keep this matcher intentionally small: Semgrep is the deep OSS path for SQLi. + // These checks provide deterministic fallback/triage signals when Semgrep is not configured. const directMatches = lineMatches(context.content, (line) => { const codeLine = maskStringLiterals(line).toLowerCase(); return ( @@ -40,16 +43,31 @@ export const sqlInjectionMatcher: SecurityMatcher = { ); }); const taintedMatches = findParameterTaintedSinkLines(context.content, [".query(", "execute(", "$queryrawunsafe", "$executerawunsafe"]); + const directLines = new Set(uniqueMatches([...directMatches, ...taintedMatches]).map((match) => match.line)); + const indirectMatches = findIndirectSqlConstructionLines(context.content).filter((match) => !directLines.has(match.line)); - return uniqueMatches([...directMatches, ...taintedMatches]).map((match) => - createCandidate({ - ...this, - file: context.filePath, - line: match.line, - snippet: match.text, - evidence: "Raw SQL or dynamic query construction is present near request-controlled input." - }) - ); + return [ + ...uniqueMatches([...directMatches, ...taintedMatches]).map((match) => + createCandidate({ + ...this, + file: context.filePath, + line: match.line, + snippet: match.text, + evidence: "Raw SQL or dynamic query construction is present near request-controlled input." + }) + ), + ...indirectMatches.map((match) => + createCandidate({ + ...this, + severity: "medium", + confidence: "indirect", + file: context.filePath, + line: match.line, + snippet: match.text, + evidence: "Dynamic SQL is built from request or function-parameter input without a visible database sink. Use the Semgrep adapter for deeper validation." + }) + ) + ]; } }; diff --git a/packages/matchers/src/utils.ts b/packages/matchers/src/utils.ts index 0fd6b44..5261d78 100644 --- a/packages/matchers/src/utils.ts +++ b/packages/matchers/src/utils.ts @@ -142,6 +142,48 @@ export function findParameterTaintedSinkLines(content: string, sinkMarkers: stri return matches; } +export function findIndirectSqlConstructionLines(content: string): LineMatch[] { + const parameters = collectFunctionParameters(content); + const matches: LineMatch[] = []; + const taintedLocals = new Set(); + const lines = content.split(/\n/); + + for (const [index, line] of lines.entries()) { + const codeLine = maskStringLiterals(line).toLowerCase(); + const originalLowerLine = line.toLowerCase(); + for (const local of collectLocalsAssignedFromTaint(codeLine, parameters)) { + taintedLocals.add(local); + } + + if (!looksLikeDynamicSqlConstruction(line)) { + continue; + } + + const hasTaintedInput = + hasTemplateUserInputExpression(line) || + hasUserInputMarker(codeLine) || + [...parameters, ...taintedLocals].some((identifier) => containsIdentifier(originalLowerLine, identifier)); + if (hasTaintedInput) { + matches.push({ line: index + 1, text: line.trim() }); + } + } + + return matches; +} + +function looksLikeDynamicSqlConstruction(line: string): boolean { + const lowerLine = line.toLowerCase(); + if (!/\b(select|insert|update|delete)\b/.test(lowerLine)) { + return false; + } + + if (!/\b(from|into|set|where|values)\b/.test(lowerLine)) { + return false; + } + + return line.includes("${") || /(?:['"`]\s*\+|\+\s*['"`])/.test(line); +} + function codeAfterFirstSink(codeLine: string, sinkMarkers: string[]): string { const indexes = sinkMarkers .map((marker) => { diff --git a/packages/matchers/test/matchers.test.ts b/packages/matchers/test/matchers.test.ts index 967414b..0ce62f6 100644 --- a/packages/matchers/test/matchers.test.ts +++ b/packages/matchers/test/matchers.test.ts @@ -187,6 +187,52 @@ describe("scanSecurityCandidates", () => { ); }); + it("flags indirect dynamic SQL built from function parameters without pretending to be a deep scan", () => { + const result = scanSecurityCandidates({ + files: [ + { + path: "src/reports/search.ts", + content: [ + "export function buildSearchQuery(status) {", + " const sql = `select * from invoices where status = '${status}'`;", + " return sql;", + "}", + "" + ].join("\n") + } + ] + }); + + expect(result.candidates).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + ruleId: "security-sql-injection", + confidence: "indirect" + }) + ]) + ); + }); + + it("does not flag dynamic SQL built only from local constants as request-controlled", () => { + const result = scanSecurityCandidates({ + files: [ + { + path: "src/reports/static.ts", + content: [ + "export function buildStaticReportQuery() {", + " const status = 'paid';", + " const sql = 'select * from invoices where status = ' + status;", + " return sql;", + "}", + "" + ].join("\n") + } + ] + }); + + expect(result.candidates.map((candidate) => candidate.ruleId)).not.toContain("security-sql-injection"); + }); + it("does not treat function names ending in request as outbound request sinks", () => { const result = scanSecurityCandidates({ files: [