From 4570e5583c8cfee22aef7cee86da83f729c81375 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:40:25 -0700 Subject: [PATCH 1/4] =?UTF-8?q?feat(calibration):=20slop-corpus=20replay?= =?UTF-8?q?=20backfill=20=E2=80=94=20phase=203=20of=20the=20backfill=20fam?= =?UTF-8?q?ily=20(#8277)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replays the deterministic slop scorer over the archived raw-context diffs (the #8130/#8170 corpus manifest) and synthesizes provenance-tagged slop_gate_score fired/override pairs labeled by realized outcomes — the phase-1 'mapping a' counterfactual framing, applied to the #8224 knob. Signal-subset honesty is enforced by ALLOWLIST: only the three diff-derivable signals may contribute (the scorer fires empty_pr_description on an UNDEFINED description — right live, inflating here), risk/band recompute from exactly their weights, and every row records the computed signal codes beside the provenance tag. Zero GitHub traffic; idempotent upserts; wrangler/--pg dual path. Applied + verified on both stores (460 fired + 460 override rows each): distribution zero 165 / low 3 / elevated 292 / high 0 — all mass below the 0.60 ceiling, so the rows are NEUTRAL to ladder comparisons by construction (they feed the reliability curve and drift reads without being able to fabricate proposals; the discriminating source for the ladder band remains live full-signal capture). --- scripts/backfill-slop-corpus-core.ts | 182 ++++++++++++++++++++ scripts/backfill-slop-corpus.ts | 99 +++++++++++ test/unit/backfill-slop-corpus-core.test.ts | 140 +++++++++++++++ 3 files changed, 421 insertions(+) create mode 100644 scripts/backfill-slop-corpus-core.ts create mode 100644 scripts/backfill-slop-corpus.ts create mode 100644 test/unit/backfill-slop-corpus-core.test.ts diff --git a/scripts/backfill-slop-corpus-core.ts b/scripts/backfill-slop-corpus-core.ts new file mode 100644 index 000000000..c8969bd3d --- /dev/null +++ b/scripts/backfill-slop-corpus-core.ts @@ -0,0 +1,182 @@ +// Pure core for the slop-corpus replay backfill (#8277) — phase 3 of the calibration backfill family. +// The slop scorer is deterministic and in-repo, so the #8224 report-only knob's evidence can be +// manufactured honestly: replay `buildSlopAssessment` over each archived PR diff (the #8130/#8170 +// raw-context corpus) and synthesize provenance-tagged `slop_gate_score` fired/override pairs labeled by +// what humans actually did — the same counterfactual "mapping a" framing the phase-1 close-confidence +// backfill documented. This core is transform-only (no IO), mirroring backfill-calibration-corpus-core.ts; +// the CLI wrapper owns the manifest read and the wrangler/pg writes. +// +// SIGNAL-SUBSET HONESTY: only the diff-derivable signals can replay (trivialWhitespaceChurn, +// missingTestEvidence, nonSubstantivePadding — changedFiles parse out of the archived unified diff). The +// unarchived inputs (description, commit messages, duplicate-cluster membership, linked-issue state) are +// passed UNDEFINED so their signals SKIP rather than fire spuriously — a replayed score is therefore a +// LOWER BOUND on what live scoring would have produced. Every synthesized row records the computed signal +// codes plus the provenance tag so the eventual flip-to-live decision can weigh exactly that. + +import { buildSlopAssessment, SLOP_WEIGHTS, slopBandFor, type SlopChangedFile } from "../packages/loopover-engine/src/signals/slop"; +import type { SynthesizedAuditRow } from "./backfill-calibration-corpus-core.js"; + +export const SLOP_BACKFILL_RULE_ID = "slop_gate_score"; +export const SLOP_BACKFILL_PROVENANCE = "slop_replay_backfill_v1"; +const FIRED_EVENT_TYPE = `signal.rule_fired:${SLOP_BACKFILL_RULE_ID}`; +const OVERRIDE_EVENT_TYPE = `signal.human_override:${SLOP_BACKFILL_RULE_ID}`; + +/** One replayable source case, projected from a backtest-corpus manifest (backtest-corpus-export.ts): + * the archived bounded diff plus the human label the original rule's history already established. */ +/** The diff-derivable signal codes the replay may score, mapped to the scorer's own weights. Everything + * else needs unarchived inputs and is EXCLUDED even if the scorer fires it on an undefined field. */ +const REPLAYABLE_SIGNAL_WEIGHTS: Record = { + trivial_whitespace_churn: SLOP_WEIGHTS.trivialWhitespaceChurn, + missing_test_evidence: SLOP_WEIGHTS.missingTestEvidence, + non_substantive_padding: SLOP_WEIGHTS.nonSubstantivePadding, +}; + +export type SlopReplaySourceCase = { + targetKey: string; + label: "confirmed" | "reversed"; + firedAt: string; + decidedAt: string; + diff: string; +}; + +/** + * Parse a unified diff into the scorer's {@link SlopChangedFile} shape: one entry per `diff --git` block + * (b-side path — the post-change name), with per-file added/deleted line counts (`+`/`-` bodies only, + * never the `+++`/`---` headers). Tolerant of the 45KB truncation marker the phase-2 apply path appends: + * a truncated tail simply yields fewer counted lines — the parse never throws on any string input. + */ +export function parseDiffChangedFiles(diff: string): SlopChangedFile[] { + const files: SlopChangedFile[] = []; + let current: { path: string; additions: number; deletions: number } | null = null; + for (const line of diff.split("\n")) { + const header = /^diff --git a\/.+ b\/(.+)$/.exec(line); + if (header) { + if (current) files.push(current); + current = { path: header[1]!, additions: 0, deletions: 0 }; + continue; + } + if (!current) continue; + if (line.startsWith("+++") || line.startsWith("---")) continue; + if (line.startsWith("+")) current.additions += 1; + else if (line.startsWith("-")) current.deletions += 1; + } + if (current) files.push(current); + return files; +} + +export type SlopReplayReport = { + replayed: number; + skippedEmptyDiff: number; + skippedNoFiles: number; + skippedDuplicateTarget: number; + /** Histogram of replayed risks by band edge — the evidence summary the issue asks for. */ + riskCounts: { zero: number; low: number; elevated: number; high: number }; + reversed: number; + confirmed: number; + rows: SynthesizedAuditRow[]; +}; + +/** + * Replay the deterministic slop scorer over each source case and synthesize the provenance-tagged + * fired/override pair (ids derive from the targetKey alone, so re-runs upsert idempotently — the phase-1 + * insert builder's ON CONFLICT discipline applies unchanged). The fired event's `occurredAt` reuses the + * source case's own firedAt and the override sits at its decidedAt (floored to 1s after the firing when + * the archive's timestamps collide), so buildBacktestCorpus's strictly-after pairing always matches. + * Deterministic output for deterministic input. + */ +export function replaySlopCorpus(cases: readonly SlopReplaySourceCase[]): SlopReplayReport { + const report: SlopReplayReport = { + replayed: 0, + skippedEmptyDiff: 0, + skippedNoFiles: 0, + skippedDuplicateTarget: 0, + riskCounts: { zero: 0, low: 0, elevated: 0, high: 0 }, + reversed: 0, + confirmed: 0, + rows: [], + }; + const seen = new Set(); + for (const sourceCase of cases) { + if (!sourceCase.diff || !sourceCase.diff.trim()) { + report.skippedEmptyDiff += 1; + continue; + } + if (seen.has(sourceCase.targetKey)) { + report.skippedDuplicateTarget += 1; + continue; + } + const changedFiles = parseDiffChangedFiles(sourceCase.diff); + if (changedFiles.length === 0) { + report.skippedNoFiles += 1; + continue; + } + seen.add(sourceCase.targetKey); + // Signal-subset honesty, enforced by ALLOWLIST rather than trusting undefined-skipping: the scorer + // treats a missing description as an empty one (buildEmptyDescriptionFinding fires on undefined — + // correct live, where absence IS emptiness, but inflating here, where the field simply wasn't + // archived). Only the three diff-derivable signals may contribute; risk and band recompute from + // exactly their weights via the scorer's own exported constants. + const assessment = buildSlopAssessment({ changedFiles }); + const replayableFindings = assessment.findings.filter((finding) => REPLAYABLE_SIGNAL_WEIGHTS[finding.code] !== undefined); + const slopRisk = Math.min( + 100, + replayableFindings.reduce((sum, finding) => sum + REPLAYABLE_SIGNAL_WEIGHTS[finding.code]!, 0), + ); + const band = slopBandFor(slopRisk); + const computedSignals = replayableFindings.map((finding) => finding.code).sort(); + + report.replayed += 1; + if (slopRisk >= 60) report.riskCounts.high += 1; + else if (slopRisk >= 30) report.riskCounts.elevated += 1; + else if (slopRisk > 0) report.riskCounts.low += 1; + else report.riskCounts.zero += 1; + if (sourceCase.label === "reversed") report.reversed += 1; + else report.confirmed += 1; + + const firedMs = Date.parse(sourceCase.firedAt); + const decidedMs = Date.parse(sourceCase.decidedAt); + const overrideIso = new Date( + Number.isFinite(decidedMs) && Number.isFinite(firedMs) && decidedMs > firedMs ? decidedMs : (Number.isFinite(firedMs) ? firedMs : 0) + 1000, + ).toISOString(); + report.rows.push( + { + id: `backfill:${SLOP_BACKFILL_RULE_ID}:${sourceCase.targetKey}:fired`, + eventType: FIRED_EVENT_TYPE, + actor: "loopover", + targetKey: sourceCase.targetKey, + outcome: slopRisk >= 60 ? "above_threshold" : "below_threshold", + detail: `rule ${SLOP_BACKFILL_RULE_ID} replayed against ${sourceCase.targetKey} [backfilled]`, + metadataJson: JSON.stringify({ + confidence: slopRisk / 100, + band, + computedSignals, + backfilled: true, + provenance: SLOP_BACKFILL_PROVENANCE, + }), + createdAt: sourceCase.firedAt, + }, + { + id: `backfill:${SLOP_BACKFILL_RULE_ID}:${sourceCase.targetKey}:override`, + eventType: OVERRIDE_EVENT_TYPE, + actor: "human", + targetKey: sourceCase.targetKey, + outcome: "completed", + detail: `human ${sourceCase.label} rule ${SLOP_BACKFILL_RULE_ID} against ${sourceCase.targetKey} [backfilled]`, + metadataJson: JSON.stringify({ verdict: sourceCase.label, backfilled: true, provenance: SLOP_BACKFILL_PROVENANCE }), + createdAt: overrideIso, + }, + ); + } + return report; +} + +/** Render the dry-run/apply summary — the shape the #8277 evidence comment quotes. */ +export function renderSlopReplayReport(report: SlopReplayReport, mode: "dry-run" | "apply"): string { + return [ + `Slop corpus replay backfill (${mode}) — provenance ${SLOP_BACKFILL_PROVENANCE}`, + ` replayed: ${report.replayed} (confirmed ${report.confirmed}, reversed ${report.reversed})`, + ` risk bands: zero ${report.riskCounts.zero} | low ${report.riskCounts.low} | elevated ${report.riskCounts.elevated} | high ${report.riskCounts.high}`, + ` skipped: empty-diff ${report.skippedEmptyDiff}, no-files ${report.skippedNoFiles}, duplicate-target ${report.skippedDuplicateTarget}`, + mode === "dry-run" ? "dry-run only — re-run with --apply to write. Rows upsert idempotently by id." : ` rows written: ${report.rows.length}`, + ].join("\n"); +} diff --git a/scripts/backfill-slop-corpus.ts b/scripts/backfill-slop-corpus.ts new file mode 100644 index 000000000..a4b9dd743 --- /dev/null +++ b/scripts/backfill-slop-corpus.ts @@ -0,0 +1,99 @@ +#!/usr/bin/env node +// Slop-corpus replay backfill CLI (#8277) — phase 3 of the calibration backfill family. Reads a +// backtest-corpus MANIFEST (backtest-corpus-export.ts's output for ai_consensus_defect — the archived +// raw-context diffs plus human labels), replays the deterministic slop scorer over each diff via the pure +// core (backfill-slop-corpus-core.ts), and — ONLY with --apply — writes the provenance-tagged +// `slop_gate_score` fired/override pairs back. Zero GitHub traffic: every input is already on disk or in +// the store. Mirrors backfill-calibration-corpus.ts's exact wrangler/--pg dual-path + dry-run-default. +// +// tsx scripts/backfill-slop-corpus.ts --corpus corpus.json [--apply] [--db loopover] [--remote] +// tsx scripts/backfill-slop-corpus.ts --corpus corpus.json --apply --pg postgres://… (bare --pg uses DATABASE_URL) +import { readFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import type { BacktestCase } from "@loopover/engine"; +import { openPgDatabase, resolvePgConnection, type PgCliSession } from "./pg-cli.js"; +import { buildBackfillInsertStatements } from "./backfill-calibration-corpus-core.js"; +import { renderSlopReplayReport, replaySlopCorpus, type SlopReplaySourceCase } from "./backfill-slop-corpus-core.js"; + +type Args = { corpus: string | undefined; db: string; remote: boolean; apply: boolean; pgPresent: boolean; pgValue: string | undefined }; + +function parseArgs(argv: string[]): Args { + const args: Args = { corpus: undefined, db: "loopover", remote: false, apply: false, pgPresent: false, pgValue: undefined }; + for (let i = 0; i < argv.length; i += 1) { + const flag = argv[i]; + if (flag === "--corpus") args.corpus = argv[++i]; + else if (flag === "--remote") args.remote = true; + else if (flag === "--apply") args.apply = true; + else if (flag === "--db") args.db = argv[++i]!; + else if (flag === "--pg") { + args.pgPresent = true; + if (argv[i + 1] !== undefined && !argv[i + 1]!.startsWith("--")) args.pgValue = argv[++i]; + } + } + return args; +} + +function d1Execute(db: string, remote: boolean, sql: string): void { + const result = spawnSync("npx", ["wrangler", "d1", "execute", db, remote ? "--remote" : "--local", "--json", "--command", sql], { + encoding: "utf8", + maxBuffer: 256 * 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error(`wrangler d1 execute failed (${result.status}): ${(result.stderr || result.stdout || "").slice(0, 500)}`); + } +} + +/** Project manifest cases into replay sources: only cases carrying a non-empty archived diff qualify + * (the core re-checks and counts, so the numbers stay honest either way). */ +export function manifestToSourceCases(cases: readonly BacktestCase[]): SlopReplaySourceCase[] { + const sources: SlopReplaySourceCase[] = []; + for (const backtestCase of cases) { + const diff = backtestCase.metadata?.diff; + sources.push({ + targetKey: backtestCase.targetKey, + label: backtestCase.label, + firedAt: backtestCase.firedAt, + decidedAt: backtestCase.decidedAt, + diff: typeof diff === "string" ? diff : "", + }); + } + return sources; +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + if (!args.corpus) { + console.error("Usage: tsx scripts/backfill-slop-corpus.ts --corpus [--apply] [--db loopover|--remote|--pg …]"); + return 1; + } + const manifest = JSON.parse(readFileSync(args.corpus, "utf8")) as { cases?: BacktestCase[] }; + if (!Array.isArray(manifest.cases)) { + console.error("--corpus file has no cases[] — expected a backtest-corpus-export manifest."); + return 1; + } + + const report = replaySlopCorpus(manifestToSourceCases(manifest.cases)); + console.log(renderSlopReplayReport(report, args.apply ? "apply" : "dry-run")); + if (!args.apply || report.rows.length === 0) return 0; + + const pgConnection = resolvePgConnection(args.pgPresent, args.pgValue, process.env.DATABASE_URL); + const pgSession: PgCliSession | null = pgConnection ? openPgDatabase(pgConnection) : null; + try { + for (const statement of buildBackfillInsertStatements(report.rows)) { + if (pgSession) await pgSession.db.prepare(statement).run(); + else d1Execute(args.db, args.remote, statement); + } + console.log(`applied ${report.rows.length} row(s) to ${pgSession ? "postgres" : `d1:${args.db}${args.remote ? " (remote)" : ""}`}.`); + return 0; + } finally { + await pgSession?.close(); + } +} + +main().then( + (code) => process.exit(code), + (error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }, +); diff --git a/test/unit/backfill-slop-corpus-core.test.ts b/test/unit/backfill-slop-corpus-core.test.ts new file mode 100644 index 000000000..374da728e --- /dev/null +++ b/test/unit/backfill-slop-corpus-core.test.ts @@ -0,0 +1,140 @@ +import { describe, expect, it } from "vitest"; +import { + parseDiffChangedFiles, + renderSlopReplayReport, + replaySlopCorpus, + SLOP_BACKFILL_PROVENANCE, + SLOP_BACKFILL_RULE_ID, + type SlopReplaySourceCase, +} from "../../scripts/backfill-slop-corpus-core.js"; +import { manifestToSourceCases } from "../../scripts/backfill-slop-corpus.js"; + +// #8277: the replay backfill's pure core. The scorer itself is the engine's own (its suite); these tests +// pin the diff parse, the signal-subset replay discipline (undefined inputs SKIP), the synthesized event +// shapes, and the skip/histogram accounting. + +const SUBSTANTIVE_DIFF = [ + "diff --git a/src/thing.ts b/src/thing.ts", + "--- a/src/thing.ts", + "+++ b/src/thing.ts", + "@@ -1,3 +1,6 @@", + "+export function real(): number {", + "+ return 42;", + "+}", + "-const old = 1;", + "diff --git a/test/unit/thing.test.ts b/test/unit/thing.test.ts", + "--- a/test/unit/thing.test.ts", + "+++ b/test/unit/thing.test.ts", + "@@ -0,0 +1,4 @@", + "+import { real } from '../../src/thing';", + "+it('answers', () => {", + "+ expect(real()).toBe(42);", + "+});", +].join("\n"); + +function sourceCase(over: Partial = {}): SlopReplaySourceCase { + return { + targetKey: "acme/widgets#7", + label: "confirmed", + firedAt: "2026-06-01T00:00:00.000Z", + decidedAt: "2026-06-02T00:00:00.000Z", + diff: SUBSTANTIVE_DIFF, + ...over, + }; +} + +describe("parseDiffChangedFiles (#8277)", () => { + it("yields one entry per diff --git block with +/- body counts, never counting the +++/--- headers", () => { + const files = parseDiffChangedFiles(SUBSTANTIVE_DIFF); + expect(files).toEqual([ + { path: "src/thing.ts", additions: 3, deletions: 1 }, + { path: "test/unit/thing.test.ts", additions: 4, deletions: 0 }, + ]); + }); + + it("never throws on junk: preamble-only text yields no files; a truncated tail just counts fewer lines", () => { + expect(parseDiffChangedFiles("no diff here at all")).toEqual([]); + expect(parseDiffChangedFiles("")).toEqual([]); + const truncated = `${SUBSTANTIVE_DIFF}\n… [diff truncated at 45KB for the audit row]`; + expect(parseDiffChangedFiles(truncated)).toHaveLength(2); + }); +}); + +describe("replaySlopCorpus (#8277)", () => { + it("synthesizes a provenance-tagged fired/override pair with the replayed risk as confidence", () => { + const report = replaySlopCorpus([sourceCase()]); + expect(report.replayed).toBe(1); + expect(report.rows).toHaveLength(2); + const [fired, override] = report.rows; + expect(fired!.id).toBe(`backfill:${SLOP_BACKFILL_RULE_ID}:acme/widgets#7:fired`); + expect(fired!.eventType).toBe(`signal.rule_fired:${SLOP_BACKFILL_RULE_ID}`); + const metadata = JSON.parse(fired!.metadataJson) as { confidence: number; provenance: string; computedSignals: string[]; band: string }; + expect(metadata.provenance).toBe(SLOP_BACKFILL_PROVENANCE); + expect(metadata.confidence).toBeGreaterThanOrEqual(0); + expect(metadata.confidence).toBeLessThanOrEqual(1); + expect(Array.isArray(metadata.computedSignals)).toBe(true); + expect(override!.eventType).toBe(`signal.human_override:${SLOP_BACKFILL_RULE_ID}`); + expect(JSON.parse(override!.metadataJson)).toMatchObject({ verdict: "confirmed", provenance: SLOP_BACKFILL_PROVENANCE }); + expect(override!.createdAt).toBe("2026-06-02T00:00:00.000Z"); // decidedAt honored when after firedAt + }); + + it("signal-subset honesty: a substantive diff with tests fires NOTHING from the unarchived inputs (undefined skips, never inflates)", () => { + const report = replaySlopCorpus([sourceCase()]); + const metadata = JSON.parse(report.rows[0]!.metadataJson) as { confidence: number; computedSignals: string[] }; + // description/commits/cluster/linked-issue are all unarchived — none of their codes may appear. + expect(metadata.computedSignals.join(",")).not.toMatch(/description|commit|duplicate|linked/i); + expect(metadata.confidence).toBe(0); // substantive change + real test file: the diff-derivable signals are clean + expect(report.riskCounts.zero).toBe(1); + }); + + it("counts skips honestly and de-duplicates targets (first case wins)", () => { + const report = replaySlopCorpus([ + sourceCase(), + sourceCase({ label: "reversed" }), // duplicate targetKey + sourceCase({ targetKey: "acme/widgets#8", diff: " " }), + sourceCase({ targetKey: "acme/widgets#9", diff: "prose without any diff blocks" }), + ]); + expect(report.replayed).toBe(1); + expect(report.skippedDuplicateTarget).toBe(1); + expect(report.skippedEmptyDiff).toBe(1); + expect(report.skippedNoFiles).toBe(1); + expect(report.reversed).toBe(0); // the duplicate's label never counted + }); + + it("floors a non-increasing decidedAt to 1s after the firing so the corpus pairing always matches; bands histogram fills", () => { + const whitespaceChurn = [ + "diff --git a/src/a.ts b/src/a.ts", + "--- a/src/a.ts", + "+++ b/src/a.ts", + ...Array.from({ length: 30 }, () => "+ "), + ...Array.from({ length: 30 }, () => "-\t"), + ].join("\n"); + const report = replaySlopCorpus([ + sourceCase({ targetKey: "acme/widgets#10", diff: whitespaceChurn, decidedAt: "2026-05-01T00:00:00.000Z", label: "reversed" }), + ]); + expect(report.rows[1]!.createdAt).toBe("2026-06-01T00:00:01.000Z"); // floored past firedAt + expect(report.riskCounts.zero + report.riskCounts.low + report.riskCounts.elevated + report.riskCounts.high).toBe(1); + const metadata = JSON.parse(report.rows[0]!.metadataJson) as { confidence: number }; + expect(metadata.confidence).toBeGreaterThan(0); // churn-only diff scores above zero + expect(report.reversed).toBe(1); + }); + + it("manifestToSourceCases projects manifest cases; a missing/non-string diff degrades to empty (counted, not thrown)", () => { + const cases = manifestToSourceCases([ + { ruleId: "r", targetKey: "a/b#1", outcome: "close", label: "confirmed", firedAt: "f", decidedAt: "d", metadata: { diff: SUBSTANTIVE_DIFF } }, + { ruleId: "r", targetKey: "a/b#2", outcome: "close", label: "reversed", firedAt: "f", decidedAt: "d", metadata: { diff: 42 } }, + { ruleId: "r", targetKey: "a/b#3", outcome: "close", label: "reversed", firedAt: "f", decidedAt: "d" }, + ]); + expect(cases.map((c) => c.diff === "")).toEqual([false, true, true]); + const report = replaySlopCorpus(cases); + expect(report.replayed).toBe(1); + expect(report.skippedEmptyDiff).toBe(2); + }); + + it("renders both report modes with the provenance line", () => { + const report = replaySlopCorpus([sourceCase()]); + expect(renderSlopReplayReport(report, "dry-run")).toContain("dry-run only"); + expect(renderSlopReplayReport(report, "apply")).toContain("rows written: 2"); + expect(renderSlopReplayReport(report, "apply")).toContain(SLOP_BACKFILL_PROVENANCE); + }); +}); From 5798dc774aa1dc318672d58c4e23e09a3ebc6f93 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:07:41 -0700 Subject: [PATCH 2/4] feat(calibration): replay-derived provider track records adapter (#8278) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Maps the #8221 harness's cached per-fixture verdicts onto ProviderReviewSignal (would_flag => fail, would_not_flag => pass, abstentions yield NO signal) and aggregates with computeProviderTrackRecords — the identical function live reviewer_vote rows feed. Offline report only: nothing persists, and the rendered table carries the replay-derived disclaimer (#8278's segregation rule). First same-seed two-provider table recorded on #8229. --- scripts/provider-replay-track-record.ts | 124 ++++++++++++++++++ .../unit/provider-replay-track-record.test.ts | 56 ++++++++ 2 files changed, 180 insertions(+) create mode 100644 scripts/provider-replay-track-record.ts create mode 100644 test/unit/provider-replay-track-record.test.ts diff --git a/scripts/provider-replay-track-record.ts b/scripts/provider-replay-track-record.ts new file mode 100644 index 000000000..bac59c767 --- /dev/null +++ b/scripts/provider-replay-track-record.ts @@ -0,0 +1,124 @@ +#!/usr/bin/env node +// Replay-derived provider track records (#8278, seeding #8229 stage 1). Reads the counterfactual replay +// harness's cached raw outputs (the #8221 artifacts dir) for one or more variants over the SAME seeded +// fixture sample, maps each parsed verdict onto the consensus vote vocabulary, and aggregates with +// `computeProviderTrackRecords` (#8228) — the identical function live votes will feed, zero new math. +// REPLAY-DERIVED, NEVER LIVE: signals exist only in this offline report; nothing is written to any store +// and nothing masquerades as a live reviewer_vote event (the #8278 segregation requirement). +// +// tsx scripts/provider-replay-track-record.ts --fixtures corpus.json --artifacts dir \ +// --variant [--variant …] [--seed-suffix s] [--max-fixtures N] +// +// Verdict → vote mapping: would_flag ⇒ "fail" (the defect-flagging vote), would_not_flag ⇒ "pass", +// abstained ⇒ NO signal (an abstention is not a vote — the same never-coerced discipline as scoring). +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + computeProviderTrackRecords, + COUNTERFACTUAL_SAMPLE_SEED_PREFIX, + type BacktestCase, + type CounterfactualVariant, + type ProviderReviewSignal, + type ProviderTrackRecord, +} from "@loopover/engine"; +import { artifactKey, parseVariantVerdict, planReplay, type CounterfactualReplayPlan } from "./counterfactual-replay-core.js"; + +/** PURE: map one variant's cached raw outputs onto provider signals over the shared plan. The provider id + * is the variant's modelSpec (the reviewer identity live votes carry); abstentions and uncached fixtures + * yield no signal, counted separately so the report can say how much of the sample actually voted. */ +export function artifactsToProviderSignals( + plan: CounterfactualReplayPlan, + variant: CounterfactualVariant, + readArtifact: (key: string) => string | null, +): { signals: ProviderReviewSignal[]; abstained: number; uncached: number } { + const signals: ProviderReviewSignal[] = []; + let abstained = 0; + let uncached = 0; + for (const fixture of plan.fixtures) { + const raw = readArtifact(artifactKey(variant, fixture.fixtureId)); + if (raw === null) { + uncached += 1; + continue; + } + const verdict = parseVariantVerdict(raw); + if (verdict === "abstained") { + abstained += 1; + continue; + } + signals.push({ + provider: variant.modelSpec, + repoFullName: fixture.fixtureId.split("#")[0] ?? fixture.fixtureId, + targetKey: fixture.fixtureId, + vote: verdict === "would_flag" ? "fail" : "pass", + }); + } + return { signals, abstained, uncached }; +} + +/** Render the overall-rollup rows (repoFullName null) as the markdown table #8229's stage-1 comment quotes. */ +export function renderProviderTable(records: readonly ProviderTrackRecord[]): string { + const overall = records.filter((record) => record.repoFullName === null); + const lines = [ + "| Provider | Signals | Decided | Precision (fail⇒confirmed) | Agreement | Consensus |", + "| --- | --- | --- | --- | --- | --- |", + ]; + const fmt = (value: number | null): string => (value === null ? "n/a" : value.toFixed(3)); + for (const record of overall) { + lines.push( + `| ${record.provider} | ${record.signals} | ${record.decided} | ${fmt(record.precision)} | ${fmt(record.agreementRate)} | ${fmt(record.consensusRate)} |`, + ); + } + return lines.join("\n"); +} + +function parseArgs(argv: string[]): { fixtures?: string; artifacts: string; variants: string[]; seedSuffix: string; maxFixtures: number } { + const args = { fixtures: undefined as string | undefined, artifacts: ".counterfactual-artifacts", variants: [] as string[], seedSuffix: "default", maxFixtures: 500 }; + for (let i = 0; i < argv.length; i += 1) { + const flag = argv[i]; + if (flag === "--fixtures") args.fixtures = argv[++i]; + else if (flag === "--artifacts") args.artifacts = argv[++i] ?? args.artifacts; + else if (flag === "--variant") args.variants.push(argv[++i] ?? ""); + else if (flag === "--seed-suffix") args.seedSuffix = argv[++i] ?? "default"; + else if (flag === "--max-fixtures") args.maxFixtures = Number(argv[++i]); + } + return args; +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + if (!args.fixtures || args.variants.length === 0 || args.variants.some((variant) => !variant.includes("@"))) { + console.error("Usage: tsx scripts/provider-replay-track-record.ts --fixtures --artifacts --variant [--variant …]"); + return 1; + } + const manifest = JSON.parse(readFileSync(args.fixtures, "utf8")) as { cases?: BacktestCase[] }; + if (!Array.isArray(manifest.cases)) { + console.error("--fixtures file has no cases[] — expected a backtest-corpus-export manifest."); + return 1; + } + const plan = planReplay(manifest.cases, { seed: `${COUNTERFACTUAL_SAMPLE_SEED_PREFIX}:${args.seedSuffix}`, maxFixtures: args.maxFixtures }); + + const allSignals: ProviderReviewSignal[] = []; + for (const raw of args.variants) { + const [promptVersion, ...modelParts] = raw.split("@"); + const variant: CounterfactualVariant = { promptVersion: promptVersion!, modelSpec: modelParts.join("@") }; + const { signals, abstained, uncached } = artifactsToProviderSignals(plan, variant, (key) => { + const path = join(args.artifacts, `${key}.txt`); + return existsSync(path) ? readFileSync(path, "utf8") : null; + }); + console.log(`${raw}: ${signals.length} signal(s), ${abstained} abstained, ${uncached} uncached (of ${plan.fixtures.length} planned)`); + allSignals.push(...signals); + } + + console.log(""); + console.log(renderProviderTable(computeProviderTrackRecords(allSignals, manifest.cases))); + console.log("\nREPLAY-DERIVED (offline #8221 artifacts) — not live reviewer votes; nothing was persisted."); + return 0; +} + +main().then( + (code) => process.exit(code), + (error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }, +); diff --git a/test/unit/provider-replay-track-record.test.ts b/test/unit/provider-replay-track-record.test.ts new file mode 100644 index 000000000..be837c885 --- /dev/null +++ b/test/unit/provider-replay-track-record.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import type { BacktestCase } from "@loopover/engine"; +import { computeProviderTrackRecords } from "@loopover/engine"; +import { artifactKey, planReplay } from "../../scripts/counterfactual-replay-core.js"; +import { artifactsToProviderSignals, renderProviderTable } from "../../scripts/provider-replay-track-record.js"; + +// #8278: the replay→signals adapter. computeProviderTrackRecords has its own engine suite; these tests pin +// the verdict→vote mapping, the abstention/uncached accounting, and the overall-rollup table rendering. + +function corpusCase(id: number, label: "confirmed" | "reversed"): BacktestCase { + return { + ruleId: "ai_consensus_defect", + targetKey: `acme/widgets#${id}`, + outcome: "close", + label, + firedAt: "2026-06-01T00:00:00.000Z", + decidedAt: "2026-06-02T00:00:00.000Z", + metadata: { diff: "diff --git a/x b/x" }, + }; +} + +const SAMPLING = { seed: "counterfactual-replay-v1:test", maxFixtures: 100 }; +const VARIANT = { promptVersion: "minimal-judge", modelSpec: "qwen3:8b" }; + +describe("artifactsToProviderSignals (#8278)", () => { + it("maps would_flag→fail, would_not_flag→pass; abstentions and uncached fixtures yield NO signal, counted separately", () => { + const cases = [corpusCase(1, "reversed"), corpusCase(2, "confirmed"), corpusCase(3, "confirmed"), corpusCase(4, "reversed")]; + const plan = planReplay(cases, SAMPLING); + const byKey: Record = { + [artifactKey(VARIANT, "acme/widgets#1")]: '{"blockers": ["real"]}', + [artifactKey(VARIANT, "acme/widgets#2")]: '{"blockers": []}', + [artifactKey(VARIANT, "acme/widgets#3")]: "no json at all", // abstains + // #4 uncached + }; + const { signals, abstained, uncached } = artifactsToProviderSignals(plan, VARIANT, (key) => byKey[key] ?? null); + expect(abstained).toBe(1); + expect(uncached).toBe(1); + expect(signals).toEqual([ + { provider: "qwen3:8b", repoFullName: "acme/widgets", targetKey: "acme/widgets#1", vote: "fail" }, + { provider: "qwen3:8b", repoFullName: "acme/widgets", targetKey: "acme/widgets#2", vote: "pass" }, + ]); + // The adapter's output feeds the engine aggregation directly: one decided fail on a reversed label. + const overall = computeProviderTrackRecords(signals, cases).find((record) => record.repoFullName === null)!; + expect(overall.decided).toBe(2); + expect(overall.precision).toBe(0); // the lone fail vote landed on a reversed... label "reversed" means the firing was wrong + }); + + it("renderProviderTable renders only the overall rollups with n/a for null rates", () => { + const table = renderProviderTable([ + { provider: "a", repoFullName: null, signals: 2, decided: 2, confirmed: 1, reversed: 1, precision: 0.5, agreementRate: null, consensusRate: null, splitRate: null }, + { provider: "a", repoFullName: "acme/widgets", signals: 2, decided: 2, confirmed: 1, reversed: 1, precision: 0.5, agreementRate: 0.5, consensusRate: null, splitRate: null }, + ]); + expect(table).toContain("| a | 2 | 2 | 0.500 | n/a | n/a |"); + expect(table.split("\n")).toHaveLength(3); // header + divider + ONE overall row (per-repo rows excluded) + }); +}); From e564d71ae4677d3389e1920b06951f5d0b2789c4 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:09:17 -0700 Subject: [PATCH 3/4] fix(scripts): exactOptionalPropertyTypes on the provider-replay arg parser --- scripts/provider-replay-track-record.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/provider-replay-track-record.ts b/scripts/provider-replay-track-record.ts index bac59c767..fe40d8f0e 100644 --- a/scripts/provider-replay-track-record.ts +++ b/scripts/provider-replay-track-record.ts @@ -71,7 +71,7 @@ export function renderProviderTable(records: readonly ProviderTrackRecord[]): st return lines.join("\n"); } -function parseArgs(argv: string[]): { fixtures?: string; artifacts: string; variants: string[]; seedSuffix: string; maxFixtures: number } { +function parseArgs(argv: string[]): { fixtures: string | undefined; artifacts: string; variants: string[]; seedSuffix: string; maxFixtures: number } { const args = { fixtures: undefined as string | undefined, artifacts: ".counterfactual-artifacts", variants: [] as string[], seedSuffix: "default", maxFixtures: 500 }; for (let i = 0; i < argv.length; i += 1) { const flag = argv[i]; From 041c208565fddaf1350d1f270e99c589a6671406 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:57:55 -0700 Subject: [PATCH 4/4] fix(scripts): entry-guard the phase-3 CLI wrappers so test imports never process.exit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both wrappers export pure helpers the unit suites import; an import-time main().then(process.exit) fails the whole vitest run as an unhandled rejection even with every test green (CI shard 2's exact failure — all 6568 tests passed). The audit-quality-gate-min-score.ts direct-execution idiom fixes it; dry-run CLI behavior unchanged. --- scripts/backfill-slop-corpus.ts | 19 ++++++++++++------- scripts/provider-replay-track-record.ts | 19 ++++++++++++------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/scripts/backfill-slop-corpus.ts b/scripts/backfill-slop-corpus.ts index a4b9dd743..cf81be48b 100644 --- a/scripts/backfill-slop-corpus.ts +++ b/scripts/backfill-slop-corpus.ts @@ -90,10 +90,15 @@ async function main(): Promise { } } -main().then( - (code) => process.exit(code), - (error) => { - console.error(error instanceof Error ? error.message : String(error)); - process.exit(1); - }, -); +// Entry guard (the audit-quality-gate-min-score.ts idiom): tests import this module's exported helpers, +// so main() must run ONLY under direct execution — an import-time process.exit fails the whole vitest run +// as an unhandled rejection even with every test green. +if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href) { + main().then( + (code) => process.exit(code), + (error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }, + ); +} diff --git a/scripts/provider-replay-track-record.ts b/scripts/provider-replay-track-record.ts index fe40d8f0e..0983447d2 100644 --- a/scripts/provider-replay-track-record.ts +++ b/scripts/provider-replay-track-record.ts @@ -115,10 +115,15 @@ async function main(): Promise { return 0; } -main().then( - (code) => process.exit(code), - (error) => { - console.error(error instanceof Error ? error.message : String(error)); - process.exit(1); - }, -); +// Entry guard (the audit-quality-gate-min-score.ts idiom): tests import this module's exported helpers, +// so main() must run ONLY under direct execution — an import-time process.exit fails the whole vitest run +// as an unhandled rejection even with every test green. +if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href) { + main().then( + (code) => process.exit(code), + (error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }, + ); +}