feat(calibration): slop-corpus replay backfill + replay-derived provider track records

scripts/backfill-slop-corpus-core.ts

@@ -0,0 +1,182 @@
+// Pure core for the slop-corpus replay backfill (#8277) — phase 3 of the calibration backfill family.
+// The slop scorer is deterministic and in-repo, so the #8224 report-only knob's evidence can be
+// manufactured honestly: replay `buildSlopAssessment` over each archived PR diff (the #8130/#8170
+// raw-context corpus) and synthesize provenance-tagged `slop_gate_score` fired/override pairs labeled by
+// what humans actually did — the same counterfactual "mapping a" framing the phase-1 close-confidence
+// backfill documented. This core is transform-only (no IO), mirroring backfill-calibration-corpus-core.ts;
+// the CLI wrapper owns the manifest read and the wrangler/pg writes.
+//
+// SIGNAL-SUBSET HONESTY: only the diff-derivable signals can replay (trivialWhitespaceChurn,
+// missingTestEvidence, nonSubstantivePadding — changedFiles parse out of the archived unified diff). The
+// unarchived inputs (description, commit messages, duplicate-cluster membership, linked-issue state) are
+// passed UNDEFINED so their signals SKIP rather than fire spuriously — a replayed score is therefore a
+// LOWER BOUND on what live scoring would have produced. Every synthesized row records the computed signal
+// codes plus the provenance tag so the eventual flip-to-live decision can weigh exactly that.
+
+import { buildSlopAssessment, SLOP_WEIGHTS, slopBandFor, type SlopChangedFile } from "../packages/loopover-engine/src/signals/slop";
+import type { SynthesizedAuditRow } from "./backfill-calibration-corpus-core.js";
+
+export const SLOP_BACKFILL_RULE_ID = "slop_gate_score";
+export const SLOP_BACKFILL_PROVENANCE = "slop_replay_backfill_v1";
+const FIRED_EVENT_TYPE = `signal.rule_fired:${SLOP_BACKFILL_RULE_ID}`;
+const OVERRIDE_EVENT_TYPE = `signal.human_override:${SLOP_BACKFILL_RULE_ID}`;
+
+/** One replayable source case, projected from a backtest-corpus manifest (backtest-corpus-export.ts):
+ *  the archived bounded diff plus the human label the original rule's history already established. */
+/** The diff-derivable signal codes the replay may score, mapped to the scorer's own weights. Everything
+ *  else needs unarchived inputs and is EXCLUDED even if the scorer fires it on an undefined field. */
+const REPLAYABLE_SIGNAL_WEIGHTS: Record<string, number | undefined> = {
+  trivial_whitespace_churn: SLOP_WEIGHTS.trivialWhitespaceChurn,
+  missing_test_evidence: SLOP_WEIGHTS.missingTestEvidence,
+  non_substantive_padding: SLOP_WEIGHTS.nonSubstantivePadding,
+};
+
+export type SlopReplaySourceCase = {
+  targetKey: string;
+  label: "confirmed" | "reversed";
+  firedAt: string;
+  decidedAt: string;
+  diff: string;
+};
+
+/**
+ * Parse a unified diff into the scorer's {@link SlopChangedFile} shape: one entry per `diff --git` block
+ * (b-side path — the post-change name), with per-file added/deleted line counts (`+`/`-` bodies only,
+ * never the `+++`/`---` headers). Tolerant of the 45KB truncation marker the phase-2 apply path appends:
+ * a truncated tail simply yields fewer counted lines — the parse never throws on any string input.
+ */
+export function parseDiffChangedFiles(diff: string): SlopChangedFile[] {
+  const files: SlopChangedFile[] = [];
+  let current: { path: string; additions: number; deletions: number } | null = null;
+  for (const line of diff.split("\n")) {
+    const header = /^diff --git a\/.+ b\/(.+)$/.exec(line);
+    if (header) {
+      if (current) files.push(current);
+      current = { path: header[1]!, additions: 0, deletions: 0 };
+      continue;
+    }
+    if (!current) continue;
+    if (line.startsWith("+++") || line.startsWith("---")) continue;
+    if (line.startsWith("+")) current.additions += 1;
+    else if (line.startsWith("-")) current.deletions += 1;
+  }
+  if (current) files.push(current);
+  return files;
+}
+
+export type SlopReplayReport = {
+  replayed: number;
+  skippedEmptyDiff: number;
+  skippedNoFiles: number;
+  skippedDuplicateTarget: number;
+  /** Histogram of replayed risks by band edge — the evidence summary the issue asks for. */
+  riskCounts: { zero: number; low: number; elevated: number; high: number };
+  reversed: number;
+  confirmed: number;
+  rows: SynthesizedAuditRow[];
+};
+
+/**
+ * Replay the deterministic slop scorer over each source case and synthesize the provenance-tagged
+ * fired/override pair (ids derive from the targetKey alone, so re-runs upsert idempotently — the phase-1
+ * insert builder's ON CONFLICT discipline applies unchanged). The fired event's `occurredAt` reuses the
+ * source case's own firedAt and the override sits at its decidedAt (floored to 1s after the firing when
+ * the archive's timestamps collide), so buildBacktestCorpus's strictly-after pairing always matches.
+ * Deterministic output for deterministic input.
+ */
+export function replaySlopCorpus(cases: readonly SlopReplaySourceCase[]): SlopReplayReport {
+  const report: SlopReplayReport = {
+    replayed: 0,
+    skippedEmptyDiff: 0,
+    skippedNoFiles: 0,
+    skippedDuplicateTarget: 0,
+    riskCounts: { zero: 0, low: 0, elevated: 0, high: 0 },
+    reversed: 0,
+    confirmed: 0,
+    rows: [],
+  };
+  const seen = new Set<string>();
+  for (const sourceCase of cases) {
+    if (!sourceCase.diff || !sourceCase.diff.trim()) {
+      report.skippedEmptyDiff += 1;
+      continue;
+    }
+    if (seen.has(sourceCase.targetKey)) {
+      report.skippedDuplicateTarget += 1;
+      continue;
+    }
+    const changedFiles = parseDiffChangedFiles(sourceCase.diff);
+    if (changedFiles.length === 0) {
+      report.skippedNoFiles += 1;
+      continue;
+    }
+    seen.add(sourceCase.targetKey);
+    // Signal-subset honesty, enforced by ALLOWLIST rather than trusting undefined-skipping: the scorer
+    // treats a missing description as an empty one (buildEmptyDescriptionFinding fires on undefined —
+    // correct live, where absence IS emptiness, but inflating here, where the field simply wasn't
+    // archived). Only the three diff-derivable signals may contribute; risk and band recompute from
+    // exactly their weights via the scorer's own exported constants.
+    const assessment = buildSlopAssessment({ changedFiles });
+    const replayableFindings = assessment.findings.filter((finding) => REPLAYABLE_SIGNAL_WEIGHTS[finding.code] !== undefined);
+    const slopRisk = Math.min(
+      100,
+      replayableFindings.reduce((sum, finding) => sum + REPLAYABLE_SIGNAL_WEIGHTS[finding.code]!, 0),
+    );
+    const band = slopBandFor(slopRisk);
+    const computedSignals = replayableFindings.map((finding) => finding.code).sort();
+
+    report.replayed += 1;
+    if (slopRisk >= 60) report.riskCounts.high += 1;
+    else if (slopRisk >= 30) report.riskCounts.elevated += 1;
+    else if (slopRisk > 0) report.riskCounts.low += 1;
+    else report.riskCounts.zero += 1;
+    if (sourceCase.label === "reversed") report.reversed += 1;
+    else report.confirmed += 1;
+
+    const firedMs = Date.parse(sourceCase.firedAt);
+    const decidedMs = Date.parse(sourceCase.decidedAt);
+    const overrideIso = new Date(
+      Number.isFinite(decidedMs) && Number.isFinite(firedMs) && decidedMs > firedMs ? decidedMs : (Number.isFinite(firedMs) ? firedMs : 0) + 1000,
+    ).toISOString();
+    report.rows.push(
+      {
+        id: `backfill:${SLOP_BACKFILL_RULE_ID}:${sourceCase.targetKey}:fired`,
+        eventType: FIRED_EVENT_TYPE,
+        actor: "loopover",
+        targetKey: sourceCase.targetKey,
+        outcome: slopRisk >= 60 ? "above_threshold" : "below_threshold",
+        detail: `rule ${SLOP_BACKFILL_RULE_ID} replayed against ${sourceCase.targetKey} [backfilled]`,
+        metadataJson: JSON.stringify({
+          confidence: slopRisk / 100,
+          band,
+          computedSignals,
+          backfilled: true,
+          provenance: SLOP_BACKFILL_PROVENANCE,
+        }),
+        createdAt: sourceCase.firedAt,
+      },
+      {
+        id: `backfill:${SLOP_BACKFILL_RULE_ID}:${sourceCase.targetKey}:override`,
+        eventType: OVERRIDE_EVENT_TYPE,
+        actor: "human",
+        targetKey: sourceCase.targetKey,
+        outcome: "completed",
+        detail: `human ${sourceCase.label} rule ${SLOP_BACKFILL_RULE_ID} against ${sourceCase.targetKey} [backfilled]`,
+        metadataJson: JSON.stringify({ verdict: sourceCase.label, backfilled: true, provenance: SLOP_BACKFILL_PROVENANCE }),
+        createdAt: overrideIso,
+      },
+    );
+  }
+  return report;
+}
+
+/** Render the dry-run/apply summary — the shape the #8277 evidence comment quotes. */
+export function renderSlopReplayReport(report: SlopReplayReport, mode: "dry-run" | "apply"): string {
+  return [
+    `Slop corpus replay backfill (${mode}) — provenance ${SLOP_BACKFILL_PROVENANCE}`,
+    `  replayed: ${report.replayed} (confirmed ${report.confirmed}, reversed ${report.reversed})`,
+    `  risk bands: zero ${report.riskCounts.zero} | low ${report.riskCounts.low} | elevated ${report.riskCounts.elevated} | high ${report.riskCounts.high}`,
+    `  skipped: empty-diff ${report.skippedEmptyDiff}, no-files ${report.skippedNoFiles}, duplicate-target ${report.skippedDuplicateTarget}`,
+    mode === "dry-run" ? "dry-run only — re-run with --apply to write. Rows upsert idempotently by id." : `  rows written: ${report.rows.length}`,
+  ].join("\n");
+}

scripts/backfill-slop-corpus.ts

@@ -0,0 +1,104 @@
+#!/usr/bin/env node
+// Slop-corpus replay backfill CLI (#8277) — phase 3 of the calibration backfill family. Reads a
+// backtest-corpus MANIFEST (backtest-corpus-export.ts's output for ai_consensus_defect — the archived
+// raw-context diffs plus human labels), replays the deterministic slop scorer over each diff via the pure
+// core (backfill-slop-corpus-core.ts), and — ONLY with --apply — writes the provenance-tagged
+// `slop_gate_score` fired/override pairs back. Zero GitHub traffic: every input is already on disk or in
+// the store. Mirrors backfill-calibration-corpus.ts's exact wrangler/--pg dual-path + dry-run-default.
+//
+//   tsx scripts/backfill-slop-corpus.ts --corpus corpus.json [--apply] [--db loopover] [--remote]
+//   tsx scripts/backfill-slop-corpus.ts --corpus corpus.json --apply --pg postgres://…   (bare --pg uses DATABASE_URL)
+import { readFileSync } from "node:fs";
+import { spawnSync } from "node:child_process";
+import type { BacktestCase } from "@loopover/engine";
+import { openPgDatabase, resolvePgConnection, type PgCliSession } from "./pg-cli.js";
+import { buildBackfillInsertStatements } from "./backfill-calibration-corpus-core.js";
+import { renderSlopReplayReport, replaySlopCorpus, type SlopReplaySourceCase } from "./backfill-slop-corpus-core.js";
+
+type Args = { corpus: string | undefined; db: string; remote: boolean; apply: boolean; pgPresent: boolean; pgValue: string | undefined };
+
+function parseArgs(argv: string[]): Args {
+  const args: Args = { corpus: undefined, db: "loopover", remote: false, apply: false, pgPresent: false, pgValue: undefined };
+  for (let i = 0; i < argv.length; i += 1) {
+    const flag = argv[i];
+    if (flag === "--corpus") args.corpus = argv[++i];
+    else if (flag === "--remote") args.remote = true;
+    else if (flag === "--apply") args.apply = true;
+    else if (flag === "--db") args.db = argv[++i]!;
+    else if (flag === "--pg") {
+      args.pgPresent = true;
+      if (argv[i + 1] !== undefined && !argv[i + 1]!.startsWith("--")) args.pgValue = argv[++i];
+    }
+  }
+  return args;
+}
+
+function d1Execute(db: string, remote: boolean, sql: string): void {
+  const result = spawnSync("npx", ["wrangler", "d1", "execute", db, remote ? "--remote" : "--local", "--json", "--command", sql], {
+    encoding: "utf8",
+    maxBuffer: 256 * 1024 * 1024,
+  });
+  if (result.status !== 0) {
+    throw new Error(`wrangler d1 execute failed (${result.status}): ${(result.stderr || result.stdout || "").slice(0, 500)}`);
+  }
+}
+
+/** Project manifest cases into replay sources: only cases carrying a non-empty archived diff qualify
+ *  (the core re-checks and counts, so the numbers stay honest either way). */
+export function manifestToSourceCases(cases: readonly BacktestCase[]): SlopReplaySourceCase[] {
+  const sources: SlopReplaySourceCase[] = [];
+  for (const backtestCase of cases) {
+    const diff = backtestCase.metadata?.diff;
+    sources.push({
+      targetKey: backtestCase.targetKey,
+      label: backtestCase.label,
+      firedAt: backtestCase.firedAt,
+      decidedAt: backtestCase.decidedAt,
+      diff: typeof diff === "string" ? diff : "",
+    });
+  }
+  return sources;
+}
+
+async function main(): Promise<number> {
+  const args = parseArgs(process.argv.slice(2));
+  if (!args.corpus) {
+    console.error("Usage: tsx scripts/backfill-slop-corpus.ts --corpus <manifest.json> [--apply] [--db loopover|--remote|--pg …]");
+    return 1;
+  }
+  const manifest = JSON.parse(readFileSync(args.corpus, "utf8")) as { cases?: BacktestCase[] };
+  if (!Array.isArray(manifest.cases)) {
+    console.error("--corpus file has no cases[] — expected a backtest-corpus-export manifest.");
+    return 1;
+  }
+
+  const report = replaySlopCorpus(manifestToSourceCases(manifest.cases));
+  console.log(renderSlopReplayReport(report, args.apply ? "apply" : "dry-run"));
+  if (!args.apply || report.rows.length === 0) return 0;
+
+  const pgConnection = resolvePgConnection(args.pgPresent, args.pgValue, process.env.DATABASE_URL);
+  const pgSession: PgCliSession | null = pgConnection ? openPgDatabase(pgConnection) : null;
+  try {
+    for (const statement of buildBackfillInsertStatements(report.rows)) {
+      if (pgSession) await pgSession.db.prepare(statement).run();
+      else d1Execute(args.db, args.remote, statement);
+    }
+    console.log(`applied ${report.rows.length} row(s) to ${pgSession ? "postgres" : `d1:${args.db}${args.remote ? " (remote)" : ""}`}.`);
+    return 0;
+  } finally {
+    await pgSession?.close();
+  }
+}
+
+// Entry guard (the audit-quality-gate-min-score.ts idiom): tests import this module's exported helpers,
+// so main() must run ONLY under direct execution — an import-time process.exit fails the whole vitest run
+// as an unhandled rejection even with every test green.
+if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href) {
+  main().then(
+    (code) => process.exit(code),
+    (error) => {
+      console.error(error instanceof Error ? error.message : String(error));
+      process.exit(1);
+    },
+  );
+}