diff --git a/.github/workflows/counterfactual-replay-check.yml b/.github/workflows/counterfactual-replay-check.yml index e5139b470..026b3fa28 100644 --- a/.github/workflows/counterfactual-replay-check.yml +++ b/.github/workflows/counterfactual-replay-check.yml @@ -36,18 +36,22 @@ jobs: timeout-minutes: 20 steps: # The budget gate comes FIRST so an unconfigured deployment pays one echo, not an npm ci. + # #8279: the default provider is Workers AI (the CLOUDFLARE_* secrets this workflow already holds for + # the corpus export double as its credentials — the token needs Workers AI run perms); an explicit + # COUNTERFACTUAL_OLLAMA_URL secret switches to a reachable ollama instead. - name: Check replay budget configuration id: budget env: REPLAY_BUDGET: ${{ vars.COUNTERFACTUAL_REPLAY_BUDGET }} REPLAY_MODEL: ${{ vars.COUNTERFACTUAL_REPLAY_MODEL }} - REPLAY_PROVIDER_URL: ${{ secrets.COUNTERFACTUAL_OLLAMA_URL }} + REPLAY_OLLAMA_URL: ${{ secrets.COUNTERFACTUAL_OLLAMA_URL }} run: | - if [ -n "$REPLAY_BUDGET" ] && [ -n "$REPLAY_MODEL" ] && [ -n "$REPLAY_PROVIDER_URL" ]; then + if [ -n "$REPLAY_BUDGET" ] && [ -n "$REPLAY_MODEL" ]; then echo "configured=true" >> "$GITHUB_OUTPUT" + if [ -n "$REPLAY_OLLAMA_URL" ]; then echo "provider=ollama" >> "$GITHUB_OUTPUT"; else echo "provider=workers-ai" >> "$GITHUB_OUTPUT"; fi else echo "configured=false" >> "$GITHUB_OUTPUT" - echo "::notice::Counterfactual replay skipped: no replay budget configured (set the COUNTERFACTUAL_REPLAY_BUDGET + COUNTERFACTUAL_REPLAY_MODEL repo variables and the COUNTERFACTUAL_OLLAMA_URL secret to enable). Advisory only, never fails the PR." + echo "::notice::Counterfactual replay skipped: no replay budget configured (set the COUNTERFACTUAL_REPLAY_BUDGET + COUNTERFACTUAL_REPLAY_MODEL repo variables; Workers AI runs on the existing CLOUDFLARE_* secrets, or set COUNTERFACTUAL_OLLAMA_URL to use a reachable ollama). Advisory only, never fails the PR." fi - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 @@ -123,7 +127,8 @@ jobs: CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} REPLAY_BUDGET: ${{ vars.COUNTERFACTUAL_REPLAY_BUDGET }} REPLAY_MODEL: ${{ vars.COUNTERFACTUAL_REPLAY_MODEL }} - REPLAY_PROVIDER_URL: ${{ secrets.COUNTERFACTUAL_OLLAMA_URL }} + REPLAY_OLLAMA_URL: ${{ secrets.COUNTERFACTUAL_OLLAMA_URL }} + REPLAY_PROVIDER: ${{ steps.budget.outputs.provider }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} BASE_SHA: ${{ github.event.pull_request.base.sha }} PR_NUMBER: ${{ github.event.pull_request.number }} @@ -136,7 +141,8 @@ jobs: --prompt-file base-prompt.txt \ --budget "$REPLAY_BUDGET" \ --seed-suffix "$HEAD_SHA" \ - --ollama-url "$REPLAY_PROVIDER_URL" \ + --provider "$REPLAY_PROVIDER" \ + --ollama-url "${REPLAY_OLLAMA_URL:-http://localhost:11434}" \ --out base-scores.json \ && npx tsx scripts/counterfactual-replay.ts \ --fixtures replay-corpus.json \ @@ -144,7 +150,8 @@ jobs: --prompt-file head-prompt.txt \ --budget "$REPLAY_BUDGET" \ --seed-suffix "$HEAD_SHA" \ - --ollama-url "$REPLAY_PROVIDER_URL" \ + --provider "$REPLAY_PROVIDER" \ + --ollama-url "${REPLAY_OLLAMA_URL:-http://localhost:11434}" \ --baseline base-scores.json \ --base-variant-label "$BASE_VERSION" \ --comment-out replay-comment.md \ diff --git a/apps/loopover-ui/content/docs/backtest-calibration.mdx b/apps/loopover-ui/content/docs/backtest-calibration.mdx index 3cee9b054..28a0eb508 100644 --- a/apps/loopover-ui/content/docs/backtest-calibration.mdx +++ b/apps/loopover-ui/content/docs/backtest-calibration.mdx @@ -193,8 +193,12 @@ base and head checkouts, replays both over the identical seeded sample (seed = h per push are deterministic), posts the comparison as an update-in-place PR comment, and persists a `calibration.counterfactual_backtest_run` event feeding the same REGRESSED-verdict track record as the threshold and logic backtests. The check spends nothing unless the deployment explicitly configures a -replay budget (repo variables `COUNTERFACTUAL_REPLAY_BUDGET`/`COUNTERFACTUAL_REPLAY_MODEL` plus the -provider endpoint secret) — otherwise it posts a visible "skipped: no replay budget configured" notice. +replay budget (repo variables `COUNTERFACTUAL_REPLAY_BUDGET`/`COUNTERFACTUAL_REPLAY_MODEL`) — otherwise +it posts a visible "skipped: no replay budget configured" notice. Provider selection: with a +`COUNTERFACTUAL_OLLAMA_URL` secret set, the check uses that ollama endpoint (a self-hosted runner +reaching its own local model is the zero-marginal-cost shape); without it, the default is the flag-gated +Workers AI fallback on the deployment's existing Cloudflare credentials — OSS models, usage-priced, +dormant until the budget variables exist. Advisory forever until a separate, explicit authority decision, exactly like every other backtest gate. ## Self-hosting diff --git a/scripts/counterfactual-replay-core.ts b/scripts/counterfactual-replay-core.ts index e44d4a77d..b4b69c237 100644 Binary files a/scripts/counterfactual-replay-core.ts and b/scripts/counterfactual-replay-core.ts differ diff --git a/scripts/counterfactual-replay.ts b/scripts/counterfactual-replay.ts index 84d8d2265..1af75bd5d 100644 --- a/scripts/counterfactual-replay.ts +++ b/scripts/counterfactual-replay.ts @@ -5,8 +5,9 @@ // prints the Pareto-floored comparison via the shared renderer. All mapping/aggregation logic lives in // counterfactual-replay-core.ts (unit-tested); this wrapper owns provider IO, the artifacts cache, and // file reads. OFFLINE-ONLY guarantees (#8219 contract point 2): never posts anywhere, never touches live -// reviews; local ollama is the smoke path and the ONLY provider wired here — BYOK providers are a later, -// explicitly-flagged addition, refused loudly today rather than silently attempted. +// reviews. Local ollama is the smoke path; `--provider workers-ai` (#8279) is the explicitly-flagged +// hosted option for CI — credentials from CLOUDFLARE_ACCOUNT_ID/CLOUDFLARE_API_TOKEN, refused loudly when +// absent. Every other provider remains unwired by design. // // tsx scripts/counterfactual-replay.ts --fixtures corpus.json --variant // [--budget N] [--max-fixtures N] [--seed-suffix s] [--baseline scores.json] [--out scores.json] @@ -30,6 +31,7 @@ import { spawnSync } from "node:child_process"; import { artifactKey, buildCounterfactualAuditInsertSql, + extractWorkersAiResponse, compareReplays, estimateFixtureNeurons, isRunAccountingValid, @@ -48,6 +50,7 @@ type Args = { out: string | undefined; artifacts: string; ollamaUrl: string; + provider: "ollama" | "workers-ai"; promptFile: string | undefined; // #8222 CI mode: emit the marker comment and/or persist the run event (wrangler d1, like #8139). commentOut: string | undefined; @@ -72,6 +75,7 @@ function parseArgs(argv: string[]): Args { out: undefined, artifacts: ".counterfactual-artifacts", ollamaUrl: "http://localhost:11434", + provider: "ollama", promptFile: undefined, commentOut: undefined, baseVariantLabel: undefined, @@ -94,6 +98,7 @@ function parseArgs(argv: string[]): Args { else if (flag === "--out") args.out = argv[++i]; else if (flag === "--artifacts") args.artifacts = argv[++i] ?? args.artifacts; else if (flag === "--ollama-url") args.ollamaUrl = argv[++i] ?? args.ollamaUrl; + else if (flag === "--provider") args.provider = (argv[++i] as "ollama" | "workers-ai") ?? "ollama"; else if (flag === "--prompt-file") args.promptFile = argv[++i]; else if (flag === "--comment-out") args.commentOut = argv[++i]; else if (flag === "--base-variant-label") args.baseVariantLabel = argv[++i]; @@ -117,6 +122,18 @@ const DEFAULT_JUDGE_PROMPT = [ 'Respond with ONLY a JSON object: {"blockers": [..strings, empty if none..], "confidence": 0..1}.', ].join("\n"); +// Workers AI REST (#8279): the same account already running production reviews. Non-streaming text run; +// a non-OK status throws (operational failure — the campaign stops loudly, never silently mis-scores). +async function judgeWithWorkersAi(accountId: string, token: string, model: string, prompt: string, diff: string): Promise { + const response = await fetch(`https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/run/${model}`, { + method: "POST", + headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, + body: JSON.stringify({ messages: [{ role: "user", content: `${prompt}\n\n--- DIFF ---\n${diff}` }], temperature: 0 }), + }); + if (!response.ok) throw new Error(`workers-ai ${response.status}: ${(await response.text()).slice(0, 300)}`); + return extractWorkersAiResponse(await response.json()); +} + async function judgeWithOllama(ollamaUrl: string, model: string, prompt: string, diff: string): Promise { const response = await fetch(`${ollamaUrl}/api/generate`, { method: "POST", @@ -157,6 +174,18 @@ async function main(): Promise { return 0; } + // #8279: hosted-provider credentials resolve ONCE, loudly, before any spend. + let workersAi: { accountId: string; token: string } | null = null; + if (args.provider === "workers-ai") { + const accountId = process.env.CLOUDFLARE_ACCOUNT_ID?.trim(); + const token = process.env.CLOUDFLARE_API_TOKEN?.trim(); + if (!accountId || !token) { + console.error("--provider workers-ai requires CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN (a token with Workers AI read/run)."); + return 1; + } + workersAi = { accountId, token }; + } + mkdirSync(args.artifacts, { recursive: true }); const prompt = args.promptFile ? readFileSync(args.promptFile, "utf8") : DEFAULT_JUDGE_PROMPT; writeFileSync(join(args.artifacts, `prompt-${variant.promptVersion}.txt`), prompt); @@ -172,7 +201,9 @@ async function main(): Promise { if (existsSync(cachePath)) { raw = readFileSync(cachePath, "utf8"); // cached raw output: re-scoring is free } else { - raw = await judgeWithOllama(args.ollamaUrl, variant.modelSpec, prompt, fixture.boundedInputs.diff); + raw = workersAi + ? await judgeWithWorkersAi(workersAi.accountId, workersAi.token, variant.modelSpec, prompt, fixture.boundedInputs.diff) + : await judgeWithOllama(args.ollamaUrl, variant.modelSpec, prompt, fixture.boundedInputs.diff); writeFileSync(cachePath, raw); neuronsSpent += cost; // cache hits spend nothing — only real provider calls count } diff --git a/test/unit/counterfactual-replay-core.test.ts b/test/unit/counterfactual-replay-core.test.ts index 562c92f7b..87964515e 100644 --- a/test/unit/counterfactual-replay-core.test.ts +++ b/test/unit/counterfactual-replay-core.test.ts @@ -3,6 +3,7 @@ import type { BacktestCase, CounterfactualVerdict } from "@loopover/engine"; import { REVIEW_PROMPT_VERSION, buildCanonicalJudgePrompt } from "../../src/services/ai-review"; import { artifactKey, + extractWorkersAiResponse, buildCounterfactualAuditInsertSql, compareReplays, COUNTERFACTUAL_BACKTEST_EVENT_TYPE, @@ -184,3 +185,13 @@ describe("#8222: prompt version + CI comment/persist helpers", () => { }); }); }); + +describe("extractWorkersAiResponse (#8279)", () => { + it("extracts result.response; every malformed envelope yields \"\" so the parser abstains rather than throwing", () => { + expect(extractWorkersAiResponse({ result: { response: '{"blockers": []}' } })).toBe('{"blockers": []}'); + for (const bad of [null, undefined, {}, { result: {} }, { result: { response: 42 } }, "text"]) { + expect(extractWorkersAiResponse(bad)).toBe(""); + } + expect(parseVariantVerdict(extractWorkersAiResponse({ result: {} }))).toBe("abstained"); // the composed fail-safe + }); +});