diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 429570b..18cac50 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -7,7 +7,7 @@ "name": "fairtask", "source": "./", "description": "Screen a SWE-bench-style task (issue + fixing PR) for an under-specified issue or tests that only accept the original fix; verdict with verified evidence.", - "version": "0.1.0", + "version": "0.1.1", "author": { "name": "Manos Kaparos" }, "homepage": "https://github.com/mnkprs/fairtask", "repository": "https://github.com/mnkprs/fairtask", diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 1c03a0f..e407177 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "fairtask", - "version": "0.1.0", + "version": "0.1.1", "description": "Screen SWE-bench-style coding tasks for fairness before they become benchmark or RL tasks: is the issue specified well enough, and do the graded tests accept every reasonable fix? A command set: /fairtask (screen a task), /fairtask-baseline (one-prompt control), /fairtask-report, /fairtask-score, /fairtask-cases, /fairtask-trajectory (reproduce and inspect the evaluation) and /fairtask-eval (evidence audit, code check, provenance).", "author": { "name": "Manos Kaparos", diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 5b3f077..0f17ee4 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "fairtask", - "version": "0.1.0", + "version": "0.1.1", "description": "Screen SWE-bench-style coding tasks for fairness: under-specified issues and tests that only accept the original fix. Verdicts with verified evidence.", "author": { "name": "Manos Kaparos", diff --git a/README.md b/README.md index f3736ea..c731431 100644 --- a/README.md +++ b/README.md @@ -244,7 +244,7 @@ imperfect. This is the interface the project is built for. The people who screen tasks in 2026 do it inside agents — a Claude Code or Codex session that already has the repository open, the PR in front of it, and the rest of the -task-authoring workflow around it. fairtask ships as **two agent skills** on top of the same engine, so the screening +task-authoring workflow around it. fairtask ships as **seven agent skills — one command per operation** — on top of the same engine, so the screening and the evaluation happen in the conversation, and the numbers a session reports come from the scripts, not from the model's memory. @@ -252,9 +252,9 @@ the model's memory. | Where you work | Install | What you get | |---|---|---| -| Any harness that reads the open skills registry (Claude Code, Codex, Cursor, OpenCode, …) | `npx skills add mnkprs/fairtask` | both skills, installed into that harness's skills directory | -| Claude Code, as a plugin | `/plugin marketplace add mnkprs/fairtask` then `/plugin install fairtask@fairtask` | both skills (plugin manifest in `.claude-plugin/`) | -| Codex, as a plugin | the repository carries `.codex-plugin/plugin.json` | both skills | +| Any harness that reads the open skills registry (Claude Code, Codex, Cursor, OpenCode, …) | `npx skills add mnkprs/fairtask` | all seven commands, installed into that harness's skills directory | +| Claude Code, as a plugin | `/plugin marketplace add mnkprs/fairtask` then `/plugin install fairtask@fairtask` | all seven commands, **namespaced**: `/fairtask:fairtask-report`, `/fairtask:fairtask-score`, … (plugin manifest in `.claude-plugin/`) | +| Codex, as a plugin | the repository carries `.codex-plugin/plugin.json` | all seven commands | One install brings the whole command set: @@ -262,14 +262,16 @@ One install brings the whole command set: |---|---|---| | `/fairtask ` | Screen a task with the full pipeline; verdict with verified evidence. | yes (~one dollar) | | `/fairtask-baseline ` | The same task through the one-prompt baseline, for comparison; its evidence is *not* machine-verified. | yes (~fifteen cents) | -| `/fairtask-report` | The headline and all-systems tables from the committed results. | no | +| `/fairtask-report` | The headline and all-systems tables from the committed results (first run shallow-clones the 30 evaluation repos, ~1.2 GB, to re-verify evidence). | no | | `/fairtask-score [run ids…]` | Metrics of runs against the human labels. | no | | `/fairtask-cases [id]` | The 30-case evaluation set, or one instance laid out with its decisive lines quoted. | no | | `/fairtask-trajectory [run]` | What the agent did on an instance, tool call by tool call. | no | | `/fairtask-eval …` | The supporting checks: evidence audit, zero-LLM code check, data provenance. | no | -All need Node ≥ 22.18 and git. `/fairtask` also needs a Claude login or `ANTHROPIC_API_KEY`, because it runs -the model; `/fairtask-eval` needs neither. +The bare command names below are what a `npx skills add` install exposes; installed as the Claude Code plugin +they are namespaced (`/fairtask:fairtask-report` and so on). All need Node ≥ 22.18 and git. The two screening +commands — `/fairtask` and `/fairtask-baseline` — run the model and need a Claude login or `ANTHROPIC_API_KEY`; +the other five make no model calls. ### `/fairtask` — screen a task @@ -411,7 +413,7 @@ a complete command; rows with the same script differ by one flag. |---|---| | `npm run typecheck` | TypeScript, no emit. | | `npm test` | 23 adversarial tests: verifier (fabricated, elided, cross-hunk, removed-line and symlinked quotes), workspace trust (symlinked path, dirty tree, wrong remote, bad commit), run lock and screening ids. | -| `npm run validate:manifests` | Semantic check of the plugin manifests and both skills' frontmatter. | +| `npm run validate:manifests` | Semantic check of the plugin manifests, version consistency, engine pins, and every skill's frontmatter. | ## 2d. Agent trajectories diff --git a/package-lock.json b/package-lock.json index 934a670..a4601b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "fairtask", - "version": "0.1.0", + "version": "0.1.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "fairtask", - "version": "0.1.0", + "version": "0.1.1", "license": "MIT", "dependencies": { "@anthropic-ai/claude-agent-sdk": "^0.3.250", @@ -19,7 +19,7 @@ "typescript": "^7.0.2" }, "engines": { - "node": ">=22.6" + "node": ">=22.18" } }, "node_modules/@anthropic-ai/claude-agent-sdk": { diff --git a/package.json b/package.json index 6e05489..5cdacb4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "fairtask", - "version": "0.1.0", + "version": "0.1.1", "private": true, "type": "module", "description": "Agentic screening of SWE-bench-style tasks for under-specification and unfair tests, scored against OpenAI's human annotations.", diff --git a/scripts/validate-manifests.mjs b/scripts/validate-manifests.mjs index 84215d3..14e24aa 100644 --- a/scripts/validate-manifests.mjs +++ b/scripts/validate-manifests.mjs @@ -30,4 +30,10 @@ const skill = readFileSync("skills/fairtask/SKILL.md", "utf8"); if (!/--branch v\d+\.\d+\.\d+/.test(skill)) fail("SKILL.md: engine install must pin a release tag"); const pkg = read("package.json"); if (pkg.version !== claude.version) fail(`package.json version ${pkg.version} differs from plugin version ${claude.version}`); +const rootPkg = JSON.parse(readFileSync("package.json", "utf8")); +for (const [f, v] of [[".claude-plugin/plugin.json", claude.version], [".codex-plugin/plugin.json", codex.version], [".claude-plugin/marketplace.json", market.plugins[0]?.version]]) + if (v !== rootPkg.version) fail(`${f}: version ${v} != package.json ${rootPkg.version}`); +const pin = new RegExp(`--branch v${rootPkg.version.replace(/\./g, "\\.")}\\b`); +for (const f of ["skills/fairtask/SKILL.md", "skills/fairtask-baseline/SKILL.md"]) + if (!pin.test(readFileSync(f, "utf8"))) fail(`${f}: engine pin does not match version v${rootPkg.version}`); if (!process.exitCode) console.log(`manifests ok: fairtask ${claude.version}, skill frontmatter ok, capabilities disclosed`); diff --git a/skills/fairtask-baseline/SKILL.md b/skills/fairtask-baseline/SKILL.md index 46e3f96..4fe7221 100644 --- a/skills/fairtask-baseline/SKILL.md +++ b/skills/fairtask-baseline/SKILL.md @@ -1,6 +1,6 @@ --- name: fairtask-baseline -description: Screen one SWE-bench-style task with fairtask's one-prompt baseline — the same rubric and material as the full pipeline but no repository access and no evidence verification — so its verdict can be compared with /fairtask on the same task. Use when asked to run the baseline on a task, to compare the baseline against the pipeline, or to show what screening looks like without an agent. Costs about fifteen cents and forty seconds per task; needs a Claude login or API key. +description: Screen one SWE-bench-style task with fairtask's one-prompt baseline — the same rubric and material as the full pipeline but no repository access and no evidence verification — so its verdict can be compared with /fairtask on the same task. Use ONLY when a specific task is supplied (an instance id or task file) and the ask is to screen it with the baseline or compare the two screeners on it; for comparing the published evaluation numbers use fairtask-report or fairtask-score instead. Costs about fifteen cents and forty seconds per task; needs a Claude login or API key. license: MIT metadata: origin: fairtask @@ -10,7 +10,8 @@ metadata: # fairtask-baseline Screen one candidate task with the **one-prompt baseline**: the same rubric, issue, gold patch and test patch the -full pipeline sees, but a single model call with no tools, no repository access, and no deterministic verification. +full pipeline sees, but a single model call with no tools and no deterministic verification. The engine does not even clone the +task's repository for it — nothing is read beyond the task text. Its purpose is comparison — run it next to `/fairtask` on the same task to show what the agent pipeline adds. ## Steps @@ -19,11 +20,11 @@ Its purpose is comparison — run it next to `/fairtask` on the same task to sho (`--task `), exactly as for `/fairtask`. 2. **Locate the engine** the same way `/fairtask` does: `$FAIRTASK_HOME` if set, else `~/.fairtask`, else the current directory if its `package.json` is named `fairtask`; if absent, clone - `https://github.com/mnkprs/fairtask` at `--branch v0.1.0` into `~/.fairtask` and run `npm ci`. + `https://github.com/mnkprs/fairtask` with `--branch v0.1.1 --depth 1` into `~/.fairtask` and run `npm ci`. 3. **Run**: `npm run screen -- --variant baseline`. Expect about forty seconds and fifteen cents. 4. **Report** the verdict in the same order as `/fairtask` (decision, both axes with reasons, every evidence item), - and state plainly that the trailer says **evidence NOT verified**: the baseline's quotes are whatever the model - wrote — on the evaluation set eight to eleven percent of them do not exist where cited. That is the point of + and state plainly what the trailer says — **evidence not machine-verified — this variant has no verifier**: the + baseline's quotes are whatever the model wrote — on the evaluation set eight to eleven percent of them do not exist where cited. That is the point of the comparison, not a malfunction. 5. If the same task was screened with `/fairtask`, put the two verdicts side by side: decision, per-axis scores, whether the evidence verified, cost. diff --git a/skills/fairtask-cases/SKILL.md b/skills/fairtask-cases/SKILL.md index ee41ea0..76b86f6 100644 --- a/skills/fairtask-cases/SKILL.md +++ b/skills/fairtask-cases/SKILL.md @@ -13,8 +13,11 @@ The evaluation set, or one case laid out so a reader can see the discrepancy wit ## Steps -1. **Locate the repository**: current directory if its `package.json` is named `fairtask`; else `$FAIRTASK_HOME`; - else `~/.fairtask`. If none exists, say so and stop. +1. **Locate the engine**, in order: the current directory if its `package.json` is named `fairtask`; + `$CLAUDE_PLUGIN_ROOT` if set and it contains that `package.json` (the Claude Code plugin install *is* the + repository); `$FAIRTASK_HOME`; `~/.fairtask`. If none exists, clone the pinned release — + `git clone --branch v0.1.1 --depth 1 https://github.com/mnkprs/fairtask ~/.fairtask && (cd ~/.fairtask && npm ci)` + — and say you did (needs network and Node ≥ 22.18). 2. **No instance id given** — show the set: `npm run data:eval-set` (needs `data/raw/swebench_test.parquet`; if missing, first run `curl --fail -L -o data/raw/swebench_test.parquet https://huggingface.co/datasets/princeton-nlp/SWE-bench/resolve/e48e2bd1e9fecd5bbd641e9414ac59da9f2e69f6/data/test-00000-of-00001.parquet` and say you did). Show the 30-row table verbatim and name the four strata with their counts. In a clean checkout diff --git a/skills/fairtask-eval/SKILL.md b/skills/fairtask-eval/SKILL.md index e5d0709..b0fa228 100644 --- a/skills/fairtask-eval/SKILL.md +++ b/skills/fairtask-eval/SKILL.md @@ -1,6 +1,6 @@ --- name: fairtask-eval -description: Inspect the fairtask evaluation's supporting checks from inside an agent session — audit cited evidence against the repositories, run the zero-LLM code pre-check, or verify the annotation data's provenance and checksum — and route anything else to the dedicated commands (/fairtask-report, /fairtask-score, /fairtask-cases, /fairtask-trajectory). Use when asked to audit evidence, run the code check, verify data provenance, or when unsure which fairtask command fits. Read-only and offline; makes no model calls. +description: Inspect the fairtask evaluation's supporting checks from inside an agent session — audit cited evidence against the repositories, run the zero-LLM code pre-check, or verify the annotation data's provenance and checksum — and route anything else to the dedicated commands (/fairtask-report, /fairtask-score, /fairtask-cases, /fairtask-trajectory). Use when asked to audit cited evidence, run the code pre-check, or verify data provenance of the fairtask project. Makes no model calls; the audit and code check need the evaluation workspaces (about 1.2 GB of shallow clones) and say so before creating them. license: MIT metadata: origin: fairtask @@ -16,8 +16,11 @@ calls a model. ## Steps -1. **Locate the repository.** The current directory if its `package.json` is named `fairtask`; else `FAIRTASK_HOME`; else - `~/.fairtask`. If none exists, say so and stop — this skill does not clone. +1. **Locate the engine**, in order: the current directory if its `package.json` is named `fairtask`; + `$CLAUDE_PLUGIN_ROOT` if set and it contains that `package.json` (the Claude Code plugin install *is* the + repository); `$FAIRTASK_HOME`; `~/.fairtask`. If none exists, clone the pinned release — + `git clone --branch v0.1.1 --depth 1 https://github.com/mnkprs/fairtask ~/.fairtask && (cd ~/.fairtask && npm ci)` + — and say you did (needs network and Node ≥ 22.18). 2. **Pick the operation** from the request (one per invocation; ask if two are equally plausible): | Request mentions | Do | diff --git a/skills/fairtask-report/SKILL.md b/skills/fairtask-report/SKILL.md index 163eb36..d9ecd5c 100644 --- a/skills/fairtask-report/SKILL.md +++ b/skills/fairtask-report/SKILL.md @@ -1,6 +1,6 @@ --- name: fairtask-report -description: Print fairtask's headline comparison (one-prompt baseline vs the final agent pipeline) and the all-systems table over the same 30 human-annotated development-set cases, generated by the repository's own report script from the committed results. Use when asked for the results, the headline numbers, the comparison table, or baseline versus final. Offline, no model calls; clones nothing. +description: Print fairtask's headline comparison (one-prompt baseline vs the final agent pipeline) and the all-systems table over the same 30 human-annotated development-set cases, generated by the repository's own report script from the committed results. Use when asked for the results, the headline numbers, the comparison table, or baseline versus final. No model calls; the first run may shallow-clone the 30 evaluation repositories (about 1.2 GB) to re-verify cited evidence, and says so before doing it. license: MIT metadata: origin: fairtask @@ -14,11 +14,14 @@ report script from `results/`, never from memory. ## Steps -1. **Locate the repository**: current directory if its `package.json` is named `fairtask`; else `$FAIRTASK_HOME`; - else `~/.fairtask`. If none exists, say so and stop. -2. **Preflight workspaces**: the bad-evidence rows re-verify every quote against the repositories, so if - `workspaces/` is missing run `npm run data:workspaces` first (about twenty seconds, shallow-clones 30 repos) - and say you did. +1. **Locate the engine**, in order: the current directory if its `package.json` is named `fairtask`; + `$CLAUDE_PLUGIN_ROOT` if set and it contains that `package.json` (the Claude Code plugin install *is* the + repository); `$FAIRTASK_HOME`; `~/.fairtask`. If none exists, clone the pinned release — + `git clone --branch v0.1.1 --depth 1 https://github.com/mnkprs/fairtask ~/.fairtask && (cd ~/.fairtask && npm ci)` + — and say you did (needs network and Node ≥ 22.18). +2. **Preflight workspaces**: the bad-evidence rows re-verify every quote against the repositories. If + `workspaces/` is missing, say that the next step shallow-clones the 30 evaluation repositories (about 1.2 GB + on disk, ~20 s on a fast link), then run `npm run data:workspaces`. 3. **Run**: `node src/report.ts --baseline baseline --final v3-verify --runs baseline,baseline-rerun,v1-context,v2-specialists,v3-verify,v4-calibrated,v5-cheap-probes,v5-rerun,v6-target-aware,v7-sonnet-nocal` 4. **Show both tables verbatim** in a code block, then two sentences: decision accuracy is the primary metric and ties at 67% (development set); what changes is flag recall (70% → 75–85%) and cited evidence that fails diff --git a/skills/fairtask-score/SKILL.md b/skills/fairtask-score/SKILL.md index 5a1dfd9..404e6a9 100644 --- a/skills/fairtask-score/SKILL.md +++ b/skills/fairtask-score/SKILL.md @@ -13,8 +13,11 @@ Metrics of fairtask runs against the human labels, straight from the scorer. ## Steps -1. **Locate the repository**: current directory if its `package.json` is named `fairtask`; else `$FAIRTASK_HOME`; - else `~/.fairtask`. If none exists, say so and stop. +1. **Locate the engine**, in order: the current directory if its `package.json` is named `fairtask`; + `$CLAUDE_PLUGIN_ROOT` if set and it contains that `package.json` (the Claude Code plugin install *is* the + repository); `$FAIRTASK_HOME`; `~/.fairtask`. If none exists, clone the pinned release — + `git clone --branch v0.1.1 --depth 1 https://github.com/mnkprs/fairtask ~/.fairtask && (cd ~/.fairtask && npm ci)` + — and say you did (needs network and Node ≥ 22.18). 2. **Run**: `npm run score -- ` — default `baseline v3-verify`; any directory names under `results/` (`baseline`, `baseline-rerun`, `v1-context`, `v2-specialists`, `v3-verify`, `v4-calibrated`, `v5-cheap-probes`, `v5-rerun`, `v6-target-aware`, `v7-sonnet-nocal`). Add `--detail` when asked for per-instance rows. diff --git a/skills/fairtask-trajectory/SKILL.md b/skills/fairtask-trajectory/SKILL.md index 6639be7..c5c59dd 100644 --- a/skills/fairtask-trajectory/SKILL.md +++ b/skills/fairtask-trajectory/SKILL.md @@ -13,8 +13,11 @@ One agent run, readable top to bottom. ## Steps -1. **Locate the repository**: current directory if its `package.json` is named `fairtask`; else `$FAIRTASK_HOME`; - else `~/.fairtask`. If none exists, say so and stop. +1. **Locate the engine**, in order: the current directory if its `package.json` is named `fairtask`; + `$CLAUDE_PLUGIN_ROOT` if set and it contains that `package.json` (the Claude Code plugin install *is* the + repository); `$FAIRTASK_HOME`; `~/.fairtask`. If none exists, clone the pinned release — + `git clone --branch v0.1.1 --depth 1 https://github.com/mnkprs/fairtask ~/.fairtask && (cd ~/.fairtask && npm ci)` + — and say you did (needs network and Node ≥ 22.18). 2. **Find the trajectory**: evaluation runs live at `trajectories//.jsonl` (default run `v3-verify`; the run ids are the directory names); single-task screenings at `screenings//trajectory.jsonl` or `examples//trajectory.jsonl`. diff --git a/skills/fairtask/SKILL.md b/skills/fairtask/SKILL.md index 83f58b0..0f3a367 100644 --- a/skills/fairtask/SKILL.md +++ b/skills/fairtask/SKILL.md @@ -32,7 +32,7 @@ This skill drives it, and falls back to doing the probes' work yourself when the screening, and surface any `_warning` the JSON carries. 2. **Locate the engine.** Use `FAIRTASK_HOME` if set, else `~/.fairtask`. If absent, install the release this skill was published with — never a moving branch: - `git clone --branch v0.1.0 --depth 1 https://github.com/mnkprs/fairtask ~/.fairtask && (cd ~/.fairtask && npm ci)`. + `git clone --branch v0.1.1 --depth 1 https://github.com/mnkprs/fairtask ~/.fairtask && (cd ~/.fairtask && npm ci)`. If it exists, confirm it is at that release (`git -C ~/.fairtask describe --tags`) before using it; otherwise re-clone. This step writes under the home directory, installs npm packages and runs code; say so if the user has not seen the engine installed before. Requirements: Node ≥ 22.18, git, and either `ANTHROPIC_API_KEY` or a diff --git a/src/lib/run.ts b/src/lib/run.ts index b26b2dd..7cc6754 100644 --- a/src/lib/run.ts +++ b/src/lib/run.ts @@ -149,8 +149,12 @@ export async function runOne(variant: Variant, inst: TaskInstance, opts: Pick [k, { description: a.description, tools: a.tools, model: a.model, prompt: a.prompt }])), }); - const guard = { matcher: "Read|Grep|Glob|NotebookRead", hooks: [workspaceGuard(ctx.workspace)] }; - const hooks = { ...(options.hooks ?? {}), PreToolUse: [guard, ...(options.hooks?.PreToolUse ?? [])] }; + // A variant with no tools and no subagents (the baseline) never touches the repository: no guard, and the + // workspace need not exist. The guard is constructed lazily so a missing workspace only fails repo-using variants. + const usesRepo = (Array.isArray(options.tools) ? options.tools.length > 0 : options.tools !== undefined) || options.agents !== undefined; + const hooks = usesRepo + ? { ...(options.hooks ?? {}), PreToolUse: [{ matcher: "Read|Grep|Glob|NotebookRead", hooks: [workspaceGuard(ctx.workspace)] }, ...(options.hooks?.PreToolUse ?? [])] } + : options.hooks ?? {}; const base: Options = { model: opts.model, permissionMode: "dontAsk", settingSources: [], strictMcpConfig: true, persistSession: true, ...options, hooks }; const t0 = Date.now(); let cost = 0, turns = 0, inTok = 0, outTok = 0, cacheTok = 0, retries = 0, sessionId: string | undefined; diff --git a/src/report.test.ts b/src/report.test.ts index d9f1b73..6bca567 100644 --- a/src/report.test.ts +++ b/src/report.test.ts @@ -8,3 +8,8 @@ test("report refuses a --final-repeat of a different configuration", () => { (e: unknown) => String((e as { stderr?: string }).stderr).includes("is not a repeat of") && String((e as { stderr?: string }).stderr).includes("variant"), ); }); + +test("score with no run ids defaults to baseline vs v3-verify", () => { + const out = execFileSync(process.execPath, ["src/score.ts"], { encoding: "utf8", stdio: "pipe" }); + assert.ok(out.includes("baseline") && out.includes("v3-verify") && out.includes("PRIMARY decision accuracy")); +}); diff --git a/src/score.ts b/src/score.ts index f5544a3..a636a02 100644 --- a/src/score.ts +++ b/src/score.ts @@ -13,7 +13,7 @@ const detail = argv.includes("--detail"); const asJson = argv.includes("--json"); const common = argv.includes("--common"); // restrict every run to the instances that ALL listed runs scored const runIds = argv.filter((a) => !a.startsWith("--")); -if (runIds.length === 0) { console.error("usage: node src/score.ts [...] [--detail] [--json]"); process.exit(2); } +if (runIds.length === 0) runIds.push("baseline", "v3-verify"); // default comparison: the baseline vs the final configuration const instances = JSON.parse(readFileSync(`${ROOT}data/eval/instances.json`, "utf8")) as EvalInstance[]; const byId = new Map(instances.map((i) => [i.instance_id, i])); diff --git a/src/screen.ts b/src/screen.ts index fd6a218..ac5d8f3 100644 --- a/src/screen.ts +++ b/src/screen.ts @@ -50,8 +50,15 @@ if (!outDir.startsWith(outRoot + sep)) throw new Error("output directory escapes mkdirSync(outDir, { recursive: true }); console.log(`fairtask · ${task.instance_id} · ${variantName} · ${args.get("model") ?? "claude-opus-5"}`); -const ws = await prepareWorkspace(task); -console.log(`repository ${task.repo} @ ${task.base_commit.slice(0, 10)} (${ws.status}) → ${ws.dir}`); +// Variants with no tools and no subagents (the baseline) never read the repository — skip the clone entirely. +const probeBuild = variant.build(task, { model: args.get("model") ?? "claude-opus-5", workspace: "/nonexistent" }); +const needsRepo = (Array.isArray(probeBuild.options.tools) ? probeBuild.options.tools.length > 0 : probeBuild.options.tools !== undefined) || probeBuild.options.agents !== undefined; +if (needsRepo) { + const ws = await prepareWorkspace(task); + console.log(`repository ${task.repo} @ ${task.base_commit.slice(0, 10)} (${ws.status}) → ${ws.dir}`); +} else { + console.log(`repository not cloned — variant "${variantName}" has no tools and reads only the task text`); +} if (task.FAIL_TO_PASS.length === 0) console.log("note: no FAIL_TO_PASS list given — the probes will treat every test in the test patch as graded"); // Every screening gets its own attempt id, so concurrent screenings of the same instance never share a trajectory file. @@ -69,7 +76,7 @@ const v = p.verdict; console.log("\n" + "─".repeat(72)); if (!v) { console.log(`NO VERDICT: ${p.error}`); process.exit(1); } const flag = v.underspecified >= 2 || v.false_negative >= 2; -console.log(`${flag ? "FLAG" : "USABLE"} underspecified=${v.underspecified} false_negative=${v.false_negative} confidence=${v.confidence}/5 ${p.verified === false ? "(evidence NOT verified)" : p.verified ? "(evidence verified)" : ""}`); +console.log(`${flag ? "FLAG" : "USABLE"} underspecified=${v.underspecified} false_negative=${v.false_negative} confidence=${v.confidence}/5 ${p.verified === false ? "(evidence NOT verified)" : p.verified ? "(evidence verified)" : "(evidence not machine-verified — this variant has no verifier)"}`); console.log(`\nIssue specification (${v.underspecified}): ${v.underspecified_rationale}`); console.log(`\nTest scope (${v.false_negative}): ${v.false_negative_rationale}`); console.log(`\nEvidence (${v.evidence.length}):`);