From 88e9094a9f69353f1c4dc63846552341ee839be9 Mon Sep 17 00:00:00 2001 From: psmyrdek Date: Mon, 17 Aug 2026 10:31:11 +0200 Subject: [PATCH] =?UTF-8?q?refactor(bench-kit):=20CLI=20wo=C5=82a=20bootst?= =?UTF-8?q?rap=20kitu=20=E2=80=94=20kontrakt=20v1,=20logika=20instancji=20?= =?UTF-8?q?w=20template'cie?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kit zna siebie, CLI zna maszynę użytkownika: init/update klonują template i wołają .bench-kit/bootstrap/index.mjs z klonu (żądanie JSON na stdin, odpowiedź w ostatniej linii stdout). Update wykonuje bootstrap z NOWEJ wersji kitu, więc migracje układu jadą z tagiem. Wchłania TEMPLATE_ONLY_PATHS (gałąź fix/bench-kit-template-only-paths zbędna). Minimalna wersja template'u: v0.10.0 (brak bootstrapu → template_incomplete). Testy zawężone do kontraktu: fake runBootstrap, asercje na żądanie, render, koperty --json i kody wyjścia. Co-Authored-By: Claude Fable 5 --- src/commands/bench-kit.ts | 854 +++++++++++--------------------- tests/bench-kit-command.test.ts | 756 ++++++++++------------------ 2 files changed, 561 insertions(+), 1049 deletions(-) diff --git a/src/commands/bench-kit.ts b/src/commands/bench-kit.ts index febd7ea..e741759 100644 --- a/src/commands/bench-kit.ts +++ b/src/commands/bench-kit.ts @@ -1,34 +1,25 @@ /** - * 10x bench-kit — installer/updater for benchmark instances. + * 10x bench-kit — thin orchestrator over the template's own bootstrap. * - * `bench-kit init` is deliberately a *thin, deterministic* installer: it - * knows nothing about the template's internal structure beyond the - * `.bench-kit/` marker directory. Everything judgment-based (rubrics, - * tasks, stack-specific images) happens later, via agent skills inside - * the instance — never here. + * Division of knowledge: the kit knows itself, the CLI knows the user's + * machine. The CLI resolves the tag, clones the template, picks the agent + * tool profile, detects the surrounding product repo (network probes + * included) — then hands everything to `.bench-kit/bootstrap/index.mjs` + * INSIDE the clone, which owns the file layout and content semantics of + * an instance (materialization, zones, manifest, base-repo registration, + * git init). Because update runs the bootstrap of the NEW template + * version, a template that changes its layout ships its own migration. * - * `bench-kit update` upgrades the template zone-by-zone: `.bench-kit/` is - * replaced wholesale; workflows, skills and shared root files (AGENTS.md) - * are synced into the working tree as an uncommitted *proposal* (the - * company reviews `git diff` and decides); company content (`tasks/`, - * `evaluation-pool/`, `bench.config.yaml`) is never touched. + * Trust boundary: the bootstrap is executed only from a clone of + * TEMPLATE_REPO_URL and only from a ref the user asked for. (npm ci + * inside the clone already runs lifecycle scripts, so this executes no + * new class of code.) */ import { spawn } from "node:child_process"; -import { - cpSync, - existsSync, - mkdirSync, - mkdtempSync, - readFileSync, - readdirSync, - renameSync, - rmSync, - rmdirSync, - writeFileSync, -} from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; -import { basename, join, resolve, sep } from "node:path"; +import { basename, join, resolve } from "node:path"; import type { CAC } from "cac"; import { ExitCodes, @@ -44,23 +35,11 @@ import { DEFAULT_TOOL, PROFILES } from "../lib/tool-profile"; export const TEMPLATE_REPO_URL = "https://github.com/przeprogramowani/10x-bench-kit"; -/** - * Root-level template files that belong to the shared zone (like skills): - * update syncs them into the working tree as a reviewable proposal. - * `init` needs no special-casing — materialize copies the template root. - */ -export const SHARED_ROOT_FILES = ["AGENTS.md"]; +/** The kit's side of the contract — the only kit-internal path the CLI knows. */ +export const BOOTSTRAP_ENTRY = join(".bench-kit", "bootstrap", "index.mjs"); -/** The template's placeholder base-repo entry that init may replace. */ -export const PLACEHOLDER_BASE_REPO = "demo-app"; - -/** - * Instance-local base-repo clones live under `.repos/` (gitignored). - * The authoring skills (bench-task, bench-refresh, bench-wiring) read this - * convention from AGENTS.md — init pre-clones the detected repo there so - * the first `/bench-task` never starts with a cold network clone. - */ -export const BASE_REPOS_DIR = ".repos"; +/** Bumped in lockstep with the kit; a mismatch is a clean error, not weird behavior. */ +export const CONTRACT_VERSION = 1; /** A git repo detected around the directory init was invoked from. */ export interface DetectedBaseRepo { @@ -72,18 +51,56 @@ export interface DetectedBaseRepo { url: string; /** Current HEAD — a candidate pin for the first task. */ headCommit: string; + /** True when the https equivalent of `url` answered `git ls-remote`. */ + httpsReachable: boolean; } -/** Instance manifest written next to the template's VERSION file. */ -export interface InstanceManifest { - templateVersion: string; +/** Request the CLI sends to the kit's bootstrap (stdin JSON). */ +export interface BootstrapRequest { + contractVersion: number; + mode: "init" | "update"; + templateDir: string; + targetDir: string; + tool: { id: string; skillRoot: string; explicit?: boolean }; + /** id → skillRoot for every supported tool (machine knowledge the kit lacks). */ + toolProfiles: Record; + cwd: string; templateRef: string; templateSource: string; - initializedAt: string; - /** Agent tool profile id (claude-code, cursor, …) — decides where skills live. */ + detectedBaseRepo: DetectedBaseRepo | null; + now: string; +} + +/** Per-file outcome counts of a directory sync, as reported by the bootstrap. */ +export interface SyncCounts { + added: number; + updated: number; + unchanged: number; +} + +/** Response parsed from the bootstrap's last stdout line. */ +export interface BootstrapResponse { + ok: boolean; + code?: string; + message?: string; + hint?: string; + mode?: string; + upToDate?: boolean; + fromVersion?: string; + templateVersion?: string; tool?: string; - updatedAt?: string; - detectedBaseRepo?: DetectedBaseRepo; + skillRoot?: string; + manifest?: Record; + filesCopied?: number; + baseRepo?: { name: string; url: string } | null; + demoTasksPinned?: number; + baseRepoClone?: { name: string; url: string; rootDir: string; dest: string } | null; + runnerDeps?: "installed" | "failed" | "skipped"; + gitInitialized?: boolean; + committed?: boolean; + zones?: { workflows?: SyncCounts; skills?: SyncCounts; shared?: SyncCounts }; + warnings?: string[]; + nextSteps?: string[]; } interface BenchKitFlags extends GlobalFlags { @@ -92,37 +109,35 @@ interface BenchKitFlags extends GlobalFlags { yes?: boolean; } -/** Per-file outcome counts of a directory sync. */ -export interface SyncCounts { - added: number; - updated: number; - unchanged: number; -} - /** * Side-effectful collaborators, injectable for tests (DI over module * mocking, per repo convention). The default implementation shells out - * to git; tests substitute a fake that materializes a fixture tree. + * to git/node; tests substitute fakes — runBootstrap included, so CLI + * tests assert on the contract, not on disk effects (those are the + * kit's tests). */ export interface BenchKitDeps { /** Resolves true when `cmd` can be spawned (used for preflight). */ toolAvailable(cmd: string): Promise; /** Clones the template at `ref` (null = default branch) into `destDir`. */ cloneTemplate(ref: string | null, destDir: string): Promise<{ ok: boolean; error: string }>; - /** Runs git with `args` inside `cwd` (init/commit/status). */ + /** Runs git with `args` inside `cwd` (clean-worktree gate). */ runGit(args: string[], cwd: string): Promise<{ ok: boolean; stdout: string; error: string }>; /** Detects the git repo containing `cwd` (null when absent or origin-less). */ - detectBaseRepo(cwd: string): Promise; + detectBaseRepo(cwd: string): Promise | null>; /** Clones the detected repo into `destDir` (local source, remote origin). */ - cloneBaseRepo(repo: DetectedBaseRepo, destDir: string): Promise<{ ok: boolean; error: string }>; + cloneBaseRepo( + repo: { rootDir: string; url: string }, + destDir: string, + ): Promise<{ ok: boolean; error: string }>; /** True when `git ls-remote` succeeds against `url` (https preference probe). */ remoteReachable(url: string): Promise; - /** Installs runner dependencies (`npm ci`) inside `runnerDir`. */ - installRunnerDeps(runnerDir: string): Promise<{ ok: boolean; error: string }>; /** Scans `cwd` for agent-tool markers (ranked, strongest first). */ detectToolSignals(cwd: string): DetectionSignal[]; /** Interactive tool picker; resolves the chosen id, or null on cancel. */ chooseTool(initial: string, detectedReason: string | null): Promise; + /** Executes the kit's bootstrap entry with `request` on stdin. */ + runBootstrap(entry: string, request: BootstrapRequest): Promise; now(): Date; } @@ -170,9 +185,7 @@ export async function runBenchKitInit( await preflight(ctx, deps); const existingVersion = readInstanceVersion(targetDir); - const repair = existingVersion !== null; - - if (repair && requestedRef !== null) { + if (existingVersion !== null && requestedRef !== null) { outputError( ctx, "version_conflict", @@ -181,93 +194,65 @@ export async function runBenchKitInit( "Run '10x bench-kit update' to change the template version of an existing instance.", ); } - if (!repair && existsSync(targetDir) && readdirSync(targetDir).length > 0) { - outputError( - ctx, - "target_not_empty", - `Directory '${targetDir}' is not empty and is not a benchmark instance.`, - ExitCodes.ERROR, - "Run '10x bench-kit init ' with an empty or new directory.", - ); - } - const existingManifest = repair ? readManifest(targetDir) : null; - const toolId = await resolveInstanceTool(ctx, options, deps, existingManifest); + const toolChoice = await resolveInstanceTool(ctx, options, deps, existingVersion !== null); - // Materialize the template into a scratch clone first, so a failed - // download can never leave a half-written instance behind. + // Clone into a scratch dir first, so a failed download can never leave a + // half-written instance behind; the bootstrap runs FROM this clone. const scratch = mkdtempSync(join(tmpdir(), "bench-kit-")); try { - verbose(ctx, `cloning ${TEMPLATE_REPO_URL} (${requestedRef ?? "latest"}) into ${scratch}`); - const clone = await deps.cloneTemplate(requestedRef, scratch); - if (!clone.ok) { + await cloneTemplateOrDie(ctx, deps, requestedRef, scratch, "init"); + const entry = requireBootstrap(ctx, scratch); + + // Running init from inside a product repo is the common flow — detect + // it here (network side), let the bootstrap decide what to register. + const detected = existingVersion !== null ? null : await deps.detectBaseRepo(process.cwd()); + let detectedBaseRepo: DetectedBaseRepo | null = null; + if (detected !== null) { + const https = toHttpsUrl(detected.url); + const httpsReachable = https !== null && (await deps.remoteReachable(https)); + if (httpsReachable) verbose(ctx, `repo answers over https — bootstrap may prefer ${https}`); + detectedBaseRepo = { ...detected, httpsReachable }; + } + + const request: BootstrapRequest = { + contractVersion: CONTRACT_VERSION, + mode: "init", + templateDir: scratch, + targetDir, + tool: { + id: toolChoice.id, + skillRoot: skillRootFor(toolChoice.id), + ...(toolChoice.explicit ? { explicit: true } : {}), + }, + toolProfiles: allToolProfiles(), + cwd: process.cwd(), + templateRef: requestedRef ?? "latest", + templateSource: TEMPLATE_REPO_URL, + detectedBaseRepo, + now: deps.now().toISOString(), + }; + verbose(ctx, `running template bootstrap (${BOOTSTRAP_ENTRY})`); + const res = await deps.runBootstrap(entry, request); + if (!res.ok) { outputError( ctx, - "clone_failed", - `Could not download the template from ${TEMPLATE_REPO_URL}.`, + res.code ?? "bootstrap_failed", + res.message ?? "The template bootstrap failed.", ExitCodes.ERROR, - clone.error - ? `Git said: ${clone.error.trim()}` - : "Check your internet connection and run '10x bench-kit init' again.", + res.hint, ); } - const templateVersion = readTemplateVersion(ctx, scratch); - mkdirSync(targetDir, { recursive: true }); - const skillSource = templateSkillSource(scratch); - // Skills are placed per tool profile, so materialize skips them here. - const copied = materialize(scratch, targetDir, { - skipExisting: repair, - skip: (rel) => { - const posix = rel.split(sep).join("/"); - return posix === skillSource || posix.startsWith(`${skillSource}/`); - }, - }); - const skills = syncDir( - join(scratch, skillSource), - join(targetDir, skillRootFor(toolId)), - { overwrite: !repair }, - ); - installWorkflows(targetDir, { skipExisting: repair }); - - // Running init from inside a product repo is the common flow — register - // that repo as the first base repo instead of leaving the placeholder. - let baseRepo: DetectedBaseRepo | null = null; - let demoTasksPinned = 0; - if (!repair) { - const detected = await deps.detectBaseRepo(process.cwd()); - if (detected !== null && resolve(detected.rootDir) !== targetDir) { - // Prefer https over SSH when the repo answers publicly: https clones - // in CI/containers with zero secrets, SSH always demands a key. - let repo = detected; - const https = toHttpsUrl(detected.url); - if (https !== null && (await deps.remoteReachable(https))) { - repo = { ...detected, url: https }; - verbose(ctx, `repo answers over https — using ${https} instead of SSH`); - } - if (await registerBaseRepo(join(targetDir, "bench.config.yaml"), repo)) { - baseRepo = repo; - verbose(ctx, `registered base repo ${repo.name} (${repo.url})`); - // The tool already knows the repo and its HEAD — a human should - // not have to retype them into the demo task's placeholders. - demoTasksPinned = await pinPlaceholderTasks(join(targetDir, "tasks"), repo); - if (demoTasksPinned > 0) { - verbose(ctx, `pinned ${demoTasksPinned} demo task(s) to ${repo.headCommit.slice(0, 12)}`); - } - } - } - } - - // Working copy for the authoring skills: clone the base repo into - // .repos/ (gitignored) so /bench-task explores locally instead of - // re-cloning into a scratchpad. Failure degrades to a hint — the clone - // is a convenience, not a prerequisite of a valid instance. + // The bootstrap decided WHETHER to clone (and where); the clone itself + // is the CLI's job (network stays on this side). Failure degrades to a + // hint — the clone is a convenience, not a prerequisite of a valid + // instance. let baseRepoClone: "cloned" | "failed" | "skipped" = "skipped"; - if (baseRepo !== null) { - ensureIgnored(targetDir, `${BASE_REPOS_DIR}/`); - const cloneDest = join(targetDir, BASE_REPOS_DIR, baseRepo.name); - verbose(ctx, `cloning base repo into ${BASE_REPOS_DIR}/${baseRepo.name}`); - const cloned = await deps.cloneBaseRepo(baseRepo, cloneDest); + if (res.baseRepoClone != null) { + const cloneDest = join(targetDir, res.baseRepoClone.dest); + verbose(ctx, `cloning base repo into ${res.baseRepoClone.dest}`); + const cloned = await deps.cloneBaseRepo(res.baseRepoClone, cloneDest); if (cloned.ok) { baseRepoClone = "cloned"; } else { @@ -277,76 +262,61 @@ export async function runBenchKitInit( } } - const runnerDeps = await installRunnerDependencies(ctx, deps, targetDir); - - const manifest: InstanceManifest = { - templateVersion, - templateRef: requestedRef ?? "latest", - templateSource: TEMPLATE_REPO_URL, - initializedAt: existingManifest?.initializedAt ?? deps.now().toISOString(), - tool: toolId, - ...(existingManifest?.updatedAt === undefined ? {} : { updatedAt: existingManifest.updatedAt }), - ...(baseRepo !== null - ? { detectedBaseRepo: baseRepo } - : existingManifest?.detectedBaseRepo !== undefined - ? { detectedBaseRepo: existingManifest.detectedBaseRepo } - : {}), - }; - writeManifest(targetDir, manifest); - - let committed = false; - if (!repair) { - committed = await freshGitInit(ctx, deps, targetDir, templateVersion); - } - + const repair = res.mode === "repair"; + const toolId = res.tool ?? toolChoice.id; const toolName = PROFILES[toolId]?.displayName ?? toolId; + const skillRoot = res.skillRoot ?? skillRootFor(toolId); const humanLines = repair ? [ - `Repaired the benchmark instance in '${targetDir}' (template ${templateVersion}).`, - `Restored ${copied + skills.added} missing file${copied + skills.added === 1 ? "" : "s"}; your tasks, evaluation pool and config were not touched.`, + `Repaired the benchmark instance in '${targetDir}' (template ${res.templateVersion}).`, + `Restored ${res.filesCopied} missing file${res.filesCopied === 1 ? "" : "s"}; your tasks, evaluation pool and config were not touched.`, ] : [ - `Created a benchmark instance in '${targetDir}' from template ${templateVersion}.`, - `Agent skills installed for ${toolName} under ${skillRootFor(toolId)}/.`, - baseRepo === null + `Created a benchmark instance in '${targetDir}' from template ${res.templateVersion}.`, + `Agent skills installed for ${toolName} under ${skillRoot}/.`, + res.baseRepo == null ? "No product repo detected here — add your base repos to bench.config.yaml." - : `Registered '${baseRepo.name}' (${baseRepo.url}) as the first base repo in bench.config.yaml.`, - ...(demoTasksPinned > 0 - ? [`Pinned the demo task to ${baseRepo?.headCommit.slice(0, 12)} (current HEAD of the base repo).`] + : `Registered '${res.baseRepo.name}' (${res.baseRepo.url}) as the first base repo in bench.config.yaml.`, + ...((res.demoTasksPinned ?? 0) > 0 && detectedBaseRepo !== null + ? [ + `Pinned the demo task to ${detectedBaseRepo.headCommit.slice(0, 12)} (current HEAD of the base repo).`, + ] : []), - ...(baseRepoClone === "cloned" + ...(baseRepoClone === "cloned" && res.baseRepoClone != null ? [ - `Cloned '${baseRepo?.name}' into ${BASE_REPOS_DIR}/${baseRepo?.name}/ — local working copy for the authoring skills (gitignored).`, + `Cloned '${res.baseRepoClone.name}' into ${res.baseRepoClone.dest}/ — local working copy for the authoring skills (gitignored).`, ] - : baseRepoClone === "failed" + : baseRepoClone === "failed" && res.baseRepoClone != null ? [ - `Base repo clone failed — run 'git clone ${baseRepo?.url} ${BASE_REPOS_DIR}/${baseRepo?.name}' yourself.`, + `Base repo clone failed — run 'git clone ${res.baseRepoClone.url} ${res.baseRepoClone.dest}' yourself.`, ] : []), - ...(runnerDeps === "failed" + ...(res.runnerDeps === "failed" ? ["Runner dependencies did not install — run 'npm ci --prefix .bench-kit/runner' yourself."] - : runnerDeps === "installed" + : res.runnerDeps === "installed" ? ["Runner dependencies installed (.bench-kit/runner/node_modules)."] : []), - committed + res.committed === true ? "Initialized a fresh git repository with an initial commit." : "Initialized a fresh git repository (initial commit skipped — commit the files yourself).", + ...(res.warnings ?? []).map((warning) => `Warning: ${warning}`), "Next: wire up secrets, then run 'bench validate' before the first run.", ]; output(ctx, humanLines.join("\n"), { dir: targetDir, mode: repair ? "repair" : "init", - templateVersion, - templateRef: manifest.templateRef, + templateVersion: res.templateVersion, + templateRef: requestedRef ?? "latest", tool: toolId, - skillRoot: skillRootFor(toolId), - filesCopied: copied + skills.added, - baseRepo, + skillRoot, + filesCopied: res.filesCopied, + baseRepo: res.baseRepo ?? null, baseRepoClone, - demoTasksPinned, - runnerDeps, - gitInitialized: !repair, - committed, + demoTasksPinned: res.demoTasksPinned ?? 0, + runnerDeps: res.runnerDeps, + gitInitialized: res.gitInitialized ?? false, + committed: res.committed ?? false, + warnings: res.warnings ?? [], }); } finally { rmSync(scratch, { recursive: true, force: true }); @@ -354,13 +324,10 @@ export async function runBenchKitInit( } /** - * Zone-aware template upgrade. `.bench-kit/` is replaced wholesale (the - * manifest survives, version-bumped); workflows and skills are synced into - * the working tree as an uncommitted proposal — hence the clean-worktree - * gate, so `git diff` afterwards shows exactly what the update changed. - * Company zones (`tasks/`, `evaluation-pool/`, `bench.config.yaml`) are - * never touched. Schema compatibility is `bench validate`'s job — the - * closing hint points there. + * Zone semantics (what gets replaced, synced or left alone) live in the + * kit's bootstrap — and run from the NEW template's clone, so migrations + * travel with the tag. The CLI's part: fail fast on a non-instance or a + * dirty worktree BEFORE the network clone, then render the report. */ export async function runBenchKitUpdate( ctx: OutputContext, @@ -384,7 +351,6 @@ export async function runBenchKitUpdate( ); return; } - const manifest = readManifest(targetDir); // The skills/workflows proposal is delivered as uncommitted changes; a // dirty tree would mix it with unrelated edits and make review impossible. @@ -404,107 +370,75 @@ export async function runBenchKitUpdate( const scratch = mkdtempSync(join(tmpdir(), "bench-kit-")); try { - verbose(ctx, `cloning ${TEMPLATE_REPO_URL} (${requestedRef ?? "latest"}) into ${scratch}`); - const clone = await deps.cloneTemplate(requestedRef, scratch); - if (!clone.ok) { + await cloneTemplateOrDie(ctx, deps, requestedRef, scratch, "update"); + const entry = requireBootstrap(ctx, scratch); + + const request: BootstrapRequest = { + contractVersion: CONTRACT_VERSION, + mode: "update", + templateDir: scratch, + targetDir, + tool: { id: DEFAULT_TOOL, skillRoot: skillRootFor(DEFAULT_TOOL) }, + toolProfiles: allToolProfiles(), + cwd: process.cwd(), + templateRef: requestedRef ?? "latest", + templateSource: TEMPLATE_REPO_URL, + detectedBaseRepo: null, + now: deps.now().toISOString(), + }; + verbose(ctx, `running template bootstrap (${BOOTSTRAP_ENTRY})`); + const res = await deps.runBootstrap(entry, request); + if (!res.ok) { outputError( ctx, - "clone_failed", - `Could not download the template from ${TEMPLATE_REPO_URL}.`, + res.code ?? "bootstrap_failed", + res.message ?? "The template bootstrap failed.", ExitCodes.ERROR, - clone.error - ? `Git said: ${clone.error.trim()}` - : "Check your internet connection and run '10x bench-kit update' again.", + res.hint, ); } - const newVersion = readTemplateVersion(ctx, scratch); - if (newVersion === currentVersion) { - output(ctx, `Already on template ${currentVersion} — nothing to update.`, { + if (res.upToDate === true) { + output(ctx, `Already on template ${res.templateVersion} — nothing to update.`, { dir: targetDir, mode: "update", upToDate: true, - templateVersion: currentVersion, + templateVersion: res.templateVersion, }); return; } - const toolId = - manifest?.tool !== undefined && PROFILES[manifest.tool] ? manifest.tool : DEFAULT_TOOL; - const skillSource = templateSkillSource(scratch); - - // Zone .bench-kit/ — wholesale, atomic-ish replacement: stage the new - // tree (with the bumped manifest) next to the old one, then swap, so a - // crash mid-copy can't leave a versionless half-instance. - const updatedManifest: InstanceManifest = { - templateVersion: newVersion, - templateRef: requestedRef ?? "latest", - templateSource: manifest?.templateSource ?? TEMPLATE_REPO_URL, - initializedAt: manifest?.initializedAt ?? deps.now().toISOString(), - tool: toolId, - updatedAt: deps.now().toISOString(), - ...(manifest?.detectedBaseRepo === undefined - ? {} - : { detectedBaseRepo: manifest.detectedBaseRepo }), - }; - const staging = join(targetDir, ".bench-kit.update-staging"); - rmSync(staging, { recursive: true, force: true }); - cpSync(join(scratch, ".bench-kit"), staging, { recursive: true }); - writeFileSync(join(staging, "instance.json"), `${JSON.stringify(updatedManifest, null, 2)}\n`); - rmSync(join(targetDir, ".bench-kit"), { recursive: true, force: true }); - renameSync(staging, join(targetDir, ".bench-kit")); - - // Zone .github/workflows/ — synced (overwrite): the company reviews the - // resulting diff before committing, same as skills. - const workflows = syncDir( - join(targetDir, ".bench-kit", "workflows"), - join(targetDir, ".github", "workflows"), - { overwrite: true }, - ); - - // Skills zone — the diff proposal: template files are added/overwritten - // in the working tree, company-only skills are never deleted. - const skills = syncDir(join(scratch, skillSource), join(targetDir, skillRootFor(toolId)), { - overwrite: true, - }); - - // Shared root files (AGENTS.md) — same proposal semantics as skills. - const shared: SyncCounts = { added: 0, updated: 0, unchanged: 0 }; - for (const file of SHARED_ROOT_FILES) { - addSync(shared, syncFile(join(scratch, file), join(targetDir, file))); - } - - // The wholesale swap just deleted the runner's node_modules — reinstall, - // so the first `bench` command after update is not MODULE_NOT_FOUND. - const runnerDeps = await installRunnerDependencies(ctx, deps, targetDir); - + const zones = res.zones ?? {}; + const skillRoot = res.skillRoot ?? skillRootFor(res.tool ?? DEFAULT_TOOL); output( ctx, [ - `Updated the benchmark instance from template ${currentVersion} to ${newVersion}.`, + `Updated the benchmark instance from template ${res.fromVersion} to ${res.templateVersion}.`, " .bench-kit/ replaced wholesale (runtime zone)", - ...(runnerDeps === "failed" + ...(res.runnerDeps === "failed" ? [" .bench-kit/runner/ npm ci FAILED — run 'npm ci --prefix .bench-kit/runner' yourself"] - : runnerDeps === "installed" + : res.runnerDeps === "installed" ? [" .bench-kit/runner/ dependencies reinstalled (npm ci)"] : []), - ` .github/workflows/ ${describeSync(workflows)}`, - ` ${`${skillRootFor(toolId)}/`.padEnd(23)}${describeSync(skills)} — proposal, review before committing`, - ` ${SHARED_ROOT_FILES.join(", ").padEnd(23)}${describeSync(shared)} — proposal, review before committing`, + ` .github/workflows/ ${describeSync(zones.workflows)}`, + ` ${`${skillRoot}/`.padEnd(23)}${describeSync(zones.skills)} — proposal, review before committing`, + ` ${"AGENTS.md".padEnd(23)}${describeSync(zones.shared)} — proposal, review before committing`, " tasks/, evaluation-pool/, bench.config.yaml untouched (company zone)", + ...(res.warnings ?? []).map((warning) => `Warning: ${warning}`), "Next: review 'git diff', run 'bench validate' (it flags any schema changes to fix), then commit via PR.", ].join("\n"), { dir: targetDir, mode: "update", upToDate: false, - fromVersion: currentVersion, - templateVersion: newVersion, - templateRef: updatedManifest.templateRef, - tool: toolId, - skillRoot: skillRootFor(toolId), - runnerDeps, - zones: { benchKit: "replaced", workflows, skills, shared }, + fromVersion: res.fromVersion, + templateVersion: res.templateVersion, + templateRef: requestedRef ?? "latest", + tool: res.tool, + skillRoot, + runnerDeps: res.runnerDeps, + zones: res.zones, + warnings: res.warnings ?? [], }, ); } finally { @@ -512,17 +446,23 @@ export async function runBenchKitUpdate( } } +function describeSync(counts: SyncCounts | undefined): string { + if (counts === undefined) return "0 added, 0 updated, 0 unchanged"; + return `${counts.added} added, ${counts.updated} updated, ${counts.unchanged} unchanged`; +} + /** * Resolves the agent tool profile for skill placement: explicit --tool > - * existing manifest (repair) > interactive pick pre-filled by marker - * detection in cwd > detection result > claude-code. + * interactive pick pre-filled by marker detection in cwd > detection + * result > claude-code. On repair the kit's bootstrap prefers the + * instance manifest's tool unless --tool was explicit — hence the flag. */ async function resolveInstanceTool( ctx: OutputContext, options: BenchKitFlags, deps: BenchKitDeps, - existingManifest: InstanceManifest | null, -): Promise { + existingInstance: boolean, +): Promise<{ id: string; explicit: boolean }> { if (options.tool !== undefined) { if (!PROFILES[options.tool]) { outputError( @@ -533,18 +473,14 @@ async function resolveInstanceTool( `Supported: ${Object.keys(PROFILES).join(", ")}.`, ); } - return options.tool; - } - if (existingManifest?.tool !== undefined && PROFILES[existingManifest.tool]) { - return existingManifest.tool; + return { id: options.tool, explicit: true }; } const signals = deps.detectToolSignals(process.cwd()); const top = signals[0]; const detected = top !== undefined && PROFILES[top.profileId] ? top.profileId : null; const initial = detected ?? DEFAULT_TOOL; - const interactive = - options.yes !== true && !ctx.json && process.stdout.isTTY && existingManifest === null; + const interactive = options.yes !== true && !ctx.json && process.stdout.isTTY && !existingInstance; if (!interactive) { verbose( ctx, @@ -552,25 +488,21 @@ async function resolveInstanceTool( ? `no agent-tool markers found — defaulting to ${initial}` : `detected ${initial} (${top?.reason}) — using it as the tool profile`, ); - return initial; + return { id: initial, explicit: false }; } const choice = await deps.chooseTool(initial, top?.reason ?? null); - return choice !== null && PROFILES[choice] ? choice : initial; + return { id: choice !== null && PROFILES[choice] ? choice : initial, explicit: choice !== null }; } -/** Skill root directory (relative) for a tool profile, e.g. `.agents/skills`. */ +/** Skill root (relative, posix — the contract is cross-platform JSON). */ export function skillRootFor(toolId: string): string { const profile = PROFILES[toolId] ?? PROFILES[DEFAULT_TOOL]!; - return join(profile.manifestDir, "skills"); + return `${profile.manifestDir}/skills`; } -/** - * Where the template keeps its skills. Today that is `.claude/skills/`; - * the planned migration to the tool-agnostic `.agents/skills/` convention - * is picked up automatically once the template moves. - */ -function templateSkillSource(templateDir: string): string { - return existsSync(join(templateDir, ".agents", "skills")) ? ".agents/skills" : ".claude/skills"; +/** id → skillRoot for every supported tool — machine knowledge the kit lacks. */ +function allToolProfiles(): Record { + return Object.fromEntries(Object.keys(PROFILES).map((id) => [id, skillRootFor(id)])); } function normalizeRef(ctx: OutputContext, raw: string | undefined): string | null { @@ -607,200 +539,60 @@ async function preflight(ctx: OutputContext, deps: BenchKitDeps): Promise } } -/** Returns the instance's template version, or null when `dir` is not an instance. */ -function readInstanceVersion(dir: string): string | null { - const versionFile = join(dir, ".bench-kit", "VERSION"); - if (!existsSync(versionFile)) return null; - return readFileSync(versionFile, "utf8").trim(); -} - -/** Reads the instance manifest, tolerating its absence (older inits). */ -function readManifest(dir: string): InstanceManifest | null { - const file = join(dir, ".bench-kit", "instance.json"); - if (!existsSync(file)) return null; - try { - return JSON.parse(readFileSync(file, "utf8")) as InstanceManifest; - } catch { - return null; - } -} - -function writeManifest(dir: string, manifest: InstanceManifest): void { - writeFileSync( - join(dir, ".bench-kit", "instance.json"), - `${JSON.stringify(manifest, null, 2)}\n`, - ); -} - -function readTemplateVersion(ctx: OutputContext, cloneDir: string): string { - const versionFile = join(cloneDir, ".bench-kit", "VERSION"); - if (!existsSync(versionFile)) { +async function cloneTemplateOrDie( + ctx: OutputContext, + deps: BenchKitDeps, + ref: string | null, + scratch: string, + action: "init" | "update", +): Promise { + verbose(ctx, `cloning ${TEMPLATE_REPO_URL} (${ref ?? "latest"}) into ${scratch}`); + const clone = await deps.cloneTemplate(ref, scratch); + if (!clone.ok) { outputError( ctx, - "invalid_template", - "The downloaded template has no .bench-kit/VERSION file.", + "clone_failed", + `Could not download the template from ${TEMPLATE_REPO_URL}.`, ExitCodes.ERROR, - "Pass a valid tag via '10x bench-kit init --template-version '.", + clone.error + ? `Git said: ${clone.error.trim()}` + : `Check your internet connection and run '10x bench-kit ${action}' again.`, ); } - return readFileSync(versionFile, "utf8").trim(); } /** - * GitHub only runs workflows from .github/workflows/, so the template's - * .bench-kit/workflows/ files are copied there. In repair mode existing - * files are kept — the company may have customized triggers or secrets. + * The bootstrap entry doubles as the minimum-template-version gate: a tag + * older than 0.10.0 ships no bootstrap, and this CLI no longer carries the + * legacy installer to fall back to. */ -function installWorkflows( - targetDir: string, - opts: { skipExisting: boolean }, -): void { - const srcDir = join(targetDir, ".bench-kit", "workflows"); - syncDir(srcDir, join(targetDir, ".github", "workflows"), { overwrite: !opts.skipExisting }); -} - -/** - * Recursively syncs `srcDir` into `destDir` and counts per-file outcomes. - * Files only ever get added or overwritten — never deleted — so company - * files living alongside template ones survive. With `overwrite: false`, - * existing files are left alone and counted as unchanged. - */ -function syncDir( - srcDir: string, - destDir: string, - opts: { overwrite: boolean }, -): SyncCounts { - const counts: SyncCounts = { added: 0, updated: 0, unchanged: 0 }; - if (!existsSync(srcDir)) return counts; - const walk = (rel: string): void => { - for (const entry of readdirSync(join(srcDir, rel), { withFileTypes: true })) { - const relPath = join(rel, entry.name); - const from = join(srcDir, relPath); - const to = join(destDir, relPath); - if (entry.isDirectory()) { - mkdirSync(to, { recursive: true }); - walk(relPath); - continue; - } - if (!entry.isFile()) continue; - if (!existsSync(to)) { - mkdirSync(join(destDir, rel), { recursive: true }); - cpSync(from, to); - counts.added++; - } else if (readFileSync(from).equals(readFileSync(to))) { - counts.unchanged++; - } else if (opts.overwrite) { - cpSync(from, to); - counts.updated++; - } else { - counts.unchanged++; - } - } - }; - mkdirSync(destDir, { recursive: true }); - walk(""); - return counts; -} - -function describeSync(counts: SyncCounts): string { - return `${counts.added} added, ${counts.updated} updated, ${counts.unchanged} unchanged`; -} - -/** Syncs a single file with the same add/overwrite semantics as syncDir. */ -function syncFile(from: string, to: string): SyncCounts { - const counts: SyncCounts = { added: 0, updated: 0, unchanged: 0 }; - if (!existsSync(from)) return counts; - if (!existsSync(to)) { - cpSync(from, to); - counts.added++; - } else if (readFileSync(from).equals(readFileSync(to))) { - counts.unchanged++; - } else { - cpSync(from, to); - counts.updated++; +function requireBootstrap(ctx: OutputContext, scratch: string): string { + const entry = join(scratch, BOOTSTRAP_ENTRY); + if (!existsSync(entry)) { + outputError( + ctx, + "template_incomplete", + `The downloaded template has no ${BOOTSTRAP_ENTRY} — it predates the bootstrap contract.`, + ExitCodes.ERROR, + "This CLI needs template v0.10.0 or newer; drop --template-version or pass a newer tag.", + ); } - return counts; + return entry; } -function addSync(into: SyncCounts, counts: SyncCounts): void { - into.added += counts.added; - into.updated += counts.updated; - into.unchanged += counts.unchanged; -} - -/** - * Copies the clone into the target without git history. In repair mode - * existing files are never overwritten — company content is untouchable. - * `skip` excludes subtrees handled elsewhere (skills go per tool profile). - * Returns the number of files copied. - */ -function materialize( - srcDir: string, - destDir: string, - opts: { skipExisting: boolean; skip?: (relPath: string) => boolean }, -): number { - let copied = 0; - const walk = (rel: string): void => { - for (const entry of readdirSync(join(srcDir, rel), { withFileTypes: true })) { - if (rel === "" && entry.name === ".git") continue; - const relPath = join(rel, entry.name); - if (opts.skip?.(relPath)) continue; - const from = join(srcDir, relPath); - const to = join(destDir, relPath); - if (entry.isDirectory()) { - mkdirSync(to, { recursive: true }); - walk(relPath); - // A directory whose whole content was skipped (e.g. `.claude/` when - // skills go elsewhere) should not linger empty in the instance. - if (readdirSync(to).length === 0) rmdirSync(to); - continue; - } - if (opts.skipExisting && existsSync(to)) continue; - cpSync(from, to); - copied++; - } - }; - walk(""); - return copied; -} - -/** - * Replaces the template's placeholder base-repo entry with the detected - * repo, editing bench.config.yaml in place (comments preserved via yaml - * document editing). Returns false when the config has no placeholder to - * replace — company content is never overwritten on a guess. - */ -export async function registerBaseRepo( - configPath: string, - repo: DetectedBaseRepo, -): Promise { - if (!existsSync(configPath)) return false; - // Lazy import: yaml is needed only on this path, and a top-level import - // would tax every CLI start (the binary smoke test budgets startup). - const { parseDocument } = await import("yaml"); - const doc = parseDocument(readFileSync(configPath, "utf8")); - const firstName = doc.getIn(["base_repos", 0, "name"]); - if (firstName !== PLACEHOLDER_BASE_REPO) return false; - doc.setIn(["base_repos", 0, "name"], repo.name); - doc.setIn(["base_repos", 0, "url"], repo.url); - // The entry is real now — drop the template's per-field placeholder - // comments (file-level comments stay). - const entry = doc.getIn(["base_repos", 0], true); - if (entry && typeof entry === "object" && "items" in entry) { - for (const pair of (entry as { items: { key?: { commentBefore?: string | null } }[] }).items) { - if (pair.key) pair.key.commentBefore = null; - } - } - writeFileSync(configPath, doc.toString()); - return true; +/** Returns the instance's template version, or null when `dir` is not an instance. */ +function readInstanceVersion(dir: string): string | null { + const versionFile = join(dir, ".bench-kit", "VERSION"); + if (!existsSync(versionFile)) return null; + return readFileSync(versionFile, "utf8").trim(); } /** * Rewrites an SSH remote URL to its https equivalent, or null when the URL - * is already https (or unrecognized). `git@host:org/repo.git` and - * `ssh://git@host/org/repo.git` both map to `https://host/org/repo.git`. + * is already https (or unrecognized). Used only to pick the URL for the + * reachability probe — WHICH url ends up in the config is the kit's call. */ -export function toHttpsUrl(url: string): string | null { +function toHttpsUrl(url: string): string | null { const scp = url.match(/^git@([^:/]+):(.+)$/); if (scp !== null) return `https://${scp[1]}/${scp[2]}`; const ssh = url.match(/^ssh:\/\/(?:[^@/]+@)?([^:/]+)(?::\d+)?\/(.+)$/); @@ -808,105 +600,6 @@ export function toHttpsUrl(url: string): string | null { return null; } -/** - * Guarantees `.gitignore` covers `entry` before the local clone lands — - * the template ships the rule, but an older template tag must not end up - * committing a whole product repo into the instance's initial commit. - */ -function ensureIgnored(dir: string, entry: string): void { - const file = join(dir, ".gitignore"); - const current = existsSync(file) ? readFileSync(file, "utf8") : ""; - if (current.split("\n").some((line) => line.trim() === entry)) return; - const prefix = current === "" || current.endsWith("\n") ? current : `${current}\n`; - writeFileSync(file, `${prefix}${entry}\n`); -} - -/** All-zeros commit the template ships in the demo task. */ -const PLACEHOLDER_COMMIT = /^0{40}$/; - -/** - * Pins template placeholder tasks to the detected base repo: any - * tasks//task.yaml still pointing at the placeholder repo gets the - * detected repo name, and its all-zeros commit gets the detected HEAD. - * Company-authored tasks are never touched (no placeholder → no edit). - * Returns the number of tasks pinned. - */ -export async function pinPlaceholderTasks( - tasksDir: string, - repo: DetectedBaseRepo, -): Promise { - if (!existsSync(tasksDir) || !/^[0-9a-f]{40}$/.test(repo.headCommit)) return 0; - const { parseDocument } = await import("yaml"); - let pinned = 0; - for (const entry of readdirSync(tasksDir, { withFileTypes: true })) { - if (!entry.isDirectory()) continue; - const taskYaml = join(tasksDir, entry.name, "task.yaml"); - if (!existsSync(taskYaml)) continue; - const doc = parseDocument(readFileSync(taskYaml, "utf8")); - if (doc.getIn(["repo"]) !== PLACEHOLDER_BASE_REPO) continue; - const commit = doc.getIn(["commit"]); - doc.setIn(["repo"], repo.name); - if (typeof commit === "string" && PLACEHOLDER_COMMIT.test(commit)) { - doc.setIn(["commit"], repo.headCommit); - } - writeFileSync(taskYaml, doc.toString()); - pinned++; - } - return pinned; -} - -/** - * Installs the runner's dependencies so the first `bench` command does not - * die with MODULE_NOT_FOUND. Returns "skipped" when the template ships no - * runner package.json; a failure degrades to a hint, never blocks init. - */ -async function installRunnerDependencies( - ctx: OutputContext, - deps: BenchKitDeps, - targetDir: string, -): Promise<"installed" | "failed" | "skipped"> { - const runnerDir = join(targetDir, ".bench-kit", "runner"); - if (!existsSync(join(runnerDir, "package.json"))) return "skipped"; - verbose(ctx, "installing runner dependencies (npm ci in .bench-kit/runner)"); - const result = await deps.installRunnerDeps(runnerDir); - if (!result.ok) { - verbose(ctx, `npm ci failed (${result.error.trim().split("\n").pop() ?? ""})`); - return "failed"; - } - return "installed"; -} - -/** Fresh `git init` + first commit. A failed commit degrades to a warning. */ -async function freshGitInit( - ctx: OutputContext, - deps: BenchKitDeps, - dir: string, - templateVersion: string, -): Promise { - const init = await deps.runGit(["init"], dir); - if (!init.ok) { - outputError( - ctx, - "git_init_failed", - "Could not initialize a git repository in the instance directory.", - ExitCodes.ERROR, - init.error ? `Git said: ${init.error.trim()}` : undefined, - ); - } - const add = await deps.runGit(["add", "-A"], dir); - const commit = add.ok - ? await deps.runGit( - ["commit", "-m", `chore: bench-kit init (template ${templateVersion})`], - dir, - ) - : add; - if (!commit.ok) { - verbose(ctx, `initial commit failed (${commit.error.trim()}) — files are staged, commit manually`); - return false; - } - return true; -} - // --------------------------------------------------------------------------- // Default (real) side effects // --------------------------------------------------------------------------- @@ -916,13 +609,18 @@ function run( args: string[], cwd?: string, env?: Record, + stdin?: string, ): Promise<{ ok: boolean; stdout: string; error: string }> { return new Promise((resolvePromise) => { const child = spawn(cmd, args, { cwd, - stdio: ["ignore", "pipe", "pipe"], + stdio: [stdin === undefined ? "ignore" : "pipe", "pipe", "pipe"], ...(env === undefined ? {} : { env: { ...process.env, ...env } }), }); + if (stdin !== undefined) { + child.stdin?.write(stdin); + child.stdin?.end(); + } let stdout = ""; let stderr = ""; child.stdout?.on("data", (chunk: Buffer) => { @@ -982,9 +680,6 @@ const defaultDeps: BenchKitDeps = { }); return result.ok; }, - installRunnerDeps(runnerDir) { - return run("npm", ["ci", "--no-audit", "--no-fund"], runnerDir); - }, detectToolSignals(cwd) { return detectTools(cwd); }, @@ -1009,5 +704,28 @@ const defaultDeps: BenchKitDeps = { } return choice as string; }, + async runBootstrap(entry, request) { + // The bootstrap streams progress on stderr and answers with a single + // JSON object as the LAST line of stdout. + const result = await run( + process.execPath, + [entry], + undefined, + undefined, + JSON.stringify(request), + ); + const lines = result.stdout.trim().split("\n"); + const last = lines[lines.length - 1] ?? ""; + try { + return JSON.parse(last) as BootstrapResponse; + } catch { + return { + ok: false, + code: "bootstrap_failed", + message: "The template bootstrap produced no parsable response.", + hint: result.error.trim().split("\n").slice(-3).join("\n") || undefined, + }; + } + }, now: () => new Date(), }; diff --git a/tests/bench-kit-command.test.ts b/tests/bench-kit-command.test.ts index 65be8fc..8b6da8a 100644 --- a/tests/bench-kit-command.test.ts +++ b/tests/bench-kit-command.test.ts @@ -1,22 +1,36 @@ /** - * 10x bench-kit — command-level behavior. + * 10x bench-kit — command-level behavior, narrowed to the bootstrap + * contract. Disk effects of init/update (zones, manifest, placeholders) + * are the KIT's tests (.github/tests/ in 10x-bench-kit); here we assert + * on the request the CLI builds, the rendering of the response, --json + * envelopes and exit codes. * * Uses dependency injection (BenchKitDeps) instead of module mocking: - * cloneTemplate materializes a fixture template tree, runGit records calls. - * All filesystem work happens in per-test temp directories. + * cloneTemplate materializes a minimal fixture tree (only what the CLI + * itself inspects: .bench-kit/VERSION and the bootstrap entry), + * runBootstrap is a fake returning canned contract responses. */ -import { afterEach, beforeEach, describe, expect, it } from "bun:test"; -import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { afterEach, describe, expect, it } from "bun:test"; +import { + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import cac from "cac"; import { type BenchKitDeps, + type BootstrapRequest, + type BootstrapResponse, + CONTRACT_VERSION, registerBenchKitCommand, runBenchKitInit, runBenchKitUpdate, - toHttpsUrl, } from "../src/commands/bench-kit"; import type { OutputContext } from "../src/lib/output"; @@ -82,83 +96,73 @@ afterEach(() => { } }); -/** Builds a minimal template fixture (with a fake .git to prove it's stripped). */ -function buildTemplateFixture(version = "0.1.0"): string { +/** + * Minimal template fixture: only what the CLI itself looks at before + * handing off — the VERSION marker and the bootstrap entry. + */ +function buildTemplateFixture(version = "0.10.0", withBootstrap = true): string { const dir = tempDir("bench-kit-template-"); - mkdirSync(join(dir, ".git"), { recursive: true }); - writeFileSync(join(dir, ".git", "HEAD"), "ref: refs/heads/main\n"); - mkdirSync(join(dir, ".bench-kit"), { recursive: true }); + mkdirSync(join(dir, ".bench-kit", "bootstrap"), { recursive: true }); writeFileSync(join(dir, ".bench-kit", "VERSION"), `${version}\n`); - mkdirSync(join(dir, ".bench-kit", "workflows"), { recursive: true }); - writeFileSync( - join(dir, ".bench-kit", "workflows", "bench-run.yaml"), - `name: bench-run (${version})\n`, - ); - mkdirSync(join(dir, ".claude", "skills", "bench-task"), { recursive: true }); - writeFileSync( - join(dir, ".claude", "skills", "bench-task", "SKILL.md"), - `# bench-task (${version})\n`, - ); - mkdirSync(join(dir, "tasks", "demo"), { recursive: true }); - writeFileSync(join(dir, "tasks", "demo", "prompt.md"), "demo prompt\n"); - writeFileSync(join(dir, "AGENTS.md"), `# agents (${version})\n`); - writeFileSync( - join(dir, "tasks", "demo", "task.yaml"), - [ - "# Zadanie-demo.", - "repo: demo-app", - "# (placeholder)", - `commit: "${"0".repeat(40)}"`, - "timeout_s: 300", - "", - ].join("\n"), - ); - writeFileSync( - join(dir, "bench.config.yaml"), - [ - "# Konfiguracja instancji benchmarku.", - "base_repos:", - " - name: demo-app", - " # (placeholder)", - " url: git@github.com:example-org/demo-app.git", - "judge:", - " model: anthropic/claude-fable-5", - "", - ].join("\n"), - ); + if (withBootstrap) { + writeFileSync(join(dir, ".bench-kit", "bootstrap", "index.mjs"), "// bootstrap\n"); + } return dir; } +function okInitResponse(overrides: Partial = {}): BootstrapResponse { + return { + ok: true, + mode: "init", + templateVersion: "0.10.0", + tool: "claude-code", + skillRoot: ".claude/skills", + filesCopied: 42, + baseRepo: null, + demoTasksPinned: 0, + baseRepoClone: null, + runnerDeps: "installed", + gitInitialized: true, + committed: true, + warnings: [], + ...overrides, + }; +} + interface FakeDepsResult { deps: BenchKitDeps; - gitCalls: string[][]; + bootstrapCalls: { entry: string; request: BootstrapRequest }[]; } -function fakeDeps(templateDir: string, overrides: Partial = {}): FakeDepsResult { - const gitCalls: string[][] = []; +function fakeDeps( + templateDir: string, + bootstrapResponse: BootstrapResponse, + overrides: Partial = {}, +): FakeDepsResult { + const bootstrapCalls: { entry: string; request: BootstrapRequest }[] = []; const deps: BenchKitDeps = { toolAvailable: () => Promise.resolve(true), cloneTemplate: (_ref, destDir) => { cpSync(templateDir, destDir, { recursive: true }); return Promise.resolve({ ok: true, error: "" }); }, - runGit: (args, _cwd) => { - gitCalls.push(args); - return Promise.resolve({ ok: true, stdout: "", error: "" }); - }, + runGit: () => Promise.resolve({ ok: true, stdout: "", error: "" }), detectBaseRepo: () => Promise.resolve(null), cloneBaseRepo: (_repo, destDir) => { mkdirSync(destDir, { recursive: true }); return Promise.resolve({ ok: true, error: "" }); }, remoteReachable: () => Promise.resolve(false), - installRunnerDeps: () => Promise.resolve({ ok: true, error: "" }), detectToolSignals: () => [], chooseTool: () => Promise.resolve(null), - now: () => new Date("2026-08-13T12:00:00.000Z"), + runBootstrap: (entry, request) => { + bootstrapCalls.push({ entry, request }); + return Promise.resolve(bootstrapResponse); + }, + now: () => new Date("2026-08-17T12:00:00.000Z"), ...overrides, }; - return { deps, gitCalls }; + return { deps, bootstrapCalls }; } function parseEnvelope(stdout: string): { status: string; data?: any; error?: any } { @@ -166,85 +170,69 @@ function parseEnvelope(stdout: string): { status: string; data?: any; error?: an } describe("10x bench-kit init", () => { - it("materializes the template without git history and inits a fresh repo", async () => { + it("builds a contract request and renders the bootstrap's report", async () => { const template = buildTemplateFixture(); const target = join(tempDir("bench-kit-target-"), "instance"); - const { deps, gitCalls } = fakeDeps(template); + const { deps, bootstrapCalls } = fakeDeps(template, okInitResponse()); - const result = await captureStreams(() => - runBenchKitInit(JSON_CTX, target, {}, deps), - ); + const result = await captureStreams(() => runBenchKitInit(JSON_CTX, target, {}, deps)); expect(result.exitCode).toBeUndefined(); - expect(existsSync(join(target, ".bench-kit", "VERSION"))).toBe(true); - expect(existsSync(join(target, "tasks", "demo", "prompt.md"))).toBe(true); - expect(existsSync(join(target, ".git", "HEAD"))).toBe(false); - // GitHub only runs workflows from .github/workflows/ — init installs them there. - expect(readFileSync(join(target, ".github", "workflows", "bench-run.yaml"), "utf8")).toBe( - "name: bench-run (0.1.0)\n", - ); - // Default tool is claude-code — skills land under .claude/skills/. - expect(readFileSync(join(target, ".claude", "skills", "bench-task", "SKILL.md"), "utf8")).toBe( - "# bench-task (0.1.0)\n", - ); - - const manifest = JSON.parse(readFileSync(join(target, ".bench-kit", "instance.json"), "utf8")); - expect(manifest.templateVersion).toBe("0.1.0"); - expect(manifest.templateRef).toBe("latest"); - expect(manifest.initializedAt).toBe("2026-08-13T12:00:00.000Z"); - expect(manifest.tool).toBe("claude-code"); - - expect(gitCalls[0]).toEqual(["init"]); - expect(gitCalls[1]).toEqual(["add", "-A"]); - expect(gitCalls[2]?.[0]).toBe("commit"); + expect(bootstrapCalls.length).toBe(1); + const { entry, request } = bootstrapCalls[0]!; + // The bootstrap runs FROM the scratch clone, not from the instance. + expect(entry.endsWith(join(".bench-kit", "bootstrap", "index.mjs"))).toBe(true); + expect(entry.startsWith(target)).toBe(false); + expect(request.contractVersion).toBe(CONTRACT_VERSION); + expect(request.mode).toBe("init"); + expect(request.targetDir).toBe(target); + expect(request.templateRef).toBe("latest"); + expect(request.tool).toEqual({ id: "claude-code", skillRoot: ".claude/skills" }); + // Machine knowledge travels with the request: the full profile map… + expect(request.toolProfiles["claude-code"]).toBe(".claude/skills"); + expect(Object.keys(request.toolProfiles).length).toBeGreaterThan(1); + // …and a deterministic timestamp. + expect(request.now).toBe("2026-08-17T12:00:00.000Z"); + expect(request.detectedBaseRepo).toBeNull(); const envelope = parseEnvelope(result.stdout); expect(envelope.status).toBe("ok"); expect(envelope.data.mode).toBe("init"); + expect(envelope.data.templateVersion).toBe("0.10.0"); expect(envelope.data.tool).toBe("claude-code"); expect(envelope.data.committed).toBe(true); }); - it("places skills per the --tool profile and records it in the manifest", async () => { + it("passes --tool through as explicit and rejects unknown tools", async () => { const template = buildTemplateFixture(); const target = join(tempDir("bench-kit-target-"), "instance"); - const { deps } = fakeDeps(template); + const { deps, bootstrapCalls } = fakeDeps( + template, + okInitResponse({ tool: "codex", skillRoot: ".agents/skills" }), + ); const result = await captureStreams(() => runBenchKitInit(JSON_CTX, target, { tool: "codex" }, deps), ); - expect(result.exitCode).toBeUndefined(); - expect(readFileSync(join(target, ".agents", "skills", "bench-task", "SKILL.md"), "utf8")).toBe( - "# bench-task (0.1.0)\n", - ); - // The template's .claude/ held only skills — no empty shell is left behind. - expect(existsSync(join(target, ".claude"))).toBe(false); - - const manifest = JSON.parse(readFileSync(join(target, ".bench-kit", "instance.json"), "utf8")); - expect(manifest.tool).toBe("codex"); - const envelope = parseEnvelope(result.stdout); - expect(envelope.data.skillRoot).toBe(join(".agents", "skills")); - }); - - it("rejects an unknown --tool with a usage error", async () => { - const template = buildTemplateFixture(); - const target = join(tempDir("bench-kit-target-"), "instance"); - const { deps } = fakeDeps(template); + expect(bootstrapCalls[0]!.request.tool).toEqual({ + id: "codex", + skillRoot: ".agents/skills", + explicit: true, + }); + expect(parseEnvelope(result.stdout).data.skillRoot).toBe(".agents/skills"); - const result = await captureStreams(() => + const rejected = await captureStreams(() => runBenchKitInit(JSON_CTX, target, { tool: "vim" }, deps), ); - - expect(result.exitCode).toBe(2); - const envelope = parseEnvelope(result.stdout); - expect(envelope.error.code).toBe("unknown_tool"); + expect(rejected.exitCode).toBe(2); + expect(parseEnvelope(rejected.stdout).error.code).toBe("unknown_tool"); }); - it("defaults the tool from marker detection when non-interactive", async () => { + it("defaults the tool from marker detection when non-interactive (not explicit)", async () => { const template = buildTemplateFixture(); const target = join(tempDir("bench-kit-target-"), "instance"); - const { deps } = fakeDeps(template, { + const { deps, bootstrapCalls } = fakeDeps(template, okInitResponse(), { detectToolSignals: () => [ { profileId: "cursor", confidence: "strong", reason: ".cursor/rules/" }, ], @@ -255,90 +243,69 @@ describe("10x bench-kit init", () => { ); expect(result.exitCode).toBeUndefined(); - expect(existsSync(join(target, ".cursor", "skills", "bench-task", "SKILL.md"))).toBe(true); - const manifest = JSON.parse(readFileSync(join(target, ".bench-kit", "instance.json"), "utf8")); - expect(manifest.tool).toBe("cursor"); - }); - - it("registers the surrounding product repo as the first base repo", async () => { - const template = buildTemplateFixture(); - const target = join(tempDir("bench-kit-target-"), "instance"); - const detected = { - rootDir: "/somewhere/shop-app", - name: "shop-app", - url: "git@github.com:acme/shop-app.git", - headCommit: "a".repeat(40), - }; - const { deps } = fakeDeps(template, { - detectBaseRepo: () => Promise.resolve(detected), + expect(bootstrapCalls[0]!.request.tool).toEqual({ + id: "cursor", + skillRoot: ".cursor/skills", }); - - const result = await captureStreams(() => - runBenchKitInit(JSON_CTX, target, {}, deps), - ); - - expect(result.exitCode).toBeUndefined(); - const config = readFileSync(join(target, "bench.config.yaml"), "utf8"); - expect(config).toContain("name: shop-app"); - expect(config).toContain("url: git@github.com:acme/shop-app.git"); - expect(config).not.toContain("demo-app"); - // Comments survive the in-place edit. - expect(config).toContain("# Konfiguracja instancji benchmarku."); - - const manifest = JSON.parse(readFileSync(join(target, ".bench-kit", "instance.json"), "utf8")); - expect(manifest.detectedBaseRepo.name).toBe("shop-app"); - expect(manifest.detectedBaseRepo.headCommit).toBe("a".repeat(40)); - - const envelope = parseEnvelope(result.stdout); - expect(envelope.data.baseRepo.name).toBe("shop-app"); }); - it("pins the placeholder demo task to the detected repo and its HEAD", async () => { + it("detects the product repo, probes https reachability, and forwards both", async () => { const template = buildTemplateFixture(); const target = join(tempDir("bench-kit-target-"), "instance"); - const head = "c".repeat(40); - const { deps } = fakeDeps(template, { + const probed: string[] = []; + const { deps, bootstrapCalls } = fakeDeps(template, okInitResponse(), { detectBaseRepo: () => Promise.resolve({ rootDir: "/somewhere/shop-app", name: "shop-app", url: "git@github.com:acme/shop-app.git", - headCommit: head, + headCommit: "a".repeat(40), }), + remoteReachable: (url) => { + probed.push(url); + return Promise.resolve(true); + }, }); const result = await captureStreams(() => runBenchKitInit(JSON_CTX, target, {}, deps)); expect(result.exitCode).toBeUndefined(); - const taskYaml = readFileSync(join(target, "tasks", "demo", "task.yaml"), "utf8"); - expect(taskYaml).toContain("repo: shop-app"); - expect(taskYaml).toContain(head); - expect(taskYaml).not.toContain("0".repeat(40)); - // Untouched fields and file comments survive the in-place edit. - expect(taskYaml).toContain("timeout_s: 300"); - expect(taskYaml).toContain("# Zadanie-demo."); - const envelope = parseEnvelope(result.stdout); - expect(envelope.data.demoTasksPinned).toBe(1); + // The CLI probes the https twin of the SSH remote; the kit decides + // which URL lands in the config. + expect(probed).toEqual(["https://github.com/acme/shop-app.git"]); + expect(bootstrapCalls[0]!.request.detectedBaseRepo).toEqual({ + rootDir: "/somewhere/shop-app", + name: "shop-app", + url: "git@github.com:acme/shop-app.git", + headCommit: "a".repeat(40), + httpsReachable: true, + }); }); - it("clones the detected repo into .repos/ and gitignores it", async () => { + it("executes the bootstrap's baseRepoClone instruction and reports the outcome", async () => { const template = buildTemplateFixture(); const target = join(tempDir("bench-kit-target-"), "instance"); - const detected = { - rootDir: "/somewhere/shop-app", + const instruction = { name: "shop-app", - url: "git@github.com:acme/shop-app.git", - headCommit: "a".repeat(40), + url: "https://github.com/acme/shop-app.git", + rootDir: "/somewhere/shop-app", + dest: ".repos/shop-app", }; const cloneCalls: { rootDir: string; destDir: string }[] = []; - const { deps } = fakeDeps(template, { - detectBaseRepo: () => Promise.resolve(detected), - cloneBaseRepo: (repo, destDir) => { - cloneCalls.push({ rootDir: repo.rootDir, destDir }); - mkdirSync(destDir, { recursive: true }); - return Promise.resolve({ ok: true, error: "" }); + const { deps } = fakeDeps( + template, + okInitResponse({ + baseRepo: { name: "shop-app", url: instruction.url }, + baseRepoClone: instruction, + }), + { + cloneBaseRepo: (repo, destDir) => { + cloneCalls.push({ rootDir: repo.rootDir, destDir }); + mkdirSync(destDir, { recursive: true }); + return Promise.resolve({ ok: true, error: "" }); + }, }, - }); + ); const result = await captureStreams(() => runBenchKitInit(JSON_CTX, target, {}, deps)); @@ -346,24 +313,27 @@ describe("10x bench-kit init", () => { expect(cloneCalls).toEqual([ { rootDir: "/somewhere/shop-app", destDir: join(target, ".repos", "shop-app") }, ]); - // The clone never enters the instance's git history. - expect(readFileSync(join(target, ".gitignore"), "utf8")).toContain(".repos/"); - expect(parseEnvelope(result.stdout).data.baseRepoClone).toBe("cloned"); + const envelope = parseEnvelope(result.stdout); + expect(envelope.data.baseRepoClone).toBe("cloned"); + expect(envelope.data.baseRepo.name).toBe("shop-app"); }); it("degrades to a hint when the base repo clone fails (init still succeeds)", async () => { const template = buildTemplateFixture(); const target = join(tempDir("bench-kit-target-"), "instance"); - const { deps } = fakeDeps(template, { - detectBaseRepo: () => - Promise.resolve({ - rootDir: "/somewhere/shop-app", + const { deps } = fakeDeps( + template, + okInitResponse({ + baseRepo: { name: "shop-app", url: "https://github.com/acme/shop-app.git" }, + baseRepoClone: { name: "shop-app", - url: "git@github.com:acme/shop-app.git", - headCommit: "a".repeat(40), - }), - cloneBaseRepo: () => Promise.resolve({ ok: false, error: "disk full" }), - }); + url: "https://github.com/acme/shop-app.git", + rootDir: "/somewhere/shop-app", + dest: ".repos/shop-app", + }, + }), + { cloneBaseRepo: () => Promise.resolve({ ok: false, error: "disk full" }) }, + ); const result = await captureStreams(() => runBenchKitInit(JSON_CTX, target, {}, deps)); @@ -372,10 +342,10 @@ describe("10x bench-kit init", () => { expect(parseEnvelope(result.stdout).data.baseRepoClone).toBe("failed"); }); - it("skips the base repo clone when no product repo was detected", async () => { + it("skips the base repo clone when the bootstrap sent no instruction", async () => { const template = buildTemplateFixture(); const target = join(tempDir("bench-kit-target-"), "instance"); - const { deps } = fakeDeps(template, { + const { deps } = fakeDeps(template, okInitResponse(), { cloneBaseRepo: () => { throw new Error("must not be called"); }, @@ -384,341 +354,186 @@ describe("10x bench-kit init", () => { const result = await captureStreams(() => runBenchKitInit(JSON_CTX, target, {}, deps)); expect(result.exitCode).toBeUndefined(); - expect(existsSync(join(target, ".repos"))).toBe(false); expect(parseEnvelope(result.stdout).data.baseRepoClone).toBe("skipped"); }); - it("prefers https over SSH when the repo answers publicly", async () => { - const template = buildTemplateFixture(); - const target = join(tempDir("bench-kit-target-"), "instance"); - const probed: string[] = []; - const { deps } = fakeDeps(template, { - detectBaseRepo: () => - Promise.resolve({ - rootDir: "/somewhere/shop-app", - name: "shop-app", - url: "git@github.com:acme/shop-app.git", - headCommit: "a".repeat(40), - }), - remoteReachable: (url) => { - probed.push(url); - return Promise.resolve(true); - }, - }); - - const result = await captureStreams(() => runBenchKitInit(JSON_CTX, target, {}, deps)); - - expect(result.exitCode).toBeUndefined(); - expect(probed).toEqual(["https://github.com/acme/shop-app.git"]); - const config = readFileSync(join(target, "bench.config.yaml"), "utf8"); - expect(config).toContain("url: https://github.com/acme/shop-app.git"); - expect(config).not.toContain("git@github.com"); - }); - - it("installs runner dependencies when the template ships a runner", async () => { - const template = buildTemplateFixture(); - mkdirSync(join(template, ".bench-kit", "runner"), { recursive: true }); - writeFileSync(join(template, ".bench-kit", "runner", "package.json"), "{}\n"); - const target = join(tempDir("bench-kit-target-"), "instance"); - const installedIn: string[] = []; - const { deps } = fakeDeps(template, { - installRunnerDeps: (runnerDir) => { - installedIn.push(runnerDir); - return Promise.resolve({ ok: true, error: "" }); - }, - }); - - const result = await captureStreams(() => runBenchKitInit(JSON_CTX, target, {}, deps)); - - expect(result.exitCode).toBeUndefined(); - expect(installedIn).toEqual([join(target, ".bench-kit", "runner")]); - expect(parseEnvelope(result.stdout).data.runnerDeps).toBe("installed"); - }); - - it("degrades to a hint when npm ci fails (init still succeeds)", async () => { + it("maps a bootstrap error envelope onto the CLI error contract", async () => { const template = buildTemplateFixture(); - mkdirSync(join(template, ".bench-kit", "runner"), { recursive: true }); - writeFileSync(join(template, ".bench-kit", "runner", "package.json"), "{}\n"); const target = join(tempDir("bench-kit-target-"), "instance"); const { deps } = fakeDeps(template, { - installRunnerDeps: () => Promise.resolve({ ok: false, error: "npm exploded" }), + ok: false, + code: "target_not_empty", + message: "Katalog nie jest pusty.", + hint: "Wskaż pusty katalog.", }); const result = await captureStreams(() => runBenchKitInit(JSON_CTX, target, {}, deps)); - expect(result.exitCode).toBeUndefined(); - expect(parseEnvelope(result.stdout).data.runnerDeps).toBe("failed"); - }); - - it("keeps the placeholder when init runs inside the instance itself", async () => { - const template = buildTemplateFixture(); - const target = join(tempDir("bench-kit-target-"), "instance"); - const { deps } = fakeDeps(template, { - detectBaseRepo: () => - Promise.resolve({ - rootDir: target, - name: "instance", - url: "git@github.com:acme/instance.git", - headCommit: "b".repeat(40), - }), - }); - - const result = await captureStreams(() => - runBenchKitInit(JSON_CTX, target, {}, deps), - ); - - expect(result.exitCode).toBeUndefined(); - const config = readFileSync(join(target, "bench.config.yaml"), "utf8"); - expect(config).toContain("name: demo-app"); - const envelope = parseEnvelope(result.stdout); - expect(envelope.data.baseRepo).toBeNull(); - }); - - it("refuses a non-empty directory that is not an instance", async () => { - const template = buildTemplateFixture(); - const target = tempDir("bench-kit-target-"); - writeFileSync(join(target, "unrelated.txt"), "not an instance\n"); - const { deps, gitCalls } = fakeDeps(template); - - const result = await captureStreams(() => - runBenchKitInit(JSON_CTX, target, {}, deps), - ); - expect(result.exitCode).toBe(1); const envelope = parseEnvelope(result.stdout); expect(envelope.status).toBe("error"); expect(envelope.error.code).toBe("target_not_empty"); - expect(gitCalls.length).toBe(0); + expect(envelope.error.hint).toBe("Wskaż pusty katalog."); }); - it("repairs an existing instance without touching company content", async () => { + it("renders repair mode from the bootstrap's response", async () => { const template = buildTemplateFixture(); const target = tempDir("bench-kit-target-"); - // Existing instance: VERSION present, company file edited, template file missing. + // Existing instance marker — the CLI only reads .bench-kit/VERSION. mkdirSync(join(target, ".bench-kit"), { recursive: true }); - writeFileSync(join(target, ".bench-kit", "VERSION"), "0.1.0\n"); - writeFileSync(join(target, "bench.config.yaml"), "base_repos: [edited by company]\n"); - // Workflow customized by the company (e.g. triggers/secrets) — repair keeps it. - mkdirSync(join(target, ".github", "workflows"), { recursive: true }); - writeFileSync(join(target, ".github", "workflows", "bench-run.yaml"), "name: customized\n"); - // Skill customized by the company — repair keeps it too. - mkdirSync(join(target, ".claude", "skills", "bench-task"), { recursive: true }); - writeFileSync( - join(target, ".claude", "skills", "bench-task", "SKILL.md"), - "# customized skill\n", + writeFileSync(join(target, ".bench-kit", "VERSION"), "0.10.0\n"); + const { deps, bootstrapCalls } = fakeDeps( + template, + okInitResponse({ mode: "repair", filesCopied: 3, gitInitialized: false, committed: false }), ); - const { deps, gitCalls } = fakeDeps(template); - const result = await captureStreams(() => - runBenchKitInit(JSON_CTX, target, {}, deps), - ); + const result = await captureStreams(() => runBenchKitInit(JSON_CTX, target, {}, deps)); expect(result.exitCode).toBeUndefined(); - // Missing template file restored… - expect(existsSync(join(target, "tasks", "demo", "prompt.md"))).toBe(true); - // …company content untouched… - expect(readFileSync(join(target, "bench.config.yaml"), "utf8")).toContain("edited by company"); - expect(readFileSync(join(target, ".github", "workflows", "bench-run.yaml"), "utf8")).toBe( - "name: customized\n", - ); - expect(readFileSync(join(target, ".claude", "skills", "bench-task", "SKILL.md"), "utf8")).toBe( - "# customized skill\n", - ); - // …and no fresh git init in repair mode. - expect(gitCalls.length).toBe(0); - + // Repair keeps mode=init on the wire — the bootstrap decides it's a repair. + expect(bootstrapCalls[0]!.request.mode).toBe("init"); + // No product-repo detection on repair. + expect(bootstrapCalls[0]!.request.detectedBaseRepo).toBeNull(); const envelope = parseEnvelope(result.stdout); expect(envelope.data.mode).toBe("repair"); - }); - - it("keeps the manifest's tool on repair and restores skills at its path", async () => { - const template = buildTemplateFixture(); - const target = tempDir("bench-kit-target-"); - mkdirSync(join(target, ".bench-kit"), { recursive: true }); - writeFileSync(join(target, ".bench-kit", "VERSION"), "0.1.0\n"); - writeFileSync( - join(target, ".bench-kit", "instance.json"), - JSON.stringify( - { - templateVersion: "0.1.0", - templateRef: "latest", - templateSource: "https://github.com/przeprogramowani/10x-bench-kit", - initializedAt: "2026-08-01T00:00:00.000Z", - tool: "codex", - }, - null, - 2, - ), - ); - const { deps } = fakeDeps(template); - - const result = await captureStreams(() => - runBenchKitInit(JSON_CTX, target, {}, deps), - ); - - expect(result.exitCode).toBeUndefined(); - // Skills restored where the instance keeps them, not at the template's path. - expect(existsSync(join(target, ".agents", "skills", "bench-task", "SKILL.md"))).toBe(true); - expect(existsSync(join(target, ".claude"))).toBe(false); - const manifest = JSON.parse(readFileSync(join(target, ".bench-kit", "instance.json"), "utf8")); - expect(manifest.tool).toBe("codex"); - // Original init timestamp survives the repair. - expect(manifest.initializedAt).toBe("2026-08-01T00:00:00.000Z"); + expect(envelope.data.filesCopied).toBe(3); }); it("rejects --template-version on an existing instance, pointing to update", async () => { const template = buildTemplateFixture(); const target = tempDir("bench-kit-target-"); mkdirSync(join(target, ".bench-kit"), { recursive: true }); - writeFileSync(join(target, ".bench-kit", "VERSION"), "0.1.0\n"); - const { deps } = fakeDeps(template); + writeFileSync(join(target, ".bench-kit", "VERSION"), "0.10.0\n"); + const { deps, bootstrapCalls } = fakeDeps(template, okInitResponse()); const result = await captureStreams(() => - runBenchKitInit(JSON_CTX, target, { templateVersion: "v0.2.0" }, deps), + runBenchKitInit(JSON_CTX, target, { templateVersion: "v0.11.0" }, deps), ); expect(result.exitCode).toBe(2); const envelope = parseEnvelope(result.stdout); expect(envelope.error.code).toBe("version_conflict"); expect(envelope.error.hint).toContain("10x bench-kit update"); + expect(bootstrapCalls.length).toBe(0); }); it("fails preflight when git is missing", async () => { const template = buildTemplateFixture(); const target = join(tempDir("bench-kit-target-"), "instance"); - const { deps } = fakeDeps(template, { + const { deps } = fakeDeps(template, okInitResponse(), { toolAvailable: (cmd) => Promise.resolve(cmd !== "git"), }); - const result = await captureStreams(() => - runBenchKitInit(JSON_CTX, target, {}, deps), - ); + const result = await captureStreams(() => runBenchKitInit(JSON_CTX, target, {}, deps)); expect(result.exitCode).toBe(1); - const envelope = parseEnvelope(result.stdout); - expect(envelope.error.code).toBe("preflight_failed"); + expect(parseEnvelope(result.stdout).error.code).toBe("preflight_failed"); }); - it("surfaces clone failures without leaving a half-written instance", async () => { + it("surfaces clone failures without calling the bootstrap", async () => { const template = buildTemplateFixture(); const target = join(tempDir("bench-kit-target-"), "instance"); - const { deps } = fakeDeps(template, { + const { deps, bootstrapCalls } = fakeDeps(template, okInitResponse(), { cloneTemplate: () => Promise.resolve({ ok: false, error: "fatal: repository not found" }), }); - const result = await captureStreams(() => - runBenchKitInit(JSON_CTX, target, {}, deps), - ); + const result = await captureStreams(() => runBenchKitInit(JSON_CTX, target, {}, deps)); expect(result.exitCode).toBe(1); - const envelope = parseEnvelope(result.stdout); - expect(envelope.error.code).toBe("clone_failed"); + expect(parseEnvelope(result.stdout).error.code).toBe("clone_failed"); + expect(bootstrapCalls.length).toBe(0); expect(existsSync(target)).toBe(false); }); -}); -describe("10x bench-kit update", () => { - /** Inits a fresh instance from the 0.1.0 fixture and returns its dir. */ - async function initInstance(overrides: Partial = {}): Promise { - const template = buildTemplateFixture("0.1.0"); + it("rejects a template without the bootstrap entry (pre-contract tag)", async () => { + const template = buildTemplateFixture("0.9.0", false); const target = join(tempDir("bench-kit-target-"), "instance"); - const { deps } = fakeDeps(template, overrides); + const { deps, bootstrapCalls } = fakeDeps(template, okInitResponse()); + const result = await captureStreams(() => runBenchKitInit(JSON_CTX, target, {}, deps)); - expect(result.exitCode).toBeUndefined(); + + expect(result.exitCode).toBe(1); + const envelope = parseEnvelope(result.stdout); + expect(envelope.error.code).toBe("template_incomplete"); + expect(envelope.error.hint).toContain("v0.10.0"); + expect(bootstrapCalls.length).toBe(0); + }); +}); + +describe("10x bench-kit update", () => { + function existingInstance(version = "0.9.0"): string { + const target = tempDir("bench-kit-target-"); + mkdirSync(join(target, ".bench-kit"), { recursive: true }); + writeFileSync(join(target, ".bench-kit", "VERSION"), `${version}\n`); return target; } - it("updates zone by zone: runtime replaced, skills proposed, company content untouched", async () => { - const target = await initInstance(); - // Company edits since init: a custom skill, an edited task, an edited config. - mkdirSync(join(target, ".claude", "skills", "company-skill"), { recursive: true }); - writeFileSync(join(target, ".claude", "skills", "company-skill", "SKILL.md"), "# ours\n"); - writeFileSync(join(target, "bench.config.yaml"), "base_repos: [edited by company]\n"); - // Stale runtime file that disappeared from the template — replacement drops it. - writeFileSync(join(target, ".bench-kit", "obsolete.txt"), "old runtime file\n"); - - const newTemplate = buildTemplateFixture("0.2.0"); - const { deps } = fakeDeps(newTemplate); - const result = await captureStreams(() => - runBenchKitUpdate(JSON_CTX, target, {}, deps), - ); + const okUpdateResponse: BootstrapResponse = { + ok: true, + mode: "update", + upToDate: false, + fromVersion: "0.9.0", + templateVersion: "0.10.0", + tool: "claude-code", + skillRoot: ".claude/skills", + runnerDeps: "installed", + zones: { + workflows: { added: 0, updated: 2, unchanged: 0 }, + skills: { added: 1, updated: 3, unchanged: 6 }, + shared: { added: 0, updated: 1, unchanged: 0 }, + }, + warnings: [], + }; + + it("sends an update request with the tool profile map and renders the zone report", async () => { + const template = buildTemplateFixture(); + const target = existingInstance(); + const { deps, bootstrapCalls } = fakeDeps(template, okUpdateResponse); + + const result = await captureStreams(() => runBenchKitUpdate(JSON_CTX, target, {}, deps)); expect(result.exitCode).toBeUndefined(); - // Runtime zone replaced wholesale. - expect(readFileSync(join(target, ".bench-kit", "VERSION"), "utf8").trim()).toBe("0.2.0"); - expect(existsSync(join(target, ".bench-kit", "obsolete.txt"))).toBe(false); - // Workflows synced to the new template version. - expect(readFileSync(join(target, ".github", "workflows", "bench-run.yaml"), "utf8")).toBe( - "name: bench-run (0.2.0)\n", - ); - // Skills: template skill updated in place (the git diff is the proposal)… - expect(readFileSync(join(target, ".claude", "skills", "bench-task", "SKILL.md"), "utf8")).toBe( - "# bench-task (0.2.0)\n", - ); - // …company-only skill never deleted. - expect(readFileSync(join(target, ".claude", "skills", "company-skill", "SKILL.md"), "utf8")).toBe( - "# ours\n", - ); - // Company zone untouched. - expect(readFileSync(join(target, "bench.config.yaml"), "utf8")).toContain("edited by company"); - // Shared root files (AGENTS.md) synced like skills — a reviewable proposal. - expect(readFileSync(join(target, "AGENTS.md"), "utf8")).toBe("# agents (0.2.0)\n"); - - // Manifest survives the wholesale replacement, version-bumped. - const manifest = JSON.parse(readFileSync(join(target, ".bench-kit", "instance.json"), "utf8")); - expect(manifest.templateVersion).toBe("0.2.0"); - expect(manifest.initializedAt).toBe("2026-08-13T12:00:00.000Z"); - expect(manifest.updatedAt).toBe("2026-08-13T12:00:00.000Z"); - expect(manifest.tool).toBe("claude-code"); + const { request } = bootstrapCalls[0]!; + expect(request.mode).toBe("update"); + expect(request.contractVersion).toBe(CONTRACT_VERSION); + // The manifest's tool is resolved by the KIT — the CLI only ships the map. + expect(request.toolProfiles["codex"]).toBe(".agents/skills"); const envelope = parseEnvelope(result.stdout); expect(envelope.data.mode).toBe("update"); - expect(envelope.data.fromVersion).toBe("0.1.0"); - expect(envelope.data.templateVersion).toBe("0.2.0"); - expect(envelope.data.zones.benchKit).toBe("replaced"); - expect(envelope.data.zones.skills.updated).toBe(1); - expect(envelope.data.zones.shared.updated).toBe(1); + expect(envelope.data.fromVersion).toBe("0.9.0"); + expect(envelope.data.templateVersion).toBe("0.10.0"); + expect(envelope.data.zones.skills.updated).toBe(3); }); - it("reinstalls runner dependencies after the wholesale .bench-kit swap", async () => { - const target = await initInstance(); - const newTemplate = buildTemplateFixture("0.2.0"); - mkdirSync(join(newTemplate, ".bench-kit", "runner"), { recursive: true }); - writeFileSync(join(newTemplate, ".bench-kit", "runner", "package.json"), "{}\n"); - const installedIn: string[] = []; - const { deps } = fakeDeps(newTemplate, { - installRunnerDeps: (runnerDir) => { - installedIn.push(runnerDir); - return Promise.resolve({ ok: true, error: "" }); - }, + it("reports up-to-date without a zone report", async () => { + const template = buildTemplateFixture(); + const target = existingInstance("0.10.0"); + const { deps } = fakeDeps(template, { + ok: true, + mode: "update", + upToDate: true, + templateVersion: "0.10.0", }); const result = await captureStreams(() => runBenchKitUpdate(JSON_CTX, target, {}, deps)); - expect(result.exitCode).toBeUndefined(); - expect(installedIn).toEqual([join(target, ".bench-kit", "runner")]); - expect(parseEnvelope(result.stdout).data.runnerDeps).toBe("installed"); - }); - - it("is a no-op when the instance is already on the template version", async () => { - const target = await initInstance(); - const sameTemplate = buildTemplateFixture("0.1.0"); - const { deps } = fakeDeps(sameTemplate); - - const result = await captureStreams(() => - runBenchKitUpdate(JSON_CTX, target, {}, deps), - ); - expect(result.exitCode).toBeUndefined(); const envelope = parseEnvelope(result.stdout); expect(envelope.data.upToDate).toBe(true); + expect(envelope.data.templateVersion).toBe("0.10.0"); }); - it("refuses to update a dirty worktree so the proposal stays reviewable", async () => { - const target = await initInstance(); - const newTemplate = buildTemplateFixture("0.2.0"); - const { deps } = fakeDeps(newTemplate, { - runGit: (args, _cwd) => + it("refuses to update a dirty worktree before any network clone", async () => { + const template = buildTemplateFixture(); + const target = existingInstance(); + const cloneCalls: string[] = []; + const { deps, bootstrapCalls } = fakeDeps(template, okUpdateResponse, { + cloneTemplate: (_ref, destDir) => { + cloneCalls.push(destDir); + return Promise.resolve({ ok: true, error: "" }); + }, + runGit: (args) => Promise.resolve( args[0] === "status" ? { ok: true, stdout: " M bench.config.yaml\n", error: "" } @@ -726,49 +541,40 @@ describe("10x bench-kit update", () => { ), }); - const result = await captureStreams(() => - runBenchKitUpdate(JSON_CTX, target, {}, deps), - ); + const result = await captureStreams(() => runBenchKitUpdate(JSON_CTX, target, {}, deps)); expect(result.exitCode).toBe(1); - const envelope = parseEnvelope(result.stdout); - expect(envelope.error.code).toBe("dirty_worktree"); - // Nothing was touched. - expect(readFileSync(join(target, ".bench-kit", "VERSION"), "utf8").trim()).toBe("0.1.0"); + expect(parseEnvelope(result.stdout).error.code).toBe("dirty_worktree"); + expect(cloneCalls.length).toBe(0); + expect(bootstrapCalls.length).toBe(0); }); - it("syncs skills to the manifest's tool path, not the template's", async () => { - const target = await initInstance({ chooseTool: () => Promise.resolve(null) }); - // Simulate an instance initialized for codex. - const manifestPath = join(target, ".bench-kit", "instance.json"); - const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); - manifest.tool = "codex"; - writeFileSync(manifestPath, JSON.stringify(manifest, null, 2)); + it("rejects a directory that is not an instance without cloning", async () => { + const template = buildTemplateFixture(); + const target = tempDir("bench-kit-target-"); + const { deps, bootstrapCalls } = fakeDeps(template, okUpdateResponse); - const newTemplate = buildTemplateFixture("0.2.0"); - const { deps } = fakeDeps(newTemplate); - const result = await captureStreams(() => - runBenchKitUpdate(JSON_CTX, target, {}, deps), - ); + const result = await captureStreams(() => runBenchKitUpdate(JSON_CTX, target, {}, deps)); - expect(result.exitCode).toBeUndefined(); - expect(readFileSync(join(target, ".agents", "skills", "bench-task", "SKILL.md"), "utf8")).toBe( - "# bench-task (0.2.0)\n", - ); + expect(result.exitCode).toBe(1); + expect(parseEnvelope(result.stdout).error.code).toBe("not_an_instance"); + expect(bootstrapCalls.length).toBe(0); }); - it("rejects a directory that is not an instance", async () => { - const template = buildTemplateFixture("0.2.0"); - const target = tempDir("bench-kit-target-"); - const { deps } = fakeDeps(template); + it("maps the bootstrap's dirty_tree defense onto the CLI error contract", async () => { + const template = buildTemplateFixture(); + const target = existingInstance(); + const { deps } = fakeDeps(template, { + ok: false, + code: "dirty_tree", + message: "Instancja ma niezacommitowane zmiany.", + hint: "Zacommituj albo zestashuj.", + }); - const result = await captureStreams(() => - runBenchKitUpdate(JSON_CTX, target, {}, deps), - ); + const result = await captureStreams(() => runBenchKitUpdate(JSON_CTX, target, {}, deps)); expect(result.exitCode).toBe(1); - const envelope = parseEnvelope(result.stdout); - expect(envelope.error.code).toBe("not_an_instance"); + expect(parseEnvelope(result.stdout).error.code).toBe("dirty_tree"); }); }); @@ -783,18 +589,6 @@ async function runCli(argv: string[]): Promise { }); } -describe("toHttpsUrl", () => { - it("maps scp-style and ssh:// URLs to https, leaves the rest alone", () => { - expect(toHttpsUrl("git@github.com:acme/shop.git")).toBe("https://github.com/acme/shop.git"); - expect(toHttpsUrl("ssh://git@github.com/acme/shop.git")).toBe("https://github.com/acme/shop.git"); - expect(toHttpsUrl("ssh://git@gitlab.example.com:2222/team/app")).toBe( - "https://gitlab.example.com/team/app", - ); - expect(toHttpsUrl("https://github.com/acme/shop.git")).toBeNull(); - expect(toHttpsUrl("/local/path/to/repo")).toBeNull(); - }); -}); - describe("10x bench-kit dispatch", () => { it("registers bench-kit unconditionally", () => { const cli = cac("10x");