Skip to content

feat(bench-kit): init safe defaults — pin demo task, install runner deps, prefer https #34

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
psmyrdek merged 1 commit into from
Aug 16, 2026
Merged

feat(bench-kit): init safe defaults — pin demo task, install runner deps, prefer https #34

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 128 additions & 4 deletions src/commands/bench-kit.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,10 @@ export interface BenchKitDeps {
runGit(args: string[], cwd: string): Promise<{ ok: boolean; stdout: string; error: string }>;
/** Detects the git repo containing `cwd` (null when absent or origin-less). */
detectBaseRepo(cwd: string): Promise<DetectedBaseRepo | null>;
/** True when `git ls-remote` succeeds against `url` (https preference probe). */
remoteReachable(url: string): Promise<boolean>;
/** Installs runner dependencies (`npm ci`) inside `runnerDir`. */
installRunnerDeps(runnerDir: string): Promise<{ ok: boolean; error: string }>;
/** Scans `cwd` for agent-tool markers (ranked, strongest first). */
detectToolSignals(cwd: string): DetectionSignal[];
/** Interactive tool picker; resolves the chosen id, or null on cancel. */
Expand Down Expand Up @@ -228,16 +232,33 @@ export async function runBenchKitInit(
// Running init from inside a product repo is the common flow — register
// that repo as the first base repo instead of leaving the placeholder.
let baseRepo: DetectedBaseRepo | null = null;
let demoTasksPinned = 0;
if (!repair) {
const detected = await deps.detectBaseRepo(process.cwd());
if (detected !== null && resolve(detected.rootDir) !== targetDir) {
if (await registerBaseRepo(join(targetDir, "bench.config.yaml"), detected)) {
baseRepo = detected;
verbose(ctx, `registered base repo ${detected.name} (${detected.url})`);
// Prefer https over SSH when the repo answers publicly: https clones
// in CI/containers with zero secrets, SSH always demands a key.
let repo = detected;
const https = toHttpsUrl(detected.url);
if (https !== null && (await deps.remoteReachable(https))) {
repo = { ...detected, url: https };
verbose(ctx, `repo answers over https — using ${https} instead of SSH`);
}
if (await registerBaseRepo(join(targetDir, "bench.config.yaml"), repo)) {
baseRepo = repo;
verbose(ctx, `registered base repo ${repo.name} (${repo.url})`);
// The tool already knows the repo and its HEAD — a human should
// not have to retype them into the demo task's placeholders.
demoTasksPinned = await pinPlaceholderTasks(join(targetDir, "tasks"), repo);
if (demoTasksPinned > 0) {
verbose(ctx, `pinned ${demoTasksPinned} demo task(s) to ${repo.headCommit.slice(0, 12)}`);
}
}
}
}

const runnerDeps = await installRunnerDependencies(ctx, deps, targetDir);

const manifest: InstanceManifest = {
templateVersion,
templateRef: requestedRef ?? "latest",
Expand Down Expand Up @@ -270,6 +291,14 @@ export async function runBenchKitInit(
baseRepo === null
? "No product repo detected here — add your base repos to bench.config.yaml."
: `Registered '${baseRepo.name}' (${baseRepo.url}) as the first base repo in bench.config.yaml.`,
...(demoTasksPinned > 0
? [`Pinned the demo task to ${baseRepo?.headCommit.slice(0, 12)} (current HEAD of the base repo).`]
: []),
...(runnerDeps === "failed"
? ["Runner dependencies did not install — run 'npm ci --prefix .bench-kit/runner' yourself."]
: runnerDeps === "installed"
? ["Runner dependencies installed (.bench-kit/runner/node_modules)."]
: []),
committed
? "Initialized a fresh git repository with an initial commit."
: "Initialized a fresh git repository (initial commit skipped — commit the files yourself).",
Expand All @@ -284,6 +313,8 @@ export async function runBenchKitInit(
skillRoot: skillRootFor(toolId),
filesCopied: copied + skills.added,
baseRepo,
demoTasksPinned,
runnerDeps,
gitInitialized: !repair,
committed,
});
Expand Down Expand Up @@ -413,11 +444,20 @@ export async function runBenchKitUpdate(
addSync(shared, syncFile(join(scratch, file), join(targetDir, file)));
}

// The wholesale swap just deleted the runner's node_modules — reinstall,
// so the first `bench` command after update is not MODULE_NOT_FOUND.
const runnerDeps = await installRunnerDependencies(ctx, deps, targetDir);

output(
ctx,
[
`Updated the benchmark instance from template ${currentVersion} to ${newVersion}.`,
" .bench-kit/ replaced wholesale (runtime zone)",
...(runnerDeps === "failed"
? [" .bench-kit/runner/ npm ci FAILED — run 'npm ci --prefix .bench-kit/runner' yourself"]
: runnerDeps === "installed"
? [" .bench-kit/runner/ dependencies reinstalled (npm ci)"]
: []),
` .github/workflows/ ${describeSync(workflows)}`,
` ${`${skillRootFor(toolId)}/`.padEnd(23)}${describeSync(skills)} — proposal, review before committing`,
` ${SHARED_ROOT_FILES.join(", ").padEnd(23)}${describeSync(shared)} — proposal, review before committing`,
Expand All @@ -433,6 +473,7 @@ export async function runBenchKitUpdate(
templateRef: updatedManifest.templateRef,
tool: toolId,
skillRoot: skillRootFor(toolId),
runnerDeps,
zones: { benchKit: "replaced", workflows, skills, shared },
},
);
Expand Down Expand Up @@ -724,6 +765,74 @@ export async function registerBaseRepo(
return true;
}

/**
* Rewrites an SSH remote URL to its https equivalent, or null when the URL
* is already https (or unrecognized). `git@host:org/repo.git` and
* `ssh://git@host/org/repo.git` both map to `https://host/org/repo.git`.
*/
export function toHttpsUrl(url: string): string | null {
const scp = url.match(/^git@([^:/]+):(.+)$/);
if (scp !== null) return `https://${scp[1]}/${scp[2]}`;
const ssh = url.match(/^ssh:\/\/(?:[^@/]+@)?([^:/]+)(?::\d+)?\/(.+)$/);
if (ssh !== null) return `https://${ssh[1]}/${ssh[2]}`;
return null;
}

/** All-zeros commit the template ships in the demo task. */
const PLACEHOLDER_COMMIT = /^0{40}$/;

/**
* Pins template placeholder tasks to the detected base repo: any
* tasks/<x>/task.yaml still pointing at the placeholder repo gets the
* detected repo name, and its all-zeros commit gets the detected HEAD.
* Company-authored tasks are never touched (no placeholder → no edit).
* Returns the number of tasks pinned.
*/
export async function pinPlaceholderTasks(
tasksDir: string,
repo: DetectedBaseRepo,
): Promise<number> {
if (!existsSync(tasksDir) || !/^[0-9a-f]{40}$/.test(repo.headCommit)) return 0;
const { parseDocument } = await import("yaml");
let pinned = 0;
for (const entry of readdirSync(tasksDir, { withFileTypes: true })) {
if (!entry.isDirectory()) continue;
const taskYaml = join(tasksDir, entry.name, "task.yaml");
if (!existsSync(taskYaml)) continue;
const doc = parseDocument(readFileSync(taskYaml, "utf8"));
if (doc.getIn(["repo"]) !== PLACEHOLDER_BASE_REPO) continue;
const commit = doc.getIn(["commit"]);
doc.setIn(["repo"], repo.name);
if (typeof commit === "string" && PLACEHOLDER_COMMIT.test(commit)) {
doc.setIn(["commit"], repo.headCommit);
}
writeFileSync(taskYaml, doc.toString());
pinned++;
}
return pinned;
}

/**
* Installs the runner's dependencies so the first `bench` command does not
* die with MODULE_NOT_FOUND. Returns "skipped" when the template ships no
* runner package.json; a failure degrades to a hint, never blocks init.
*/
async function installRunnerDependencies(
ctx: OutputContext,
deps: BenchKitDeps,
targetDir: string,
): Promise<"installed" | "failed" | "skipped"> {
const runnerDir = join(targetDir, ".bench-kit", "runner");
if (!existsSync(join(runnerDir, "package.json"))) return "skipped";
verbose(ctx, "installing runner dependencies (npm ci in .bench-kit/runner)");
const result = await deps.installRunnerDeps(runnerDir);
if (!result.ok) {
verbose(ctx, `npm ci failed (${result.error.trim().split("\n").pop() ?? ""})`);
return "failed";
}
return "installed";
}

/** Fresh `git init` + first commit. A failed commit degrades to a warning. */
async function freshGitInit(
ctx: OutputContext,
Expand Down Expand Up @@ -763,9 +872,14 @@ function run(
cmd: string,
args: string[],
cwd?: string,
env?: Record<string, string>,
): Promise<{ ok: boolean; stdout: string; error: string }> {
return new Promise((resolvePromise) => {
const child = spawn(cmd, args, { cwd, stdio: ["ignore", "pipe", "pipe"] });
const child = spawn(cmd, args, {
cwd,
stdio: ["ignore", "pipe", "pipe"],
...(env === undefined ? {} : { env: { ...process.env, ...env } }),
});
let stdout = "";
let stderr = "";
child.stdout?.on("data", (chunk: Buffer) => {
Expand Down Expand Up @@ -810,6 +924,16 @@ const defaultDeps: BenchKitDeps = {
if (headCommit === null) return null;
return { rootDir, name: basename(rootDir), url, headCommit };
},
async remoteReachable(url) {
// No terminal prompt: an auth-gated remote must fail fast, not hang.
const result = await run("git", ["ls-remote", "--heads", url], undefined, {
GIT_TERMINAL_PROMPT: "0",
});
return result.ok;
},
installRunnerDeps(runnerDir) {
return run("npm", ["ci", "--no-audit", "--no-fund"], runnerDir);
},
detectToolSignals(cwd) {
return detectTools(cwd);
},
Expand Down
136 changes: 136 additions & 0 deletions tests/bench-kit-command.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
registerBenchKitCommand,
runBenchKitInit,
runBenchKitUpdate,
toHttpsUrl,
} from "../src/commands/bench-kit";
import { EXPERIMENTAL_ENV, experimentalEnabled } from "../src/lib/experimental";
import type { OutputContext } from "../src/lib/output";
Expand Down Expand Up @@ -102,6 +103,17 @@ function buildTemplateFixture(version = "0.1.0"): string {
mkdirSync(join(dir, "tasks", "demo"), { recursive: true });
writeFileSync(join(dir, "tasks", "demo", "prompt.md"), "demo prompt\n");
writeFileSync(join(dir, "AGENTS.md"), `# agents (${version})\n`);
writeFileSync(
join(dir, "tasks", "demo", "task.yaml"),
[
"# Zadanie-demo.",
"repo: demo-app",
"# (placeholder)",
`commit: "${"0".repeat(40)}"`,
"timeout_s: 300",
"",
].join("\n"),
);
writeFileSync(
join(dir, "bench.config.yaml"),
[
Expand Down Expand Up @@ -136,6 +148,8 @@ function fakeDeps(templateDir: string, overrides: Partial<BenchKitDeps> = {}): F
return Promise.resolve({ ok: true, stdout: "", error: "" });
},
detectBaseRepo: () => Promise.resolve(null),
remoteReachable: () => Promise.resolve(false),
installRunnerDeps: () => Promise.resolve({ ok: true, error: "" }),
detectToolSignals: () => [],
chooseTool: () => Promise.resolve(null),
now: () => new Date("2026-08-13T12:00:00.000Z"),
Expand Down Expand Up @@ -276,6 +290,96 @@ describe("10x bench-kit init", () => {
expect(envelope.data.baseRepo.name).toBe("shop-app");
});

it("pins the placeholder demo task to the detected repo and its HEAD", async () => {
const template = buildTemplateFixture();
const target = join(tempDir("bench-kit-target-"), "instance");
const head = "c".repeat(40);
const { deps } = fakeDeps(template, {
detectBaseRepo: () =>
Promise.resolve({
rootDir: "/somewhere/shop-app",
name: "shop-app",
url: "git@github.com:acme/shop-app.git",
headCommit: head,
}),
});

const result = await captureStreams(() => runBenchKitInit(JSON_CTX, target, {}, deps));

expect(result.exitCode).toBeUndefined();
const taskYaml = readFileSync(join(target, "tasks", "demo", "task.yaml"), "utf8");
expect(taskYaml).toContain("repo: shop-app");
expect(taskYaml).toContain(head);
expect(taskYaml).not.toContain("0".repeat(40));
// Untouched fields and file comments survive the in-place edit.
expect(taskYaml).toContain("timeout_s: 300");
expect(taskYaml).toContain("# Zadanie-demo.");
const envelope = parseEnvelope(result.stdout);
expect(envelope.data.demoTasksPinned).toBe(1);
});

it("prefers https over SSH when the repo answers publicly", async () => {
const template = buildTemplateFixture();
const target = join(tempDir("bench-kit-target-"), "instance");
const probed: string[] = [];
const { deps } = fakeDeps(template, {
detectBaseRepo: () =>
Promise.resolve({
rootDir: "/somewhere/shop-app",
name: "shop-app",
url: "git@github.com:acme/shop-app.git",
headCommit: "a".repeat(40),
}),
remoteReachable: (url) => {
probed.push(url);
return Promise.resolve(true);
},
});

const result = await captureStreams(() => runBenchKitInit(JSON_CTX, target, {}, deps));

expect(result.exitCode).toBeUndefined();
expect(probed).toEqual(["https://github.com/acme/shop-app.git"]);
const config = readFileSync(join(target, "bench.config.yaml"), "utf8");
expect(config).toContain("url: https://github.com/acme/shop-app.git");
expect(config).not.toContain("git@github.com");
});

it("installs runner dependencies when the template ships a runner", async () => {
const template = buildTemplateFixture();
mkdirSync(join(template, ".bench-kit", "runner"), { recursive: true });
writeFileSync(join(template, ".bench-kit", "runner", "package.json"), "{}\n");
const target = join(tempDir("bench-kit-target-"), "instance");
const installedIn: string[] = [];
const { deps } = fakeDeps(template, {
installRunnerDeps: (runnerDir) => {
installedIn.push(runnerDir);
return Promise.resolve({ ok: true, error: "" });
},
});

const result = await captureStreams(() => runBenchKitInit(JSON_CTX, target, {}, deps));

expect(result.exitCode).toBeUndefined();
expect(installedIn).toEqual([join(target, ".bench-kit", "runner")]);
expect(parseEnvelope(result.stdout).data.runnerDeps).toBe("installed");
});

it("degrades to a hint when npm ci fails (init still succeeds)", async () => {
const template = buildTemplateFixture();
mkdirSync(join(template, ".bench-kit", "runner"), { recursive: true });
writeFileSync(join(template, ".bench-kit", "runner", "package.json"), "{}\n");
const target = join(tempDir("bench-kit-target-"), "instance");
const { deps } = fakeDeps(template, {
installRunnerDeps: () => Promise.resolve({ ok: false, error: "npm exploded" }),
});

const result = await captureStreams(() => runBenchKitInit(JSON_CTX, target, {}, deps));

expect(result.exitCode).toBeUndefined();
expect(parseEnvelope(result.stdout).data.runnerDeps).toBe("failed");
});

it("keeps the placeholder when init runs inside the instance itself", async () => {
const template = buildTemplateFixture();
const target = join(tempDir("bench-kit-target-"), "instance");
Expand Down Expand Up @@ -506,6 +610,26 @@ describe("10x bench-kit update", () => {
expect(envelope.data.zones.shared.updated).toBe(1);
});

it("reinstalls runner dependencies after the wholesale .bench-kit swap", async () => {
const target = await initInstance();
const newTemplate = buildTemplateFixture("0.2.0");
mkdirSync(join(newTemplate, ".bench-kit", "runner"), { recursive: true });
writeFileSync(join(newTemplate, ".bench-kit", "runner", "package.json"), "{}\n");
const installedIn: string[] = [];
const { deps } = fakeDeps(newTemplate, {
installRunnerDeps: (runnerDir) => {
installedIn.push(runnerDir);
return Promise.resolve({ ok: true, error: "" });
},
});

const result = await captureStreams(() => runBenchKitUpdate(JSON_CTX, target, {}, deps));

expect(result.exitCode).toBeUndefined();
expect(installedIn).toEqual([join(target, ".bench-kit", "runner")]);
expect(parseEnvelope(result.stdout).data.runnerDeps).toBe("installed");
});

it("is a no-op when the instance is already on the template version", async () => {
const target = await initInstance();
const sameTemplate = buildTemplateFixture("0.1.0");
Expand Down Expand Up @@ -589,6 +713,18 @@ async function runCli(argv: string[]): Promise<CaptureResult> {
});
}

describe("toHttpsUrl", () => {
it("maps scp-style and ssh:// URLs to https, leaves the rest alone", () => {
expect(toHttpsUrl("git@github.com:acme/shop.git")).toBe("https://github.com/acme/shop.git");
expect(toHttpsUrl("ssh://git@github.com/acme/shop.git")).toBe("https://github.com/acme/shop.git");
expect(toHttpsUrl("ssh://git@gitlab.example.com:2222/team/app")).toBe(
"https://gitlab.example.com/team/app",
);
expect(toHttpsUrl("https://github.com/acme/shop.git")).toBeNull();
expect(toHttpsUrl("/local/path/to/repo")).toBeNull();
});
});

describe("experimental gate", () => {
const savedEnv = process.env[EXPERIMENTAL_ENV];

Expand Down
Loading