From f3174ed42242367389796b5a422924357076b667 Mon Sep 17 00:00:00 2001 From: psmyrdek Date: Sun, 16 Aug 2026 22:50:41 +0200 Subject: [PATCH] =?UTF-8?q?feat(bench-kit):=20remove=20TENX=5FCLI=5FEXPERI?= =?UTF-8?q?MENTAL=20gating=20=E2=80=94=20init=20and=20update=20generally?= =?UTF-8?q?=20available?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bench-kit command is feature-complete (zone-aware update, agent-tool profiles, safe init defaults shipped in v1.17.0), so the experimental opt-in has served its purpose. The command now registers unconditionally, shows up in --help, and is documented in the README; the gating module and its error envelope (experimental_locked) are removed. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 7 +++++ README.md | 35 ++++++++++++++++++++++ src/commands/bench-kit.ts | 11 +------ src/lib/experimental.ts | 39 ------------------------- tests/bench-kit-command.test.ts | 51 ++------------------------------- 5 files changed, 45 insertions(+), 98 deletions(-) delete mode 100644 src/lib/experimental.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f8fd9f..d3108d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **`10x bench-kit` is generally available.** The `TENX_CLI_EXPERIMENTAL` + opt-in is gone: `init` and `update` are registered unconditionally, appear + in `--help`, and are documented in the README. The `experimental_locked` + error envelope no longer exists (the gating module has been removed). + ### Added - **`10x sync` — bulk download & update with change visibility.** One command to diff --git a/README.md b/README.md index acf8922..c1d9afa 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ Once installed, just tell your agent to **set up 10x-cli** and it will pick up t | `10x sync` | Bulk-download / refresh lessons and report what changed upstream | | `10x doctor` | Diagnose auth, API connectivity, and local config | | `10x bench` | Live top-10 AI model leaderboard from [10xbench.ai](https://10xbench.ai) — no login needed | +| `10x bench-kit ` | Create (`init`) and update (`update`) a company benchmark instance from the [10x-bench-kit](https://github.com/przeprogramowani/10x-bench-kit) template | ### `10x get` Flags @@ -180,6 +181,40 @@ color-coded score bars in your terminal. Public data, works without logging in. The data refreshes whenever new benchmark results are published on 10xbench.ai. Colors honor `NO_COLOR` and are disabled automatically when output is piped. +### `10x bench-kit` + +Creates and maintains a **company benchmark instance** from the +[10x-bench-kit](https://github.com/przeprogramowani/10x-bench-kit) template — +a self-hosted benchmark that scores AI agents on tasks embedded in your own +repositories. The CLI is a thin installer/updater; tasks, assertions, and +scoring live in the template and the instance. + +| Action | Description | +|--------|-------------| +| `init [dir]` | Materialize a fresh instance from the template (no git history, fresh `git init`), register the detected base repo, install runner dependencies | +| `update [dir]` | Upgrade the instance to a newer template: runtime zone replaced wholesale, skills proposed as a reviewable diff, company content untouched | + +| Flag | Description | +|------|-------------| +| `--template-version ` | Template tag to install (default: latest) | +| `--tool ` | Agent tool for skill placement (`claude-code`, `cursor`, `copilot`, `codex`, `windsurf`, `gemini`, `generic`) | +| `--yes` | Run non-interactively, accepting defaults | + +```bash +# Create an instance next to your product repo (run inside it to auto-register) +10x bench-kit init my-benchmark + +# Pin the template version +10x bench-kit init my-benchmark --template-version v0.8.0 + +# Upgrade an existing instance to the latest template +10x bench-kit update +``` + +Re-running `init` on an existing instance is a **repair**: missing template +files are restored, company content is never touched. After `update`, run the +instance's `bench validate` to confirm the content still matches the new kit. + ### Global Flags - `--json` — Machine-readable JSON output (auto-detected when piped) diff --git a/src/commands/bench-kit.ts b/src/commands/bench-kit.ts index ede2dee..210aed0 100644 --- a/src/commands/bench-kit.ts +++ b/src/commands/bench-kit.ts @@ -30,7 +30,6 @@ import { import { tmpdir } from "node:os"; import { basename, join, resolve, sep } from "node:path"; import type { CAC } from "cac"; -import { experimentalEnabled, requireExperimental } from "../lib/experimental"; import { ExitCodes, type GlobalFlags, @@ -121,15 +120,8 @@ export interface BenchKitDeps { // bench-kit follows the `auth` precedent: one command dispatching on an // action argument. export function registerBenchKitCommand(cli: CAC): void { - // Experimental commands are all-or-nothing: without the opt-in the - // command is not registered at all — absent from help and behaving like - // any unknown command — instead of showing up half-locked. - if (!experimentalEnabled()) return; cli - .command( - "bench-kit [dir]", - "Manage a benchmark instance (actions: init, update; experimental)", - ) + .command("bench-kit [dir]", "Manage a benchmark instance (actions: init, update)") .option("--template-version ", "Template tag to install (default: latest)") .option("--tool ", `Agent tool for skill placement (${Object.keys(PROFILES).join(", ")})`) .option("--yes", "Run non-interactively, accepting defaults") @@ -137,7 +129,6 @@ export function registerBenchKitCommand(cli: CAC): void { .example("10x bench-kit init my-benchmark --template-version v0.1.0") .example("10x bench-kit update") .action(async (action: string, dir: string | undefined, options: BenchKitFlags) => { - requireExperimental(`bench-kit ${action}`, options); const ctx = resolveContext(options); if (action === "init") { await runBenchKitInit(ctx, dir, options); diff --git a/src/lib/experimental.ts b/src/lib/experimental.ts deleted file mode 100644 index 4aea303..0000000 --- a/src/lib/experimental.ts +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Gating for experimental commands. - * - * Commands that should ship to master before they are ready for students - * register normally (so `10x ` never dies silently and help stays - * discoverable) but call `requireExperimental` first: without the opt-in - * env var the action exits with a stable, parseable error envelope. - * - * Opt-in: TENX_CLI_EXPERIMENTAL=1 (or "true"). - */ - -import { ExitCodes, type GlobalFlags, outputError, resolveContext } from "./output"; - -export const EXPERIMENTAL_ENV = "TENX_CLI_EXPERIMENTAL"; - -export function experimentalEnabled(env: NodeJS.ProcessEnv = process.env): boolean { - const value = env[EXPERIMENTAL_ENV]; - return value === "1" || value === "true"; -} - -/** - * Exits with `experimental_locked` unless the experimental opt-in is set. - * Call at the top of an action callback, before any side effects. - */ -export function requireExperimental( - command: string, - flags: GlobalFlags, - env: NodeJS.ProcessEnv = process.env, -): void { - if (experimentalEnabled(env)) return; - const ctx = resolveContext(flags); - outputError( - ctx, - "experimental_locked", - `'10x ${command}' is experimental and currently locked.`, - ExitCodes.FORBIDDEN, - `Set ${EXPERIMENTAL_ENV}=1 and run '10x ${command}' again.`, - ); -} diff --git a/tests/bench-kit-command.test.ts b/tests/bench-kit-command.test.ts index 347fe87..ffa24a4 100644 --- a/tests/bench-kit-command.test.ts +++ b/tests/bench-kit-command.test.ts @@ -18,7 +18,6 @@ import { runBenchKitUpdate, toHttpsUrl, } from "../src/commands/bench-kit"; -import { EXPERIMENTAL_ENV, experimentalEnabled } from "../src/lib/experimental"; import type { OutputContext } from "../src/lib/output"; interface CaptureResult { @@ -725,58 +724,12 @@ describe("toHttpsUrl", () => { }); }); -describe("experimental gate", () => { - const savedEnv = process.env[EXPERIMENTAL_ENV]; - - afterEach(() => { - if (savedEnv === undefined) { - delete process.env[EXPERIMENTAL_ENV]; - } else { - process.env[EXPERIMENTAL_ENV] = savedEnv; - } - }); - - it("is off by default and accepts 1 / true", () => { - expect(experimentalEnabled({})).toBe(false); - expect(experimentalEnabled({ [EXPERIMENTAL_ENV]: "0" })).toBe(false); - expect(experimentalEnabled({ [EXPERIMENTAL_ENV]: "1" })).toBe(true); - expect(experimentalEnabled({ [EXPERIMENTAL_ENV]: "true" })).toBe(true); - }); - - it("keeps bench-kit fully hidden without the opt-in", async () => { - delete process.env[EXPERIMENTAL_ENV]; - const cli = cac("10x"); - registerBenchKitCommand(cli); - expect(cli.commands.map((c) => c.name)).not.toContain("bench-kit"); - - // Invoking it behaves like any unknown command: no output, no error. - const result = await runCli(["bench-kit", "init", "some-dir", "--json"]); - expect(result.exitCode).toBeUndefined(); - expect(result.stdout).toBe(""); - }); - - it("registers bench-kit when the opt-in is set", () => { - process.env[EXPERIMENTAL_ENV] = "1"; +describe("10x bench-kit dispatch", () => { + it("registers bench-kit unconditionally", () => { const cli = cac("10x"); registerBenchKitCommand(cli); expect(cli.commands.map((c) => c.name)).toContain("bench-kit"); }); -}); - -describe("10x bench-kit dispatch", () => { - const savedEnv = process.env[EXPERIMENTAL_ENV]; - - beforeEach(() => { - process.env[EXPERIMENTAL_ENV] = "1"; - }); - - afterEach(() => { - if (savedEnv === undefined) { - delete process.env[EXPERIMENTAL_ENV]; - } else { - process.env[EXPERIMENTAL_ENV] = savedEnv; - } - }); it("routes 'update' to the real implementation", async () => { // A temp dir that is not an instance — proves dispatch reaches update.