diff --git a/.github/scripts/koth_score.py b/.github/scripts/koth_score.py new file mode 100644 index 00000000..1b5af781 --- /dev/null +++ b/.github/scripts/koth_score.py @@ -0,0 +1,127 @@ +"""Paired champion-vs-challenger scoring for the koth ladder. + +Runs vouchbench twice per seed — once with the reigning kit, once with the +challenger kit — over seeds derived from the base sha and the utc date, and +applies the dethrone test from docs/vouchbench-seasons.md: + + dethroned iff mean(challenger - champion) >= max(0.007, 1.96 x SE) + +where SE is the standard error of the per-seed paired differences (common +random numbers: both arms see identical generated sessions per seed, so the +difference cancels seed-to-seed variance). + +Seeds are deterministic for a given (base sha, utc day): anyone can recompute +the day's seeds and reproduce the run locally. Same-day overfitting to known +seeds is bounded by the margin band and settled monthly by the season's +commit-reveal scored run, which uses seeds that do not exist until the +cutoff. + +Usage: + python koth_score.py --champion kit.yaml --challenger kit-pr.yaml \ + --base-sha [--date YYYY-MM-DD] [--out report.json] + +Exit code 0 = dethroned (challenger wins), 3 = held (champion stands), +1 = error. The distinct win/hold codes let the workflow branch without +parsing the report. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import statistics +from pathlib import Path + +from vouch import bench + +# 12 paired seeds, not 4: the band is built from the standard error of the +# paired differences, and n=4 makes that SE noisy and its normal-z reading +# optimistic. z=1.96 is the two-sided 95% normal quantile; with a small n +# the paired diffs are not perfectly normal, so the floor below still does +# the real gatekeeping when the SE collapses on a deterministic overfit. +N_SEEDS = 12 +FLOOR = 0.007 +Z = 1.96 +# bench sleeps this long between session ingests to spread created_at +# timestamps; keep it small — it is wall-clock, not simulated time. the +# 3600s that lived here briefly would have out-slept the CI job timeout +# many times over. +SESSION_GAP_SECONDS = 2.0 + + +def day_seeds(base_sha: str, date: str, n: int = N_SEEDS) -> list[int]: + """Derive the day's seed list from the champion sha and the utc date.""" + seeds = [] + for i in range(n): + digest = hashlib.sha256(f"{base_sha}:{date}:{i}".encode()).hexdigest() + seeds.append(int(digest[:12], 16)) + return seeds + + +def score(kit_text: str, seeds: list[int]) -> list[float]: + extra = kit_text if kit_text.strip() else None + return [ + bench.run( + s, extra_config=extra, session_gap_seconds=SESSION_GAP_SECONDS + )["composite"] + for s in seeds + ] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--champion", required=True) + parser.add_argument("--challenger", required=True) + parser.add_argument("--base-sha", required=True) + parser.add_argument("--date", default=None) + parser.add_argument("--out", default=None) + args = parser.parse_args() + + date = args.date or dt.datetime.now(dt.UTC).strftime("%Y-%m-%d") + seeds = day_seeds(args.base_sha, date) + + champion_text = Path(args.champion).read_text(encoding="utf-8") + challenger_text = Path(args.challenger).read_text(encoding="utf-8") + + champion_scores = score(champion_text, seeds) + challenger_scores = score(challenger_text, seeds) + diffs = [ + c - r for c, r in zip(challenger_scores, champion_scores, strict=True) + ] + mean_diff = statistics.mean(diffs) + se = ( + statistics.stdev(diffs) / (len(diffs) ** 0.5) + if len(diffs) > 1 + else 0.0 + ) + band = max(FLOOR, Z * se) + dethroned = mean_diff >= band + + report = { + "date": date, + "base_sha": args.base_sha, + "seeds": seeds, + "champion": { + "scores": champion_scores, + "mean": statistics.mean(champion_scores), + }, + "challenger": { + "scores": challenger_scores, + "mean": statistics.mean(challenger_scores), + }, + "mean_diff": mean_diff, + "se": se, + "band": band, + "dethroned": dethroned, + } + text = json.dumps(report, indent=1) + print(text) + if args.out: + Path(args.out).write_text(text + "\n", encoding="utf-8") + return 0 if dethroned else 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/validate_kit.py b/.github/scripts/validate_kit.py new file mode 100644 index 00000000..7edbab58 --- /dev/null +++ b/.github/scripts/validate_kit.py @@ -0,0 +1,107 @@ +"""Validate a ladder kit file against the strict allowlist. + +The kit is the one file an untrusted PR may change, and it is applied as +extra_config inside a write-token CI context — so the surface must be data +only. In particular, config keys that name executables (compile.llm_cmd, +enrich.llm_cmd) must never be reachable from a kit: a kit that could set a +command string would execute it on the runner. The allowlist below is +therefore closed-world: unknown keys are an error, not a warning. + +Usage: python validate_kit.py +Exits 0 and prints the canonical yaml on success; exits 1 with a reason +on any violation. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +import yaml + +MAX_KIT_BYTES = 4096 + +BACKENDS = {"auto", "fts5", "embedding", "hybrid", "substring"} + +# key path -> (type, validator) +BOUNDS: dict[tuple[str, ...], Any] = { + ("retrieval", "backend"): lambda v: isinstance(v, str) and v in BACKENDS, + ("retrieval", "default_limit"): lambda v: ( + isinstance(v, int) and not isinstance(v, bool) and 1 <= v <= 100 + ), + ("retrieval", "prompt_gate", "enabled"): lambda v: isinstance(v, bool), + ("retrieval", "recency", "enabled"): lambda v: isinstance(v, bool), + ("retrieval", "recency", "half_life_days"): lambda v: ( + isinstance(v, (int, float)) + and not isinstance(v, bool) + and 0.01 <= float(v) <= 3650.0 + ), + ("retrieval", "rerank", "enabled"): lambda v: isinstance(v, bool), + ("retrieval", "rerank", "top_k"): lambda v: ( + isinstance(v, int) and not isinstance(v, bool) and 1 <= v <= 500 + ), + ("retrieval", "pages_first", "enabled"): lambda v: isinstance(v, bool), + ("retrieval", "pages_first", "boost"): lambda v: ( + isinstance(v, (int, float)) + and not isinstance(v, bool) + and 0.1 <= float(v) <= 10.0 + ), +} + +ALLOWED_BRANCHES = {path[:i] for path in BOUNDS for i in range(1, len(path))} + + +def _walk(node: Any, path: tuple[str, ...], errors: list[str]) -> None: + if isinstance(node, dict): + for key, value in node.items(): + if not isinstance(key, str): + errors.append(f"non-string key at {'.'.join(path) or ''}") + continue + child = (*path, key) + if child in BOUNDS: + if not BOUNDS[child](value): + errors.append(f"{'.'.join(child)}: value {value!r} out of bounds") + elif child in ALLOWED_BRANCHES: + _walk(value, child, errors) + else: + errors.append(f"{'.'.join(child)}: key not in allowlist") + else: + errors.append(f"{'.'.join(path) or ''}: expected a mapping") + + +def validate(text: str) -> list[str]: + # an empty kit must NOT validate: over the contents-api, a file larger + # than the api's inline limit comes back with empty content, which would + # otherwise decode to "" and pass vacuously — the PR would then be scored + # as engine defaults rather than its real, oversized contents. require a + # real retrieval mapping so "empty" can never mean "champion defaults". + encoded = text.encode("utf-8") + if not encoded.strip(): + return ["empty kit (nothing to score — did the fetch truncate?)"] + if len(encoded) > MAX_KIT_BYTES: + return [f"kit larger than {MAX_KIT_BYTES} bytes"] + try: + data = yaml.safe_load(text) + except yaml.YAMLError as exc: + return [f"not valid yaml: {exc}"] + if not isinstance(data, dict) or "retrieval" not in data: + return ["kit must be a mapping with a top-level 'retrieval' section"] + errors: list[str] = [] + _walk(data, (), errors) + return errors + + +def main() -> int: + text = Path(sys.argv[1]).read_text(encoding="utf-8") + errors = validate(text) + if errors: + for err in errors: + print(f"kit rejected: {err}", file=sys.stderr) + return 1 + print(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/koth-gate.yml b/.github/workflows/koth-gate.yml new file mode 100644 index 00000000..b6e3b243 --- /dev/null +++ b/.github/workflows/koth-gate.yml @@ -0,0 +1,181 @@ +# koth ladder gate - scores kit-only PRs against the reigning kit and +# enables auto-merge on a dethrone. the ditto mining loop, rebuilt on +# github primitives: the bench is the validator, branch protection is the +# chain, auto-merge is the emission. +# +# security model (do not weaken): +# - pull_request_target: this definition and every line of executed code +# come from the BASE branch. PR code is never checked out or executed. +# - the only thing read from the PR is competition/kits/current/kit.yaml, +# fetched over the api as data and schema-validated (closed allowlist) +# before it is passed to the bench as extra_config. +# - a PR qualifies for the ladder only if the kit file is the ONLY file +# it touches. anything else is a normal PR: the gate passes without +# scoring and a human reviews as usual. +# - untrusted strings (branch names, repo names, titles) reach scripts +# via env, never by interpolation into run bodies. +name: koth-gate + +on: + pull_request_target: + types: [opened, synchronize, reopened, ready_for_review] + +concurrency: + group: koth-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + gate: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write # auto-merge (squash) needs it + pull-requests: write # enable auto-merge + scorecard comment + steps: + - name: checkout base branch (trusted code only) + uses: actions/checkout@v4 + # no ref: pull_request_target checks out the base branch tip + + - name: classify the PR + id: classify + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \ + --paginate --jq '.[].filename' > /tmp/changed.txt + count=$(wc -l < /tmp/changed.txt) + if [ "$count" = "1" ] && \ + grep -qx 'competition/kits/current/kit.yaml' /tmp/changed.txt; then + echo "mode=ladder" >> "$GITHUB_OUTPUT" + else + echo "mode=normal" >> "$GITHUB_OUTPUT" + fi + + - name: pass through (not a ladder PR) + if: steps.classify.outputs.mode == 'normal' + run: echo "not a kit-only PR - koth gate does not apply, humans review." + + - name: fetch challenger kit from the PR (data only) + if: steps.classify.outputs.mode == 'ladder' + env: + GH_TOKEN: ${{ github.token }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + KIT_PATH: competition/kits/current/kit.yaml + run: | + # the contents api inlines base64 only for files under its size + # limit (~1MB); a larger file returns empty content and a download + # url instead. read size + content together and refuse anything + # that is not a small inlined blob, so an oversized kit can never + # decode to "" and be scored as champion defaults. + gh api "repos/${HEAD_REPO}/contents/${KIT_PATH}?ref=${HEAD_SHA}" \ + > /tmp/kit-meta.json + size=$(jq -r '.size // 0' /tmp/kit-meta.json) + encoding=$(jq -r '.encoding // ""' /tmp/kit-meta.json) + if [ "$encoding" != "base64" ]; then + echo "kit not returned as an inlined base64 blob (encoding=$encoding, size=$size)" >&2 + exit 1 + fi + jq -r '.content' /tmp/kit-meta.json | base64 -d > /tmp/kit-challenger.yaml + if [ ! -s /tmp/kit-challenger.yaml ]; then + echo "fetched kit is empty" >&2 + exit 1 + fi + + - name: set up python + if: steps.classify.outputs.mode == 'ladder' + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: install vouch (base branch code) + if: steps.classify.outputs.mode == 'ladder' + run: python -m pip install -e . + + - name: validate challenger kit against the allowlist + if: steps.classify.outputs.mode == 'ladder' + run: python .github/scripts/validate_kit.py /tmp/kit-challenger.yaml + + - name: paired scoring - challenger vs reigning kit + if: steps.classify.outputs.mode == 'ladder' + id: score + env: + BASE_SHA: ${{ github.sha }} + run: | + set +e + python .github/scripts/koth_score.py \ + --champion competition/kits/current/kit.yaml \ + --challenger /tmp/kit-challenger.yaml \ + --base-sha "$BASE_SHA" \ + --out /tmp/koth-report.json + code=$? + set -e + if [ "$code" = "0" ]; then + echo "verdict=dethroned" >> "$GITHUB_OUTPUT" + elif [ "$code" = "3" ]; then + echo "verdict=held" >> "$GITHUB_OUTPUT" + else + exit "$code" + fi + + - name: post the scorecard + if: steps.classify.outputs.mode == 'ladder' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + VERDICT: ${{ steps.score.outputs.verdict }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + LADDER_BASE: ${{ vars.KOTH_LADDER_BASE }} + run: | + if [ "$BASE_REF" = "$LADDER_BASE" ] && [ -n "$LADDER_BASE" ]; then + gate_note="auto-merge on dethrone (provisional ladder branch)" + else + gate_note="scored only - auto-merge is disabled for base '${BASE_REF}'; a maintainer promotes proven champions to shipped defaults by hand" + fi + { + echo "koth ladder - ${VERDICT}" + echo + echo '```json' + cat /tmp/koth-report.json + echo '```' + echo + echo "${gate_note}." + echo + echo "seeds derive from base sha + utc date; rerun locally with" + echo '`python .github/scripts/koth_score.py --date ` to reproduce.' + echo "the daily result is provisional and overfittable - payout rank is" + echo "settled monthly on sealed commit-reveal seeds (docs/vouchbench-seasons.md)." + } > /tmp/comment.md + gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --body-file /tmp/comment.md + + # auto-merge fires ONLY on a dedicated, non-shipped ladder branch + # (repo variable KOTH_LADDER_BASE). the trunk is never auto-written: + # a kit-only PR against main is scored and commented, then a human + # merges. this keeps the review gate load-bearing for everything that + # ships - a beatable benchmark must never be the sole writer to code + # users install. when KOTH_LADDER_BASE is unset, nothing auto-merges. + - name: enable auto-merge on dethrone (ladder branch only) + if: >- + steps.classify.outputs.mode == 'ladder' && + steps.score.outputs.verdict == 'dethroned' && + vars.KOTH_LADDER_BASE != '' && + github.event.pull_request.base.ref == vars.KOTH_LADDER_BASE + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + gh pr merge "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --auto --squash + + - name: hold the throne (challenger did not clear the band) + if: >- + steps.classify.outputs.mode == 'ladder' && + steps.score.outputs.verdict == 'held' + run: | + echo "champion holds - challenger did not clear max(0.007, 1.96 x paired SE)." + exit 1 diff --git a/competition/LEADERBOARD.md b/competition/LEADERBOARD.md new file mode 100644 index 00000000..cc0204e0 --- /dev/null +++ b/competition/LEADERBOARD.md @@ -0,0 +1,17 @@ +# koth ladder — throne history + +every row is a dethrone: a kit-only PR that beat the reigning kit by the +margin band and auto-merged on the ladder branch. the merged PR is the +authoritative record; this file is the human-readable ledger, appended by +the maintainer when the monthly season closes and payouts are computed. + +the daily throne is provisional (public seeds — see docs/koth-ladder.md); +payout rank is settled by the monthly sealed commit-reveal run. + +| # | champion | PR | dethroned on | scored mean | margin over prior | +|---|----------|----|--------------|-------------|-------------------| +| 0 | baseline kit (repo defaults) | — | 2026-07-27 | 0.57 ± 0.04 (seeds 1–6) | — | + +payouts follow the season shares in docs/vouchbench-seasons.md +(65/14/10/7/4). days-on-throne accrue between dethrones; the monthly +commit-reveal scored run settles the season standings. diff --git a/competition/kits/current/kit.yaml b/competition/kits/current/kit.yaml new file mode 100644 index 00000000..59fb05dc --- /dev/null +++ b/competition/kits/current/kit.yaml @@ -0,0 +1,28 @@ +# the reigning retrieval kit - vouch's answer to ditto's starter-kit crate. +# ascii only in this file: it crosses tools and locales. +# +# this file is the ONLY thing a ladder PR may change, and it is the WHOLE +# retrieval config of the bench kb: vouchbench writes `review:` plus this +# file verbatim as the throwaway kb's config.yaml. omitting a key does not +# inherit anything - it means the engine's code default for that knob. +# the reigning kit spells out every knob it relies on, and the current +# values reproduce the repo baseline exactly. +# +# dethrone these values by max(0.007, 1.96 * paired SE) over the day's +# seeds and your PR auto-merges on the ladder branch. allowed keys and +# bounds are enforced by .github/scripts/validate_kit.py - anything else +# fails before scoring. +retrieval: + backend: auto + default_limit: 10 + recency: + enabled: true + half_life_days: 90.0 + prompt_gate: + enabled: true + rerank: + enabled: false + top_k: 50 + pages_first: + enabled: false + boost: 1.25 diff --git a/docs/koth-ladder.md b/docs/koth-ladder.md new file mode 100644 index 00000000..9a7f5bc8 --- /dev/null +++ b/docs/koth-ladder.md @@ -0,0 +1,141 @@ +# the koth ladder — mine on vouch, through a pull request + +ditto onboards contributors by letting miners tweak a retrieval starter +kit, scoring the result, and paying whoever holds the throne. the vouch +ladder is the same loop rebuilt on github primitives — no wallet, no +subnet, no tarball uploads. your mining rig is a fork and a text editor. + +## the loop + +1. **fork the repo** and edit exactly one file: + [`competition/kits/current/kit.yaml`](../competition/kits/current/kit.yaml) + — the reigning retrieval kit. it is the whole retrieval config of the + benchmark kb: backend, result limit, recency half-life, prompt-gate, + rerank, and pages-first knobs, all bounded by a strict allowlist + (`.github/scripts/validate_kit.py`). it is pure data — a kit carries + no code. +2. **open a PR** that touches only that file, **against the ladder + branch** (see setup below). the `koth-gate` workflow classifies it as + a ladder entry, validates your kit, and runs vouchbench across the + day's seeds — reigning kit vs yours, paired on identical generated + sessions (common random numbers). +3. **dethrone** if your mean improvement clears the band + `max(0.007, 1.96 x paired SE)`. the gate posts the full scorecard as a + PR comment either way. +4. **auto-merge on the ladder branch.** on a dethrone against the ladder + branch the workflow enables github auto-merge; once required checks + are green your PR lands with no human in the loop. your kit is now the + champion every later challenger must beat, and the merge commit is + your permanent, public receipt. +5. **earn.** days-on-throne accrue until someone dethrones you. the daily + ladder is provisional (see "why the daily throne is only practice"); + seasons close monthly with a sealed commit-reveal scored run + (docs/vouchbench-seasons.md), and that is what settles payouts + (shares 65/14/10/7/4). the ledger lives in `competition/LEADERBOARD.md`. + +## reproduce any score locally + +```bash +pip install -e . +python .github/scripts/koth_score.py \ + --champion competition/kits/current/kit.yaml \ + --challenger my-kit.yaml \ + --base-sha "$(git rev-parse origin/main)" \ + --date "$(date -u +%F)" +``` + +seeds derive from the champion sha and the utc date, so every scorecard +in ci can be recomputed by anyone — pass the same `--date` the run used +(it is printed in the scorecard) to reproduce a past result exactly. no +judge model, no hidden eval set, no api key. + +## why the daily throne is only practice + +the daily seeds are a deterministic function of public inputs (the base +sha and the utc date), and the whole scorer is offline-reproducible — by +design, so anyone can audit a score. the cost is that a contributor can +grid-search the knob space against today's exact seeds and submit only a +kit that already wins. the daily ladder therefore behaves like a public +practice leaderboard: fast, transparent, and overfittable. + +that is fine, because **money is never tied to the daily throne.** payout +rank is decided by the monthly season's scored run, whose seeds come from +a commit-reveal drand round at the cutoff and do not exist until entries +are frozen (docs/vouchbench-seasons.md). the daily loop is where you +iterate and get instant feedback; the sealed monthly run is where a kit +has to generalise to seeds nobody could tune against. + +the band (`max(0.007, 1.96 x paired SE)`) still matters daily: it stops a +kit that merely ties the champion from taking the throne, and the 0.007 +floor holds the line when a deterministic overfit collapses the SE toward +zero. + +## why only a data file auto-merges, and only on the ladder branch + +the review gate is vouch's load-bearing invariant, and it stays load +bearing here in two ways: + +- **code never auto-merges.** the ladder surface is a schema-validated + config fragment; the worst a malicious entry can do is score badly. a + merged ladder PR provably contains exactly one bounded data file. +- **the trunk is never auto-written.** auto-merge fires only for PRs + whose base is the dedicated ladder branch (repo variable + `KOTH_LADDER_BASE`). a kit-only PR opened against `main` is scored and + commented, then a human merges — so a beatable benchmark is never the + sole gate to code that users install. promoting the reigning kit into + the shipped starter-config defaults is a periodic, human-reviewed PR. + +engine improvements — new retrieval stages, new signals, bench +extensions — are welcome as normal PRs with human review, and the +maintainer may then expose a new knob in the kit allowlist so the ladder +can tune it. that is the division of labour: humans review capabilities, +the ladder optimises coefficients. miners on ditto never write to ditto's +core either — they submit kits that a validator scores. same boundary, +expressed as a path allowlist and a branch boundary instead of a tarball +contract. + +## anti-cheat, mapped from ditto + +| threat | ditto's answer | the ladder's answer | +|---|---|---| +| overfitting the eval | hidden benchmark versions | conceded for the daily ladder (public seeds) — it is explicitly practice; the monthly settlement uses commit-reveal seeds that do not exist until cutoff, so the paying rank cannot be pre-fit | +| lookup tables | ast minhash scan | impossible by construction: kits carry no code | +| self-reported scores | tee-locked validator | scoring runs from base-branch code under `pull_request_target`; a PR cannot alter the grader, the workflow, or the seeds it is scored with | +| copy the champion + epsilon | first-seen protection | the band forces a real margin over the reigning kit, and a tie loses to the throne | +| oversized / empty kit | — | the fetch refuses any file not returned as a small inlined blob, and the validator rejects an empty or non-`retrieval` kit, so "empty" can never be scored as champion defaults | +| stale wins racing a new champion | — | branch protection in strict mode on the ladder branch: a dethrone that lands invalidates every open challenger's check until it rebases and re-scores against the new champion | +| benchmark as the only gate to shipped code | tee isolation | auto-merge is confined to the ladder branch; the trunk keeps human review, and champions reach shipped defaults only through a human PR | + +## one-time repo setup (maintainer) + +```bash +# 1. create the dedicated ladder branch off the trunk and seed the champion kit +git switch -c ladder origin/main && git push -u origin ladder + +# 2. tell the gate which base is the auto-merge ladder branch +gh variable set KOTH_LADDER_BASE --body ladder + +# 3. allow the auto-merge button repo-wide +gh repo edit --enable-auto-merge + +# 4. protect the ladder branch: gate is a required check, strict mode. +# (json body via --input so booleans stay booleans — a bare -f sends +# the string "true" and the api 422s.) +cat > /tmp/ladder-protection.json <<'JSON' +{ + "required_status_checks": { "strict": true, "contexts": ["gate"] }, + "enforce_admins": false, + "required_pull_request_reviews": null, + "restrictions": null +} +JSON +gh api -X PUT repos/OWNER/REPO/branches/ladder/protection \ + --input /tmp/ladder-protection.json +``` + +strict mode is what closes the race window: after any merge to the ladder +branch, every open ladder PR must update its branch, which re-triggers +scoring against the new champion. `koth-gate` runs on every PR and passes +trivially for non-kit PRs, so requiring it never blocks normal +development. leave `main` protected with your usual human-review rules — +the gate never auto-merges there. diff --git a/docs/vouchbench-seasons.md b/docs/vouchbench-seasons.md index d74f352f..7e61164a 100644 --- a/docs/vouchbench-seasons.md +++ b/docs/vouchbench-seasons.md @@ -29,7 +29,7 @@ machine; that reproducibility is the whole trust model. around `proposals.approve()`, plaintext storage, no baked model deps). Near-identical entries: the earlier-opened PR wins (first-seen). 6. **Pay and merge.** Every entry that beats `main`'s composite by the - margin band — `max(0.007, 1.64 x SE_paired)` over the scored seeds — + margin band — `max(0.007, 1.96 x SE_paired)` over the scored seeds — earns its rank share of the pool (65 / 14 / 10 / 7 / 4 while the field is small). The winner merges and becomes the champion the next season must dethrone. diff --git a/tests/test_koth_kit.py b/tests/test_koth_kit.py new file mode 100644 index 00000000..e809df00 --- /dev/null +++ b/tests/test_koth_kit.py @@ -0,0 +1,66 @@ +"""The koth kit validator — the closed-world allowlist that lets a kit-only +PR auto-merge without a human. Every test here guards a way an untrusted kit +could smuggle something past the gate.""" + +import importlib.util +from pathlib import Path + +_SPEC = importlib.util.spec_from_file_location( + "validate_kit", + Path(__file__).resolve().parents[1] / ".github" / "scripts" / "validate_kit.py", +) +assert _SPEC and _SPEC.loader +validate_kit = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(validate_kit) +validate = validate_kit.validate + +CHAMPION = ( + Path(__file__).resolve().parents[1] + / "competition" + / "kits" + / "current" + / "kit.yaml" +).read_text(encoding="utf-8") + + +def test_champion_kit_validates() -> None: + assert validate(CHAMPION) == [] + + +def test_empty_kit_is_rejected() -> None: + # the >1MB contents-api bypass: an oversized file returns empty content, + # which must NOT validate as "champion defaults". + assert validate("") != [] + assert validate(" \n \n") != [] + + +def test_kit_without_retrieval_section_is_rejected() -> None: + assert validate("review:\n auto_approve_on_receipt: true\n") != [] + + +def test_command_key_is_rejected() -> None: + # config can name executables (compile.llm_cmd); a kit must never reach one. + errors = validate("retrieval:\n backend: auto\ncompile:\n llm_cmd: 'sh -c evil'\n") + assert any("not in allowlist" in e for e in errors) + + +def test_unknown_retrieval_key_is_rejected() -> None: + errors = validate("retrieval:\n backend: auto\n secret_knob: 1\n") + assert any("not in allowlist" in e for e in errors) + + +def test_out_of_bounds_values_are_rejected() -> None: + assert validate("retrieval:\n backend: nonsense\n") != [] + assert validate("retrieval:\n recency:\n half_life_days: 99999\n") != [] + assert validate("retrieval:\n rerank:\n top_k: 0\n") != [] + assert validate("retrieval:\n pages_first:\n boost: -1\n") != [] + + +def test_bool_is_not_accepted_as_a_number() -> None: + # yaml true is an int subclass in python; a numeric knob must refuse it. + assert validate("retrieval:\n recency:\n half_life_days: true\n") != [] + + +def test_oversized_kit_is_rejected() -> None: + big = "retrieval:\n backend: auto\n" + ("# pad\n" * 2000) + assert any("larger than" in e for e in validate(big))