From aed409372c17af568ab2c2d395dbcc73e273c925 Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Mon, 27 Jul 2026 23:49:32 +0900 Subject: [PATCH 01/13] feat(competition): koth ladder - auto-merge kit PRs scored by vouchbench the contributor-onboarding half of the beat-ditto plan: ditto's mining loop (tweak a retrieval kit, get scored, dethrone the champion, earn while on top) rebuilt on github primitives. stacks on the bench from the session-answer-mode branch. the ladder surface is one schema-validated data file, competition/kits/current/kit.yaml, which is the whole retrieval config of the throwaway bench kb. a PR touching only that file is scored by vouchbench, paired per seed against the reigning kit; a dethrone over max(0.007, 1.96 x paired se) enables github auto-merge. the review gate stays load-bearing. code never auto-merges - the ladder surface is data, closed-world allowlisted (config can name executables, so unknown keys are an error). and the trunk is never auto-written - auto-merge fires only for PRs whose base is a dedicated non-shipped ladder branch (repo var koth_ladder_base); a kit-only PR against main is scored then merged by a human. promoting a champion into shipped defaults is a periodic human PR. the daily ladder is provisional by design: public seeds (base sha + utc date) make it offline-reproducible and thus overfittable, so payout rank is settled by the monthly sealed commit-reveal run, not the daily throne. hardened against an adversarial review: the empty/oversized-kit contents- api bypass, a stray 1-hour real sleep that out-slept the ci job, the auto-merge-to-trunk gate bypass (now ladder-branch-only), band calibration (12 paired seeds, z=1.96), the draft-pr trigger gap, and a branch-protection command that 422'd on a stringified boolean. --- .github/scripts/koth_score.py | 127 +++++++++++++++++++++ .github/scripts/validate_kit.py | 107 ++++++++++++++++++ .github/workflows/koth-gate.yml | 181 ++++++++++++++++++++++++++++++ competition/LEADERBOARD.md | 17 +++ competition/kits/current/kit.yaml | 27 +++++ docs/koth-ladder.md | 141 +++++++++++++++++++++++ docs/vouchbench-seasons.md | 2 +- tests/test_koth_kit.py | 66 +++++++++++ 8 files changed, 667 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/koth_score.py create mode 100644 .github/scripts/validate_kit.py create mode 100644 .github/workflows/koth-gate.yml create mode 100644 competition/LEADERBOARD.md create mode 100644 competition/kits/current/kit.yaml create mode 100644 docs/koth-ladder.md create mode 100644 tests/test_koth_kit.py diff --git a/.github/scripts/koth_score.py b/.github/scripts/koth_score.py new file mode 100644 index 00000000..1b5af781 --- /dev/null +++ b/.github/scripts/koth_score.py @@ -0,0 +1,127 @@ +"""Paired champion-vs-challenger scoring for the koth ladder. + +Runs vouchbench twice per seed — once with the reigning kit, once with the +challenger kit — over seeds derived from the base sha and the utc date, and +applies the dethrone test from docs/vouchbench-seasons.md: + + dethroned iff mean(challenger - champion) >= max(0.007, 1.96 x SE) + +where SE is the standard error of the per-seed paired differences (common +random numbers: both arms see identical generated sessions per seed, so the +difference cancels seed-to-seed variance). + +Seeds are deterministic for a given (base sha, utc day): anyone can recompute +the day's seeds and reproduce the run locally. Same-day overfitting to known +seeds is bounded by the margin band and settled monthly by the season's +commit-reveal scored run, which uses seeds that do not exist until the +cutoff. + +Usage: + python koth_score.py --champion kit.yaml --challenger kit-pr.yaml \ + --base-sha [--date YYYY-MM-DD] [--out report.json] + +Exit code 0 = dethroned (challenger wins), 3 = held (champion stands), +1 = error. The distinct win/hold codes let the workflow branch without +parsing the report. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import statistics +from pathlib import Path + +from vouch import bench + +# 12 paired seeds, not 4: the band is built from the standard error of the +# paired differences, and n=4 makes that SE noisy and its normal-z reading +# optimistic. z=1.96 is the two-sided 95% normal quantile; with a small n +# the paired diffs are not perfectly normal, so the floor below still does +# the real gatekeeping when the SE collapses on a deterministic overfit. +N_SEEDS = 12 +FLOOR = 0.007 +Z = 1.96 +# bench sleeps this long between session ingests to spread created_at +# timestamps; keep it small — it is wall-clock, not simulated time. the +# 3600s that lived here briefly would have out-slept the CI job timeout +# many times over. +SESSION_GAP_SECONDS = 2.0 + + +def day_seeds(base_sha: str, date: str, n: int = N_SEEDS) -> list[int]: + """Derive the day's seed list from the champion sha and the utc date.""" + seeds = [] + for i in range(n): + digest = hashlib.sha256(f"{base_sha}:{date}:{i}".encode()).hexdigest() + seeds.append(int(digest[:12], 16)) + return seeds + + +def score(kit_text: str, seeds: list[int]) -> list[float]: + extra = kit_text if kit_text.strip() else None + return [ + bench.run( + s, extra_config=extra, session_gap_seconds=SESSION_GAP_SECONDS + )["composite"] + for s in seeds + ] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--champion", required=True) + parser.add_argument("--challenger", required=True) + parser.add_argument("--base-sha", required=True) + parser.add_argument("--date", default=None) + parser.add_argument("--out", default=None) + args = parser.parse_args() + + date = args.date or dt.datetime.now(dt.UTC).strftime("%Y-%m-%d") + seeds = day_seeds(args.base_sha, date) + + champion_text = Path(args.champion).read_text(encoding="utf-8") + challenger_text = Path(args.challenger).read_text(encoding="utf-8") + + champion_scores = score(champion_text, seeds) + challenger_scores = score(challenger_text, seeds) + diffs = [ + c - r for c, r in zip(challenger_scores, champion_scores, strict=True) + ] + mean_diff = statistics.mean(diffs) + se = ( + statistics.stdev(diffs) / (len(diffs) ** 0.5) + if len(diffs) > 1 + else 0.0 + ) + band = max(FLOOR, Z * se) + dethroned = mean_diff >= band + + report = { + "date": date, + "base_sha": args.base_sha, + "seeds": seeds, + "champion": { + "scores": champion_scores, + "mean": statistics.mean(champion_scores), + }, + "challenger": { + "scores": challenger_scores, + "mean": statistics.mean(challenger_scores), + }, + "mean_diff": mean_diff, + "se": se, + "band": band, + "dethroned": dethroned, + } + text = json.dumps(report, indent=1) + print(text) + if args.out: + Path(args.out).write_text(text + "\n", encoding="utf-8") + return 0 if dethroned else 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/validate_kit.py b/.github/scripts/validate_kit.py new file mode 100644 index 00000000..7edbab58 --- /dev/null +++ b/.github/scripts/validate_kit.py @@ -0,0 +1,107 @@ +"""Validate a ladder kit file against the strict allowlist. + +The kit is the one file an untrusted PR may change, and it is applied as +extra_config inside a write-token CI context — so the surface must be data +only. In particular, config keys that name executables (compile.llm_cmd, +enrich.llm_cmd) must never be reachable from a kit: a kit that could set a +command string would execute it on the runner. The allowlist below is +therefore closed-world: unknown keys are an error, not a warning. + +Usage: python validate_kit.py +Exits 0 and prints the canonical yaml on success; exits 1 with a reason +on any violation. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +import yaml + +MAX_KIT_BYTES = 4096 + +BACKENDS = {"auto", "fts5", "embedding", "hybrid", "substring"} + +# key path -> (type, validator) +BOUNDS: dict[tuple[str, ...], Any] = { + ("retrieval", "backend"): lambda v: isinstance(v, str) and v in BACKENDS, + ("retrieval", "default_limit"): lambda v: ( + isinstance(v, int) and not isinstance(v, bool) and 1 <= v <= 100 + ), + ("retrieval", "prompt_gate", "enabled"): lambda v: isinstance(v, bool), + ("retrieval", "recency", "enabled"): lambda v: isinstance(v, bool), + ("retrieval", "recency", "half_life_days"): lambda v: ( + isinstance(v, (int, float)) + and not isinstance(v, bool) + and 0.01 <= float(v) <= 3650.0 + ), + ("retrieval", "rerank", "enabled"): lambda v: isinstance(v, bool), + ("retrieval", "rerank", "top_k"): lambda v: ( + isinstance(v, int) and not isinstance(v, bool) and 1 <= v <= 500 + ), + ("retrieval", "pages_first", "enabled"): lambda v: isinstance(v, bool), + ("retrieval", "pages_first", "boost"): lambda v: ( + isinstance(v, (int, float)) + and not isinstance(v, bool) + and 0.1 <= float(v) <= 10.0 + ), +} + +ALLOWED_BRANCHES = {path[:i] for path in BOUNDS for i in range(1, len(path))} + + +def _walk(node: Any, path: tuple[str, ...], errors: list[str]) -> None: + if isinstance(node, dict): + for key, value in node.items(): + if not isinstance(key, str): + errors.append(f"non-string key at {'.'.join(path) or ''}") + continue + child = (*path, key) + if child in BOUNDS: + if not BOUNDS[child](value): + errors.append(f"{'.'.join(child)}: value {value!r} out of bounds") + elif child in ALLOWED_BRANCHES: + _walk(value, child, errors) + else: + errors.append(f"{'.'.join(child)}: key not in allowlist") + else: + errors.append(f"{'.'.join(path) or ''}: expected a mapping") + + +def validate(text: str) -> list[str]: + # an empty kit must NOT validate: over the contents-api, a file larger + # than the api's inline limit comes back with empty content, which would + # otherwise decode to "" and pass vacuously — the PR would then be scored + # as engine defaults rather than its real, oversized contents. require a + # real retrieval mapping so "empty" can never mean "champion defaults". + encoded = text.encode("utf-8") + if not encoded.strip(): + return ["empty kit (nothing to score — did the fetch truncate?)"] + if len(encoded) > MAX_KIT_BYTES: + return [f"kit larger than {MAX_KIT_BYTES} bytes"] + try: + data = yaml.safe_load(text) + except yaml.YAMLError as exc: + return [f"not valid yaml: {exc}"] + if not isinstance(data, dict) or "retrieval" not in data: + return ["kit must be a mapping with a top-level 'retrieval' section"] + errors: list[str] = [] + _walk(data, (), errors) + return errors + + +def main() -> int: + text = Path(sys.argv[1]).read_text(encoding="utf-8") + errors = validate(text) + if errors: + for err in errors: + print(f"kit rejected: {err}", file=sys.stderr) + return 1 + print(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/koth-gate.yml b/.github/workflows/koth-gate.yml new file mode 100644 index 00000000..b6e3b243 --- /dev/null +++ b/.github/workflows/koth-gate.yml @@ -0,0 +1,181 @@ +# koth ladder gate - scores kit-only PRs against the reigning kit and +# enables auto-merge on a dethrone. the ditto mining loop, rebuilt on +# github primitives: the bench is the validator, branch protection is the +# chain, auto-merge is the emission. +# +# security model (do not weaken): +# - pull_request_target: this definition and every line of executed code +# come from the BASE branch. PR code is never checked out or executed. +# - the only thing read from the PR is competition/kits/current/kit.yaml, +# fetched over the api as data and schema-validated (closed allowlist) +# before it is passed to the bench as extra_config. +# - a PR qualifies for the ladder only if the kit file is the ONLY file +# it touches. anything else is a normal PR: the gate passes without +# scoring and a human reviews as usual. +# - untrusted strings (branch names, repo names, titles) reach scripts +# via env, never by interpolation into run bodies. +name: koth-gate + +on: + pull_request_target: + types: [opened, synchronize, reopened, ready_for_review] + +concurrency: + group: koth-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + gate: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write # auto-merge (squash) needs it + pull-requests: write # enable auto-merge + scorecard comment + steps: + - name: checkout base branch (trusted code only) + uses: actions/checkout@v4 + # no ref: pull_request_target checks out the base branch tip + + - name: classify the PR + id: classify + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \ + --paginate --jq '.[].filename' > /tmp/changed.txt + count=$(wc -l < /tmp/changed.txt) + if [ "$count" = "1" ] && \ + grep -qx 'competition/kits/current/kit.yaml' /tmp/changed.txt; then + echo "mode=ladder" >> "$GITHUB_OUTPUT" + else + echo "mode=normal" >> "$GITHUB_OUTPUT" + fi + + - name: pass through (not a ladder PR) + if: steps.classify.outputs.mode == 'normal' + run: echo "not a kit-only PR - koth gate does not apply, humans review." + + - name: fetch challenger kit from the PR (data only) + if: steps.classify.outputs.mode == 'ladder' + env: + GH_TOKEN: ${{ github.token }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + KIT_PATH: competition/kits/current/kit.yaml + run: | + # the contents api inlines base64 only for files under its size + # limit (~1MB); a larger file returns empty content and a download + # url instead. read size + content together and refuse anything + # that is not a small inlined blob, so an oversized kit can never + # decode to "" and be scored as champion defaults. + gh api "repos/${HEAD_REPO}/contents/${KIT_PATH}?ref=${HEAD_SHA}" \ + > /tmp/kit-meta.json + size=$(jq -r '.size // 0' /tmp/kit-meta.json) + encoding=$(jq -r '.encoding // ""' /tmp/kit-meta.json) + if [ "$encoding" != "base64" ]; then + echo "kit not returned as an inlined base64 blob (encoding=$encoding, size=$size)" >&2 + exit 1 + fi + jq -r '.content' /tmp/kit-meta.json | base64 -d > /tmp/kit-challenger.yaml + if [ ! -s /tmp/kit-challenger.yaml ]; then + echo "fetched kit is empty" >&2 + exit 1 + fi + + - name: set up python + if: steps.classify.outputs.mode == 'ladder' + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: install vouch (base branch code) + if: steps.classify.outputs.mode == 'ladder' + run: python -m pip install -e . + + - name: validate challenger kit against the allowlist + if: steps.classify.outputs.mode == 'ladder' + run: python .github/scripts/validate_kit.py /tmp/kit-challenger.yaml + + - name: paired scoring - challenger vs reigning kit + if: steps.classify.outputs.mode == 'ladder' + id: score + env: + BASE_SHA: ${{ github.sha }} + run: | + set +e + python .github/scripts/koth_score.py \ + --champion competition/kits/current/kit.yaml \ + --challenger /tmp/kit-challenger.yaml \ + --base-sha "$BASE_SHA" \ + --out /tmp/koth-report.json + code=$? + set -e + if [ "$code" = "0" ]; then + echo "verdict=dethroned" >> "$GITHUB_OUTPUT" + elif [ "$code" = "3" ]; then + echo "verdict=held" >> "$GITHUB_OUTPUT" + else + exit "$code" + fi + + - name: post the scorecard + if: steps.classify.outputs.mode == 'ladder' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + VERDICT: ${{ steps.score.outputs.verdict }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + LADDER_BASE: ${{ vars.KOTH_LADDER_BASE }} + run: | + if [ "$BASE_REF" = "$LADDER_BASE" ] && [ -n "$LADDER_BASE" ]; then + gate_note="auto-merge on dethrone (provisional ladder branch)" + else + gate_note="scored only - auto-merge is disabled for base '${BASE_REF}'; a maintainer promotes proven champions to shipped defaults by hand" + fi + { + echo "koth ladder - ${VERDICT}" + echo + echo '```json' + cat /tmp/koth-report.json + echo '```' + echo + echo "${gate_note}." + echo + echo "seeds derive from base sha + utc date; rerun locally with" + echo '`python .github/scripts/koth_score.py --date ` to reproduce.' + echo "the daily result is provisional and overfittable - payout rank is" + echo "settled monthly on sealed commit-reveal seeds (docs/vouchbench-seasons.md)." + } > /tmp/comment.md + gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --body-file /tmp/comment.md + + # auto-merge fires ONLY on a dedicated, non-shipped ladder branch + # (repo variable KOTH_LADDER_BASE). the trunk is never auto-written: + # a kit-only PR against main is scored and commented, then a human + # merges. this keeps the review gate load-bearing for everything that + # ships - a beatable benchmark must never be the sole writer to code + # users install. when KOTH_LADDER_BASE is unset, nothing auto-merges. + - name: enable auto-merge on dethrone (ladder branch only) + if: >- + steps.classify.outputs.mode == 'ladder' && + steps.score.outputs.verdict == 'dethroned' && + vars.KOTH_LADDER_BASE != '' && + github.event.pull_request.base.ref == vars.KOTH_LADDER_BASE + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + gh pr merge "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --auto --squash + + - name: hold the throne (challenger did not clear the band) + if: >- + steps.classify.outputs.mode == 'ladder' && + steps.score.outputs.verdict == 'held' + run: | + echo "champion holds - challenger did not clear max(0.007, 1.96 x paired SE)." + exit 1 diff --git a/competition/LEADERBOARD.md b/competition/LEADERBOARD.md new file mode 100644 index 00000000..cc0204e0 --- /dev/null +++ b/competition/LEADERBOARD.md @@ -0,0 +1,17 @@ +# koth ladder — throne history + +every row is a dethrone: a kit-only PR that beat the reigning kit by the +margin band and auto-merged on the ladder branch. the merged PR is the +authoritative record; this file is the human-readable ledger, appended by +the maintainer when the monthly season closes and payouts are computed. + +the daily throne is provisional (public seeds — see docs/koth-ladder.md); +payout rank is settled by the monthly sealed commit-reveal run. + +| # | champion | PR | dethroned on | scored mean | margin over prior | +|---|----------|----|--------------|-------------|-------------------| +| 0 | baseline kit (repo defaults) | — | 2026-07-27 | 0.57 ± 0.04 (seeds 1–6) | — | + +payouts follow the season shares in docs/vouchbench-seasons.md +(65/14/10/7/4). days-on-throne accrue between dethrones; the monthly +commit-reveal scored run settles the season standings. diff --git a/competition/kits/current/kit.yaml b/competition/kits/current/kit.yaml new file mode 100644 index 00000000..61ee53ad --- /dev/null +++ b/competition/kits/current/kit.yaml @@ -0,0 +1,27 @@ +# the reigning retrieval kit - vouch's answer to ditto's starter-kit crate. +# ascii only in this file: it crosses tools and locales. +# +# this file is the ONLY thing a ladder PR may change, and it is the WHOLE +# retrieval config of the bench kb: vouchbench writes `review:` plus this +# file verbatim as the throwaway kb's config.yaml. omitting a key does not +# inherit anything - it means the engine's code default for that knob. +# the reigning kit spells out every knob it relies on, and the current +# values reproduce the repo baseline exactly. +# +# dethrone these values by max(0.007, 1.64 * paired SE) over the day's +# seeds and your PR auto-merges. allowed keys and bounds are enforced by +# .github/scripts/validate_kit.py - anything else fails before scoring. +retrieval: + backend: auto + default_limit: 10 + recency: + enabled: true + half_life_days: 90.0 + prompt_gate: + enabled: true + rerank: + enabled: false + top_k: 50 + pages_first: + enabled: false + boost: 1.25 diff --git a/docs/koth-ladder.md b/docs/koth-ladder.md new file mode 100644 index 00000000..9a7f5bc8 --- /dev/null +++ b/docs/koth-ladder.md @@ -0,0 +1,141 @@ +# the koth ladder — mine on vouch, through a pull request + +ditto onboards contributors by letting miners tweak a retrieval starter +kit, scoring the result, and paying whoever holds the throne. the vouch +ladder is the same loop rebuilt on github primitives — no wallet, no +subnet, no tarball uploads. your mining rig is a fork and a text editor. + +## the loop + +1. **fork the repo** and edit exactly one file: + [`competition/kits/current/kit.yaml`](../competition/kits/current/kit.yaml) + — the reigning retrieval kit. it is the whole retrieval config of the + benchmark kb: backend, result limit, recency half-life, prompt-gate, + rerank, and pages-first knobs, all bounded by a strict allowlist + (`.github/scripts/validate_kit.py`). it is pure data — a kit carries + no code. +2. **open a PR** that touches only that file, **against the ladder + branch** (see setup below). the `koth-gate` workflow classifies it as + a ladder entry, validates your kit, and runs vouchbench across the + day's seeds — reigning kit vs yours, paired on identical generated + sessions (common random numbers). +3. **dethrone** if your mean improvement clears the band + `max(0.007, 1.96 x paired SE)`. the gate posts the full scorecard as a + PR comment either way. +4. **auto-merge on the ladder branch.** on a dethrone against the ladder + branch the workflow enables github auto-merge; once required checks + are green your PR lands with no human in the loop. your kit is now the + champion every later challenger must beat, and the merge commit is + your permanent, public receipt. +5. **earn.** days-on-throne accrue until someone dethrones you. the daily + ladder is provisional (see "why the daily throne is only practice"); + seasons close monthly with a sealed commit-reveal scored run + (docs/vouchbench-seasons.md), and that is what settles payouts + (shares 65/14/10/7/4). the ledger lives in `competition/LEADERBOARD.md`. + +## reproduce any score locally + +```bash +pip install -e . +python .github/scripts/koth_score.py \ + --champion competition/kits/current/kit.yaml \ + --challenger my-kit.yaml \ + --base-sha "$(git rev-parse origin/main)" \ + --date "$(date -u +%F)" +``` + +seeds derive from the champion sha and the utc date, so every scorecard +in ci can be recomputed by anyone — pass the same `--date` the run used +(it is printed in the scorecard) to reproduce a past result exactly. no +judge model, no hidden eval set, no api key. + +## why the daily throne is only practice + +the daily seeds are a deterministic function of public inputs (the base +sha and the utc date), and the whole scorer is offline-reproducible — by +design, so anyone can audit a score. the cost is that a contributor can +grid-search the knob space against today's exact seeds and submit only a +kit that already wins. the daily ladder therefore behaves like a public +practice leaderboard: fast, transparent, and overfittable. + +that is fine, because **money is never tied to the daily throne.** payout +rank is decided by the monthly season's scored run, whose seeds come from +a commit-reveal drand round at the cutoff and do not exist until entries +are frozen (docs/vouchbench-seasons.md). the daily loop is where you +iterate and get instant feedback; the sealed monthly run is where a kit +has to generalise to seeds nobody could tune against. + +the band (`max(0.007, 1.96 x paired SE)`) still matters daily: it stops a +kit that merely ties the champion from taking the throne, and the 0.007 +floor holds the line when a deterministic overfit collapses the SE toward +zero. + +## why only a data file auto-merges, and only on the ladder branch + +the review gate is vouch's load-bearing invariant, and it stays load +bearing here in two ways: + +- **code never auto-merges.** the ladder surface is a schema-validated + config fragment; the worst a malicious entry can do is score badly. a + merged ladder PR provably contains exactly one bounded data file. +- **the trunk is never auto-written.** auto-merge fires only for PRs + whose base is the dedicated ladder branch (repo variable + `KOTH_LADDER_BASE`). a kit-only PR opened against `main` is scored and + commented, then a human merges — so a beatable benchmark is never the + sole gate to code that users install. promoting the reigning kit into + the shipped starter-config defaults is a periodic, human-reviewed PR. + +engine improvements — new retrieval stages, new signals, bench +extensions — are welcome as normal PRs with human review, and the +maintainer may then expose a new knob in the kit allowlist so the ladder +can tune it. that is the division of labour: humans review capabilities, +the ladder optimises coefficients. miners on ditto never write to ditto's +core either — they submit kits that a validator scores. same boundary, +expressed as a path allowlist and a branch boundary instead of a tarball +contract. + +## anti-cheat, mapped from ditto + +| threat | ditto's answer | the ladder's answer | +|---|---|---| +| overfitting the eval | hidden benchmark versions | conceded for the daily ladder (public seeds) — it is explicitly practice; the monthly settlement uses commit-reveal seeds that do not exist until cutoff, so the paying rank cannot be pre-fit | +| lookup tables | ast minhash scan | impossible by construction: kits carry no code | +| self-reported scores | tee-locked validator | scoring runs from base-branch code under `pull_request_target`; a PR cannot alter the grader, the workflow, or the seeds it is scored with | +| copy the champion + epsilon | first-seen protection | the band forces a real margin over the reigning kit, and a tie loses to the throne | +| oversized / empty kit | — | the fetch refuses any file not returned as a small inlined blob, and the validator rejects an empty or non-`retrieval` kit, so "empty" can never be scored as champion defaults | +| stale wins racing a new champion | — | branch protection in strict mode on the ladder branch: a dethrone that lands invalidates every open challenger's check until it rebases and re-scores against the new champion | +| benchmark as the only gate to shipped code | tee isolation | auto-merge is confined to the ladder branch; the trunk keeps human review, and champions reach shipped defaults only through a human PR | + +## one-time repo setup (maintainer) + +```bash +# 1. create the dedicated ladder branch off the trunk and seed the champion kit +git switch -c ladder origin/main && git push -u origin ladder + +# 2. tell the gate which base is the auto-merge ladder branch +gh variable set KOTH_LADDER_BASE --body ladder + +# 3. allow the auto-merge button repo-wide +gh repo edit --enable-auto-merge + +# 4. protect the ladder branch: gate is a required check, strict mode. +# (json body via --input so booleans stay booleans — a bare -f sends +# the string "true" and the api 422s.) +cat > /tmp/ladder-protection.json <<'JSON' +{ + "required_status_checks": { "strict": true, "contexts": ["gate"] }, + "enforce_admins": false, + "required_pull_request_reviews": null, + "restrictions": null +} +JSON +gh api -X PUT repos/OWNER/REPO/branches/ladder/protection \ + --input /tmp/ladder-protection.json +``` + +strict mode is what closes the race window: after any merge to the ladder +branch, every open ladder PR must update its branch, which re-triggers +scoring against the new champion. `koth-gate` runs on every PR and passes +trivially for non-kit PRs, so requiring it never blocks normal +development. leave `main` protected with your usual human-review rules — +the gate never auto-merges there. diff --git a/docs/vouchbench-seasons.md b/docs/vouchbench-seasons.md index d74f352f..7e61164a 100644 --- a/docs/vouchbench-seasons.md +++ b/docs/vouchbench-seasons.md @@ -29,7 +29,7 @@ machine; that reproducibility is the whole trust model. around `proposals.approve()`, plaintext storage, no baked model deps). Near-identical entries: the earlier-opened PR wins (first-seen). 6. **Pay and merge.** Every entry that beats `main`'s composite by the - margin band — `max(0.007, 1.64 x SE_paired)` over the scored seeds — + margin band — `max(0.007, 1.96 x SE_paired)` over the scored seeds — earns its rank share of the pool (65 / 14 / 10 / 7 / 4 while the field is small). The winner merges and becomes the champion the next season must dethrone. diff --git a/tests/test_koth_kit.py b/tests/test_koth_kit.py new file mode 100644 index 00000000..e809df00 --- /dev/null +++ b/tests/test_koth_kit.py @@ -0,0 +1,66 @@ +"""The koth kit validator — the closed-world allowlist that lets a kit-only +PR auto-merge without a human. Every test here guards a way an untrusted kit +could smuggle something past the gate.""" + +import importlib.util +from pathlib import Path + +_SPEC = importlib.util.spec_from_file_location( + "validate_kit", + Path(__file__).resolve().parents[1] / ".github" / "scripts" / "validate_kit.py", +) +assert _SPEC and _SPEC.loader +validate_kit = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(validate_kit) +validate = validate_kit.validate + +CHAMPION = ( + Path(__file__).resolve().parents[1] + / "competition" + / "kits" + / "current" + / "kit.yaml" +).read_text(encoding="utf-8") + + +def test_champion_kit_validates() -> None: + assert validate(CHAMPION) == [] + + +def test_empty_kit_is_rejected() -> None: + # the >1MB contents-api bypass: an oversized file returns empty content, + # which must NOT validate as "champion defaults". + assert validate("") != [] + assert validate(" \n \n") != [] + + +def test_kit_without_retrieval_section_is_rejected() -> None: + assert validate("review:\n auto_approve_on_receipt: true\n") != [] + + +def test_command_key_is_rejected() -> None: + # config can name executables (compile.llm_cmd); a kit must never reach one. + errors = validate("retrieval:\n backend: auto\ncompile:\n llm_cmd: 'sh -c evil'\n") + assert any("not in allowlist" in e for e in errors) + + +def test_unknown_retrieval_key_is_rejected() -> None: + errors = validate("retrieval:\n backend: auto\n secret_knob: 1\n") + assert any("not in allowlist" in e for e in errors) + + +def test_out_of_bounds_values_are_rejected() -> None: + assert validate("retrieval:\n backend: nonsense\n") != [] + assert validate("retrieval:\n recency:\n half_life_days: 99999\n") != [] + assert validate("retrieval:\n rerank:\n top_k: 0\n") != [] + assert validate("retrieval:\n pages_first:\n boost: -1\n") != [] + + +def test_bool_is_not_accepted_as_a_number() -> None: + # yaml true is an int subclass in python; a numeric knob must refuse it. + assert validate("retrieval:\n recency:\n half_life_days: true\n") != [] + + +def test_oversized_kit_is_rejected() -> None: + big = "retrieval:\n backend: auto\n" + ("# pad\n" * 2000) + assert any("larger than" in e for e in validate(big)) From 81787ef71a39d192eae504a295ecc1ba6f189347 Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Mon, 27 Jul 2026 23:57:35 +0900 Subject: [PATCH 02/13] docs(competition): fix stale band value in kit comment the kit's own comment still read 1.64; the band is 1.96 everywhere else (koth_score, docs, leaderboard). also note auto-merge is ladder-branch scoped. --- competition/kits/current/kit.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/competition/kits/current/kit.yaml b/competition/kits/current/kit.yaml index 61ee53ad..59fb05dc 100644 --- a/competition/kits/current/kit.yaml +++ b/competition/kits/current/kit.yaml @@ -8,9 +8,10 @@ # the reigning kit spells out every knob it relies on, and the current # values reproduce the repo baseline exactly. # -# dethrone these values by max(0.007, 1.64 * paired SE) over the day's -# seeds and your PR auto-merges. allowed keys and bounds are enforced by -# .github/scripts/validate_kit.py - anything else fails before scoring. +# dethrone these values by max(0.007, 1.96 * paired SE) over the day's +# seeds and your PR auto-merges on the ladder branch. allowed keys and +# bounds are enforced by .github/scripts/validate_kit.py - anything else +# fails before scoring. retrieval: backend: auto default_limit: 10 From 57db093d6a18567299d312b6b4c9351745023487 Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:24:19 +0900 Subject: [PATCH 03/13] feat(bench): verifiability categories and the five-tool memory contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit benchmark v2 grows the axis recall-only benchmarks cannot measure: citation-correctness (the surfaced answer must be spelled by a quote whose byte-offset receipt verifies), receipt-coverage (fraction of surfaced claim items carrying a verifying receipt), and supersede-hygiene (a landed update must retire the stale value from the live claim set). the first two are guards that score 1.00 stock; the third is a lever at 0.00 alongside knowledge-update, pointing at lifecycle-driven supersession. memory_contract.MemoryContract implements ditto's five mining-contract tools (save, search, search subjects, fetch by id, search in subject) over the real store, so a shared-contract head-to-head can run both engines on one task set. saves route through the receipt-gated capture loop and return nothing durable when the gate is off — the review gate is never silently bypassed, even inside a harness. reference baseline refreshed to 0.52 +- 0.03 over seeds 1-6 (the category set change is a bench-contract version bump). --- src/vouch/bench.py | 197 ++++++++++++++++++++++++++++++++-- src/vouch/memory_contract.py | 147 +++++++++++++++++++++++++ tests/test_bench.py | 102 +++++++++++++++++- tests/test_memory_contract.py | 89 +++++++++++++++ 4 files changed, 527 insertions(+), 8 deletions(-) create mode 100644 src/vouch/memory_contract.py create mode 100644 tests/test_memory_contract.py diff --git a/src/vouch/bench.py b/src/vouch/bench.py index 7aca5b4d..1955d968 100644 --- a/src/vouch/bench.py +++ b/src/vouch/bench.py @@ -22,6 +22,15 @@ rebuilds the index, and retrieves through ``context.build_context_pack``. A score means vouch-as-shipped retrieved it, under the same review-gate invariants as always. +* **Verifiability axes.** Three categories grade the store's receipts and + lifecycle state, not the pack text: citation-correctness (the surfaced + answer must be spelled by a quote whose byte-offset receipt verifies), + receipt-coverage (fraction of surfaced claim items carrying a verifying + receipt), and supersede-hygiene (once an update landed, the stale value + must not survive as a live claim). Recall-only engines cannot score here + by construction — the axis receipts make measurable. The shared-contract + half of a head-to-head lives in ``memory_contract.MemoryContract``, the + five-tool (Ditto-contract) adapter over the same store. No model, no network, no wall-clock dependence: `vouch bench run --seed 7` gives the same number on every machine, which is what makes scores comparable @@ -30,17 +39,21 @@ Reference baseline (update when retrieval changes; the levers are the zeros): ====================== ===================================================== -run ``vouch bench run --seeds 1,2,3,4,5,6`` @ 2026-07-27 -composite 0.57 ± 0.04 (SE) +run ``vouch bench run --seeds 1,2,3,4,5,6`` @ 2026-07-28 +composite 0.52 ± 0.03 (SE) single-session-recall 1.00 — verbatim receipts + FTS: plain recall is won -multi-session 0.83 +multi-session 0.50 knowledge-update 0.00 — superseded value stays in the pack; needs lifecycle-driven supersession, not reranking (recency reorders, dump-guard still zeroes) -point-in-time 1.00 +point-in-time 0.83 decoy-discrimination 0.00 — same-attribute other-person value outranks injection-resistance 0.83 -abstention 0.33 — cross-person leak under lexical match +abstention 0.00 — cross-person leak under lexical match +citation-correctness 1.00 — guard: the surfaced answer is receipt-quoted +receipt-coverage 1.00 — guard: surfaced claims are receipt-backed +supersede-hygiene 0.00 — stale value stays live; the lifecycle lever + (same root cause as knowledge-update's zero) ====================== ===================================================== For calibration only (different benchmarks, not directly comparable): @@ -57,7 +70,8 @@ from pathlib import Path from typing import Any -from .storage import KBStore +from .models import ClaimStatus +from .storage import ArtifactNotFoundError, KBStore BENCH_ACTOR = "vouch-bench" DEFAULT_BUDGET_CHARS = 2000 @@ -74,6 +88,19 @@ "decoy-discrimination", "injection-resistance", "abstention", + # The verifiability axes — graded against the store's receipts and + # lifecycle state, not the pack text alone. A recall-only benchmark + # (DittoBench included) cannot measure any of these: they require the + # engine to carry byte-offset receipts in the first place. + "citation-correctness", + "receipt-coverage", + "supersede-hygiene", +) + +# A superseded, archived, or redacted claim is not a live memory (mirrors +# the set the context pack excludes). +_RETIRED_STATUSES = frozenset( + (ClaimStatus.SUPERSEDED, ClaimStatus.ARCHIVED, ClaimStatus.REDACTED) ) _DECOY_PERSON = "alice-example" @@ -116,6 +143,18 @@ "what is my parking spot?", "which parking spot is mine?", )), + ("test runner", ( + "which test runner do we use?", + "what runner executes the test suite?", + )), + ("backup cadence", ( + "how often do backups run?", + "what is the backup cadence?", + )), + ("standup time", ( + "when is the daily standup?", + "what time is standup?", + )), ) _STATEMENT_TEMPLATES = ( @@ -201,6 +240,10 @@ def _coin_value(rng: random.Random, attr: str) -> str: return f"spot {rng.randrange(11, 99)}{rng.choice('bcdfg')}" if attr == "staging region": return f"{_coin_word(rng, 2)}-{rng.randrange(2, 9)}" + if attr == "backup cadence": + return f"every {rng.randrange(3, 9)} hours" + if attr == "standup time": + return f"{rng.randrange(8, 12)}:{rng.choice(('05', '15', '35', '45'))}" return _coin_word(rng) @@ -293,6 +336,36 @@ def question(phrasings: tuple[str, ...]) -> str: )) cases.append(MemoryCase("abstention", question(asks), None, (decoy_value,))) + # 8. citation-correctness: stated once, but graded on the receipt — the + # surfaced answer must be provably quoted from the source bytes. + attr, asks = attrs.pop() + value = _coin_value(rng, attr) + docs.add(spot(), statement(attr, value)) + cases.append(MemoryCase("citation-correctness", question(asks), value)) + + # 9. receipt-coverage: every claim item in the answering pack must carry + # a verifying receipt. A guard category: stock scores 1.0, and a change + # that starts surfacing unbacked content pays for it here. + attr, asks = attrs.pop() + value = _coin_value(rng, attr) + docs.add(spot(), statement(attr, value)) + cases.append(MemoryCase("receipt-coverage", question(asks), value)) + + # 10. supersede-hygiene: v1 then an update to v2, graded on the store — + # the stale value must not survive as a live claim while a live claim + # holds the current one. The lifecycle lever knowledge-update's zero + # points at, made a scored axis of its own. + attr, asks = attrs.pop() + v1 = _coin_value(rng, attr) + v2 = _coin_value(rng, attr) + while v2 == v1: + v2 = _coin_value(rng, attr) + early = rng.randrange(sessions - 1) + late = rng.randrange(early + 1, sessions) + docs.add(early, statement(attr, v1)) + docs.add(late, rng.choice(_UPDATE_TEMPLATES).format(attr=attr, value=v2)) + cases.append(MemoryCase("supersede-hygiene", question(asks), v2, (v1,))) + # Filler prose in every session, shuffled placement. for idx in range(sessions): for _ in range(rng.randrange(2, 5)): @@ -334,6 +407,101 @@ def grade_case(case: MemoryCase, pack_text: str) -> tuple[float, str | None]: return 0.0, "expected value not surfaced" +def _verified_quotes(store: KBStore, citation_ids: list[str]) -> list[str]: + """Quotes of the citations whose byte-offset receipts verify. + + A bare source-id citation, a dangling id, or a forged span contributes + nothing — only a receipt that verifies by string comparison counts. + """ + from .receipts import verify_receipt + + quotes: list[str] = [] + for cid in citation_ids: + try: + ev = store.get_evidence(cid) + raw = store.read_source_content(ev.source_id) + except (ArtifactNotFoundError, OSError): + continue + if verify_receipt(ev, raw).verified and ev.quote: + quotes.append(ev.quote) + return quotes + + +def _item_citations(item: dict[str, Any]) -> list[str]: + return [str(c) for c in item.get("citations", [])] + + +def grade_citation_correctness( + store: KBStore, case: MemoryCase, pack: dict[str, Any] +) -> tuple[float, str | None]: + """The answer must be receipt-backed, not merely present. + + Some claim item surfacing the expected value has to carry a citation + whose *verified* quote spells that value — proof the answer was quoted + from real source bytes rather than drifted or fabricated en route. + """ + expected = (case.expected or "").lower() + if expected not in _pack_text(pack).lower(): + return 0.0, "expected value not surfaced" + for item in pack.get("items", []): + if item.get("type") != "claim": + continue + if expected not in str(item.get("summary", "")).lower(): + continue + quotes = _verified_quotes(store, _item_citations(item)) + if any(expected in q.lower() for q in quotes): + return 1.0, None + return 0.0, "answer surfaced without a verifying receipt" + + +def grade_receipt_coverage( + store: KBStore, case: MemoryCase, pack: dict[str, Any] +) -> tuple[float, str | None]: + """Fraction of the pack's claim items backed by a verifying receipt. + + Deliberately not gated on the expected value surfacing — recall misses + are priced by the recall categories. This axis measures only that what + the pack *does* surface is mechanically backed; an empty pack surfaces + nothing unbacked and scores full. + """ + claim_items = [ + i for i in pack.get("items", []) if i.get("type") == "claim" + ] + if not claim_items: + return 1.0, None + backed = sum( + 1 for i in claim_items + if _verified_quotes(store, _item_citations(i)) + ) + if backed == len(claim_items): + return 1.0, None + return ( + round(backed / len(claim_items), 4), + f"{len(claim_items) - backed} of {len(claim_items)} claim items " + "lack a verifying receipt", + ) + + +def grade_supersede_hygiene( + store: KBStore, case: MemoryCase +) -> tuple[float, str | None]: + """Graded on the store, not the pack: after ingest the stale value must + not survive as a live claim while a live claim holds the current one — + the KB tells one truth, enforced by lifecycle state.""" + stale = [f.lower() for f in case.forbidden] + current = (case.expected or "").lower() + live = [ + c for c in store.list_claims() if c.status not in _RETIRED_STATUSES + ] + for claim in live: + text = claim.text.lower() + if any(s in text for s in stale): + return 0.0, f"stale value still live in claim {claim.id!r}" + if not any(current in c.text.lower() for c in live): + return 0.0, "no live claim carries the current value" + return 1.0, None + + def _pack_text(pack: dict[str, Any]) -> str: # build_context_pack returns a ContextPack.model_dump() dict (plus # transport extras); the graded surface is what an agent would read. @@ -351,6 +519,7 @@ def run( workdir: Path | None = None, extra_config: str | None = None, session_gap_seconds: float = 0.0, + strategy: Any = None, ) -> dict[str, Any]: """Generate, ingest through the real pipeline, retrieve, and grade. @@ -367,6 +536,11 @@ def run( timestamps carry the sessions' temporal order — the structure a recency-aware arm ranks by. Zero (the default) keeps runs fast; the generated dataset is identical either way. + + ``strategy`` is an optional pluggable ranking strategy (see + ``vouch.strategy``) — the engine-lane arm. For a competition submission + it is a ``SandboxProxy`` wrapping the untrusted file; the same generated + dataset scored with vs without it isolates the strategy's contribution. """ import tempfile import time as time_mod @@ -396,8 +570,17 @@ def run( for case in dataset.cases: pack = build_context_pack( store, query=case.question, limit=limit, max_chars=budget_chars, + strategy=strategy, ) - score, reason = grade_case(case, _pack_text(dict(pack))) + pack_dict = dict(pack) + if case.category == "citation-correctness": + score, reason = grade_citation_correctness(store, case, pack_dict) + elif case.category == "receipt-coverage": + score, reason = grade_receipt_coverage(store, case, pack_dict) + elif case.category == "supersede-hygiene": + score, reason = grade_supersede_hygiene(store, case) + else: + score, reason = grade_case(case, _pack_text(pack_dict)) per_category[case.category].append(score) if reason is not None: failures.append({ diff --git a/src/vouch/memory_contract.py b/src/vouch/memory_contract.py new file mode 100644 index 00000000..90ca94b4 --- /dev/null +++ b/src/vouch/memory_contract.py @@ -0,0 +1,147 @@ +"""The five-tool memory contract, implemented over a vouch KB. + +Ditto's mining contract (SN118) scores a harness on exactly five tools: +save a memory, search memories, search subjects, fetch by id, and search +within a subject. This module is vouch's side of that shared contract — a +thin adapter over the real store, so a head-to-head can run both engines +on one task set instead of comparing incomparable benchmark numbers. + +Two properties are non-negotiable and differ from a plain memory store: + +* **Saves go through the review gate.** ``save_memory`` runs the same + receipt-gated capture loop production uses (``extract.ingest_source``). + With ``review.auto_approve_on_receipt`` off, a save files a proposal and + returns nothing durable — the gate is never silently bypassed, even + inside a benchmark harness. +* **Subjects are provenance, not labels.** A memory's subject is the title + of the source it cites; there is no free-floating subject field to + drift from the evidence. Subject search and subject-scoped search both + resolve through citations. +""" + +from __future__ import annotations + +from contextlib import suppress +from dataclasses import dataclass + +from .context import search_kb +from .extract import ingest_source +from .models import Claim, ClaimStatus, Evidence +from .receipts import verify_receipt +from .storage import ArtifactNotFoundError, KBStore + +CONTRACT_TOOLS = ( + "save_memory", + "search_memories", + "search_subjects", + "fetch_by_id", + "search_in_subject", +) + +# Mirrors the retracted set the context pack excludes: a superseded, +# archived, or redacted claim is not a live memory. +_RETRACTED = frozenset( + (ClaimStatus.SUPERSEDED, ClaimStatus.ARCHIVED, ClaimStatus.REDACTED) +) + + +@dataclass(frozen=True) +class MemoryHit: + """One memory as the contract returns it.""" + + id: str + text: str + score: float + subject: str | None + receipt_backed: bool + + +class MemoryContract: + """The five contract tools over one vouch KB.""" + + def __init__(self, store: KBStore, *, actor: str = "memory-contract") -> None: + self.store = store + self.actor = actor + + def save_memory(self, text: str, *, subject: str | None = None) -> list[str]: + """Store one memory; return the ids that became durable. + + An empty list means the receipt gate is off and the memory is + pending human review — filed, not durable. + """ + _, approved = ingest_source( + self.store, text.encode("utf-8"), + proposed_by=self.actor, title=subject, + ) + return [claim.id for claim in approved] + + def search_memories(self, query: str, *, limit: int = 10) -> list[MemoryHit]: + result = search_kb(self.store, query=query, limit=limit) + hits: list[MemoryHit] = [] + for row in result["hits"]: + if row["kind"] != "claim": + continue + try: + claim = self.store.get_claim(str(row["id"])) + except ArtifactNotFoundError: + continue + if claim.status in _RETRACTED: + continue + hits.append(self._hit(claim, score=float(row["score"]))) + return hits + + def search_subjects(self, query: str, *, limit: int = 10) -> list[str]: + needle = query.lower() + subjects = { + subject + for claim in self.store.list_claims() + if claim.status not in _RETRACTED + and (subject := self._subject_of(claim)) is not None + and needle in subject.lower() + } + return sorted(subjects)[:limit] + + def fetch_by_id(self, memory_id: str) -> MemoryHit: + return self._hit(self.store.get_claim(memory_id), score=1.0) + + def search_in_subject( + self, subject: str, query: str, *, limit: int = 10 + ) -> list[MemoryHit]: + wide = self.search_memories(query, limit=max(limit * 5, 25)) + return [hit for hit in wide if hit.subject == subject][:limit] + + def _hit(self, claim: Claim, *, score: float) -> MemoryHit: + return MemoryHit( + id=claim.id, + text=claim.text, + score=score, + subject=self._subject_of(claim), + receipt_backed=any( + verify_receipt( + ev, self.store.read_source_content(ev.source_id) + ).verified + for ev in self._evidence_of(claim) + ), + ) + + def _evidence_of(self, claim: Claim) -> list[Evidence]: + out: list[Evidence] = [] + for cid in claim.evidence: + try: + out.append(self.store.get_evidence(cid)) + except ArtifactNotFoundError: + continue # a bare source-id citation carries no receipt + return out + + def _subject_of(self, claim: Claim) -> str | None: + for cid in claim.evidence: + source_id = cid + with suppress(ArtifactNotFoundError): + source_id = self.store.get_evidence(cid).source_id + try: + title = self.store.get_source(source_id).title + except ArtifactNotFoundError: + continue + if title: + return title + return None diff --git a/tests/test_bench.py b/tests/test_bench.py index ca59e98b..676c706e 100644 --- a/tests/test_bench.py +++ b/tests/test_bench.py @@ -3,12 +3,27 @@ from __future__ import annotations import json +from pathlib import Path from click.testing import CliRunner -from vouch import bench +from vouch import bench, health, lifecycle from vouch.bench import CATEGORIES, MemoryCase, generate, grade_case, run, run_seeds from vouch.cli import cli +from vouch.context import build_context_pack +from vouch.extract import ingest_source +from vouch.storage import KBStore + + +def _mini_kb(tmp_path: Path, *sentences: str) -> KBStore: + store = KBStore.init(tmp_path / "kb") + store.config_path.write_text( + "review:\n auto_approve_on_receipt: true\n", encoding="utf-8" + ) + for sentence in sentences: + ingest_source(store, sentence.encode("utf-8"), proposed_by="bench-test") + health.rebuild_index(store) + return store def test_generate_is_deterministic() -> None: @@ -88,3 +103,88 @@ def test_cli_bench_gen_emits_dataset() -> None: assert payload["seed"] == 5 assert len(payload["cases"]) == len(CATEGORIES) assert payload["sessions"] + + +def test_categories_include_verifiability_axes() -> None: + assert "citation-correctness" in CATEGORIES + assert "receipt-coverage" in CATEGORIES + assert "supersede-hygiene" in CATEGORIES + + +def test_grade_citation_correctness_requires_verifying_receipt( + tmp_path: Path, +) -> None: + store = _mini_kb( + tmp_path, "for the record, my favorite editor is zorvex right now." + ) + case = MemoryCase("citation-correctness", "what is my favorite editor?", "zorvex") + pack = dict( + build_context_pack(store, query=case.question, limit=10, max_chars=2000) + ) + assert bench.grade_citation_correctness(store, case, pack)[0] == 1.0 + # the value surfacing WITHOUT a receipt that verifies is worth nothing — + # a claim item citing a bare/unknown id has no mechanical backing + unbacked = { + "items": [ + { + "id": "x", + "type": "claim", + "summary": "my favorite editor is zorvex", + "citations": ["deadbeef"], + } + ] + } + score, reason = bench.grade_citation_correctness(store, case, unbacked) + assert score == 0.0 + assert reason is not None + + +def test_grade_receipt_coverage_full_on_receipt_backed_pack( + tmp_path: Path, +) -> None: + store = _mini_kb(tmp_path, "the project codename is mulopi now.") + case = MemoryCase("receipt-coverage", "what is the project codename?", "mulopi") + pack = dict( + build_context_pack(store, query=case.question, limit=10, max_chars=2000) + ) + assert bench.grade_receipt_coverage(store, case, pack)[0] == 1.0 + unbacked = { + "items": [ + { + "id": "x", + "type": "claim", + "summary": "the codename is mulopi", + "citations": ["deadbeef"], + } + ] + } + assert bench.grade_receipt_coverage(store, case, unbacked)[0] == 0.0 + + +def test_grade_supersede_hygiene_rewards_lifecycle(tmp_path: Path) -> None: + store = _mini_kb( + tmp_path, + "for the record, my deploy day is monday right now.", + "heads up, the deploy day changed to friday this week.", + ) + case = MemoryCase( + "supersede-hygiene", "which day do we deploy?", "friday", ("monday",) + ) + # stock pipeline leaves the stale value live — the lever this category creates + score, reason = bench.grade_supersede_hygiene(store, case) + assert score == 0.0 + assert reason is not None + old = next(c for c in store.list_claims() if "monday" in c.text) + new = next(c for c in store.list_claims() if "friday" in c.text) + lifecycle.supersede(store, old_claim_id=old.id, new_claim_id=new.id, actor="t") + assert bench.grade_supersede_hygiene(store, case)[0] == 1.0 + + +def test_run_verifiability_stock_scores() -> None: + report = run(1, sessions=4) + # guard categories: the receipt path makes these perfect on the stock + # engine; a change that surfaces unbacked content loses points here + assert report["categories"]["receipt-coverage"]["mean"] == 1.0 + assert report["categories"]["citation-correctness"]["mean"] == 1.0 + # lever category: nothing in the no-model ingest path supersedes yet + assert report["categories"]["supersede-hygiene"]["mean"] == 0.0 diff --git a/tests/test_memory_contract.py b/tests/test_memory_contract.py new file mode 100644 index 00000000..7e8c8b3f --- /dev/null +++ b/tests/test_memory_contract.py @@ -0,0 +1,89 @@ +"""The five-tool memory contract: gate-respecting saves, subject-scoped search.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from vouch.memory_contract import CONTRACT_TOOLS, MemoryContract, MemoryHit +from vouch.storage import ArtifactNotFoundError, KBStore + + +def _kb(tmp_path: Path, *, receipt_gate: bool = True) -> KBStore: + # init's starter config already opts into the receipt gate (phase d); + # the gate-off arm must configure it off explicitly. + store = KBStore.init(tmp_path / "kb") + flag = "true" if receipt_gate else "false" + store.config_path.write_text( + f"review:\n auto_approve_on_receipt: {flag}\n", encoding="utf-8" + ) + return store + + +def test_contract_names_the_five_ditto_tools() -> None: + assert CONTRACT_TOOLS == ( + "save_memory", + "search_memories", + "search_subjects", + "fetch_by_id", + "search_in_subject", + ) + + +def test_save_then_search_roundtrip(tmp_path: Path) -> None: + contract = MemoryContract(_kb(tmp_path)) + saved = contract.save_memory( + "for the record, my favorite editor is zorvex right now." + ) + assert saved, "receipt-gated save produced no durable memory" + hits = contract.search_memories("favorite editor") + assert any("zorvex" in h.text for h in hits) + + +def test_save_without_receipt_gate_stays_pending(tmp_path: Path) -> None: + # The review gate is never silently bypassed: with auto-approve off the + # save files a proposal and nothing becomes durable or searchable. + contract = MemoryContract(_kb(tmp_path, receipt_gate=False)) + saved = contract.save_memory("the staging region is vora-3 as of today.") + assert saved == [] + assert contract.search_memories("staging region") == [] + + +def test_fetch_by_id_returns_saved_memory(tmp_path: Path) -> None: + contract = MemoryContract(_kb(tmp_path)) + saved = contract.save_memory("the project codename is mulopi now.") + hit = contract.fetch_by_id(saved[0]) + assert isinstance(hit, MemoryHit) + assert hit.id == saved[0] + assert "mulopi" in hit.text + assert hit.receipt_backed is True + + +def test_fetch_by_id_unknown_raises(tmp_path: Path) -> None: + contract = MemoryContract(_kb(tmp_path)) + with pytest.raises(ArtifactNotFoundError): + contract.fetch_by_id("no-such-memory") + + +def test_search_subjects_matches_saved_subjects(tmp_path: Path) -> None: + contract = MemoryContract(_kb(tmp_path)) + contract.save_memory("my usual coffee order is a flat white.", subject="preferences") + contract.save_memory("the deploy day moved to tuesday.", subject="ops-runbook") + assert contract.search_subjects("prefer") == ["preferences"] + assert contract.search_subjects("runbook") == ["ops-runbook"] + assert contract.search_subjects("r") == ["ops-runbook", "preferences"] + + +def test_search_in_subject_scopes_hits(tmp_path: Path) -> None: + contract = MemoryContract(_kb(tmp_path)) + contract.save_memory( + "the api rate limit is 640 requests per minute.", subject="ops-runbook" + ) + contract.save_memory( + "personal note: my api rate limit worry is overblown.", subject="journal" + ) + hits = contract.search_in_subject("ops-runbook", "api rate limit") + assert hits, "subject-scoped search found nothing" + assert all(h.subject == "ops-runbook" for h in hits) + assert any("640" in h.text for h in hits) From 7a62b73a2ccf3e81573ebe8add1f73cf329b06a0 Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:24:49 +0900 Subject: [PATCH 04/13] docs(competition): refresh ladder baseline for the v2 bench contract the verifiability categories change the category set, so seeds 1-6 rescore: composite 0.52 +- 0.03 replaces 0.57 +- 0.04 as row 0. the koth fold compares only max-bench-version scores, so the reigning-kit number and the challenger number stay on the same contract. --- competition/LEADERBOARD.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/competition/LEADERBOARD.md b/competition/LEADERBOARD.md index cc0204e0..f6e40405 100644 --- a/competition/LEADERBOARD.md +++ b/competition/LEADERBOARD.md @@ -10,7 +10,7 @@ payout rank is settled by the monthly sealed commit-reveal run. | # | champion | PR | dethroned on | scored mean | margin over prior | |---|----------|----|--------------|-------------|-------------------| -| 0 | baseline kit (repo defaults) | — | 2026-07-27 | 0.57 ± 0.04 (seeds 1–6) | — | +| 0 | baseline kit (repo defaults) | — | 2026-07-28 | 0.52 ± 0.03 (seeds 1–6) | — | payouts follow the season shares in docs/vouchbench-seasons.md (65/14/10/7/4). days-on-throne accrue between dethrones; the monthly From 8bcec71eb1294a6888ea72b3deba76a34ba32a3a Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:45:41 +0900 Subject: [PATCH 05/13] feat(competition): engine lane - submit ranking code, scored in a sandbox the engine-submission half of the koth ladder, and the real ditto equivalent: contributors submit a retrieval STRATEGY as python code (a rank(query, candidates, *, limit) -> ids function under contrib/strategies/), and vouchbench scores it. the kit lane only tuned coefficients; this is where a new fusion or reranker actually competes. the load-bearing rule: a kit is data and auto-merges; a strategy is code and NEVER auto-merges. vouch is a library people install, so a winning strategy earns the leaderboard and payout and ships only after human review merges it as a default - the review gate applied to engine code. scoring runs untrusted code, so vouch.strategy.run_sandboxed executes each submission in a python -I child that sets rlimits and installs an audit hook blocking network, subprocess, os.exec/fork, ctypes.dlopen, and filesystem writes before importing the file; the parent enforces a wall-clock timeout and treats any failure as no-reordering. the engine hook in build_context_pack is off by default and byte-identical when unset (retrieval.strategy, a dotted path, wires a merged strategy). koth-engine-gate.yml runs pull_request_target with contents:read, no secrets, no write token, no merge. hardened against an adversarial review. two confirmed defects fixed with regression tests: the audit hook read its blocklists from module globals (disarmable in two lines by reassigning __main__ globals) - now bound in closure cells; and the result travelled on the strategy's own stdout (any print corrupted it, silently scoring a good strategy as a no-op) - the child now writes the result to a private dup of stdout with fd 1 pointed at /dev/null. the docstring is honest that in-process python isolation is best-effort and the real boundary is the ephemeral runner plus human merge. --- .github/scripts/score_strategy.py | 98 +++++++ .github/workflows/koth-engine-gate.yml | 152 +++++++++++ contrib/strategies/README.md | 58 ++++ contrib/strategies/baseline.py | 19 ++ contrib/strategies/example_lexical.py | 34 +++ docs/koth-strategy-lane.md | 96 +++++++ src/vouch/context.py | 70 +++++ src/vouch/strategy.py | 356 +++++++++++++++++++++++++ tests/test_strategy.py | 202 ++++++++++++++ 9 files changed, 1085 insertions(+) create mode 100644 .github/scripts/score_strategy.py create mode 100644 .github/workflows/koth-engine-gate.yml create mode 100644 contrib/strategies/README.md create mode 100644 contrib/strategies/baseline.py create mode 100644 contrib/strategies/example_lexical.py create mode 100644 docs/koth-strategy-lane.md create mode 100644 src/vouch/strategy.py create mode 100644 tests/test_strategy.py diff --git a/.github/scripts/score_strategy.py b/.github/scripts/score_strategy.py new file mode 100644 index 00000000..b3d16d5e --- /dev/null +++ b/.github/scripts/score_strategy.py @@ -0,0 +1,98 @@ +"""Paired champion-vs-challenger scoring for the engine (strategy) lane. + +Same dethrone test as the kit lane (docs/vouchbench-seasons.md), but the arm +is a pluggable ranking strategy instead of a config fragment: each submission +is loaded as an untrusted file and run through vouch.strategy.SandboxProxy, so +the scored code executes only inside the sandbox child. + + dethroned iff mean(challenger - champion) >= max(0.007, 1.96 x SE) + +Exit code 0 = dethroned, 3 = held, 1 = error. Unlike the kit lane, a winning +verdict here does NOT auto-merge: engine code ships only through human review. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import statistics +from pathlib import Path + +from vouch import bench +from vouch.strategy import SandboxProxy + +N_SEEDS = 8 +FLOOR = 0.007 +Z = 1.96 +SESSION_GAP_SECONDS = 2.0 + + +def day_seeds(base_sha: str, date: str, n: int = N_SEEDS) -> list[int]: + seeds = [] + for i in range(n): + digest = hashlib.sha256(f"{base_sha}:{date}:{i}".encode()).hexdigest() + seeds.append(int(digest[:12], 16)) + return seeds + + +def score(path: str, seeds: list[int]) -> list[float]: + proxy = SandboxProxy(path) + return [ + bench.run(s, strategy=proxy, session_gap_seconds=SESSION_GAP_SECONDS)[ + "composite" + ] + for s in seeds + ] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--champion", required=True) + parser.add_argument("--challenger", required=True) + parser.add_argument("--base-sha", required=True) + parser.add_argument("--date", default=None) + parser.add_argument("--out", default=None) + args = parser.parse_args() + + date = args.date or dt.datetime.now(dt.UTC).strftime("%Y-%m-%d") + seeds = day_seeds(args.base_sha, date) + + champion_scores = score(args.champion, seeds) + challenger_scores = score(args.challenger, seeds) + diffs = [ + c - r for c, r in zip(challenger_scores, champion_scores, strict=True) + ] + mean_diff = statistics.mean(diffs) + se = statistics.stdev(diffs) / (len(diffs) ** 0.5) if len(diffs) > 1 else 0.0 + band = max(FLOOR, Z * se) + dethroned = mean_diff >= band + + report = { + "date": date, + "base_sha": args.base_sha, + "seeds": seeds, + "lane": "engine", + "champion": { + "scores": champion_scores, + "mean": statistics.mean(champion_scores), + }, + "challenger": { + "scores": challenger_scores, + "mean": statistics.mean(challenger_scores), + }, + "mean_diff": mean_diff, + "se": se, + "band": band, + "dethroned": dethroned, + } + text = json.dumps(report, indent=1) + print(text) + if args.out: + Path(args.out).write_text(text + "\n", encoding="utf-8") + return 0 if dethroned else 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/koth-engine-gate.yml b/.github/workflows/koth-engine-gate.yml new file mode 100644 index 00000000..6eb8a56f --- /dev/null +++ b/.github/workflows/koth-engine-gate.yml @@ -0,0 +1,152 @@ +# koth engine gate - scores strategy (ranking-code) submissions. +# +# the deliberate contrast with koth-gate.yml (the kit lane): a kit is data +# and auto-merges; a STRATEGY is code and NEVER auto-merges. this job has no +# write token, holds no secrets, and merges nothing. it scores the submission +# in a sandbox and posts a scorecard + leaderboard note; a human reviews the +# code and merges it as a new default if it wins. that human review is the +# review gate applied to engine code. +# +# security model: +# - pull_request_target: the workflow, the grader (score_strategy.py), and +# the champion strategy all come from the BASE branch. only the challenger +# .py is read from the PR, and it is executed ONLY inside the sandbox +# child (vouch.strategy.run_sandboxed: rlimits + an audit hook blocking +# network/subprocess/writes). +# - permissions: contents: read only. even a full sandbox escape lands on +# an ephemeral runner with a read-only token and no secrets, and cannot +# merge itself. +name: koth-engine-gate + +on: + pull_request_target: + types: [opened, synchronize, reopened, ready_for_review] + +concurrency: + group: koth-engine-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + gate: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + pull-requests: write # scorecard comment only - no merge + steps: + - name: checkout base branch (trusted code only) + uses: actions/checkout@v4 + + - name: classify the PR + id: classify + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \ + --paginate --jq '.[].filename' > /tmp/changed.txt + count=$(wc -l < /tmp/changed.txt) + only=$(head -1 /tmp/changed.txt) + case "$only" in + contrib/strategies/baseline.py|contrib/strategies/README.md) only="" ;; + esac + if [ "$count" = "1" ] && \ + printf '%s' "$only" | grep -qE '^contrib/strategies/[A-Za-z0-9_]+\.py$'; then + echo "mode=engine" >> "$GITHUB_OUTPUT" + echo "path=$only" >> "$GITHUB_OUTPUT" + else + echo "mode=normal" >> "$GITHUB_OUTPUT" + fi + + - name: pass through (not a strategy PR) + if: steps.classify.outputs.mode == 'normal' + run: echo "not a single-strategy PR - engine gate does not apply." + + - name: fetch challenger strategy from the PR (as data) + if: steps.classify.outputs.mode == 'engine' + env: + GH_TOKEN: ${{ github.token }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + KIT_PATH: ${{ steps.classify.outputs.path }} + run: | + gh api "repos/${HEAD_REPO}/contents/${KIT_PATH}?ref=${HEAD_SHA}" \ + > /tmp/strat-meta.json + encoding=$(jq -r '.encoding // ""' /tmp/strat-meta.json) + if [ "$encoding" != "base64" ]; then + echo "strategy not returned as an inlined blob (encoding=$encoding)" >&2 + exit 1 + fi + jq -r '.content' /tmp/strat-meta.json | base64 -d > /tmp/challenger.py + if [ ! -s /tmp/challenger.py ]; then + echo "fetched strategy is empty" >&2 + exit 1 + fi + + - name: set up python + if: steps.classify.outputs.mode == 'engine' + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: install vouch (base branch code) + if: steps.classify.outputs.mode == 'engine' + run: python -m pip install -e . + + - name: paired scoring - challenger vs baseline champion (sandboxed) + if: steps.classify.outputs.mode == 'engine' + id: score + env: + BASE_SHA: ${{ github.sha }} + run: | + set +e + python .github/scripts/score_strategy.py \ + --champion contrib/strategies/baseline.py \ + --challenger /tmp/challenger.py \ + --base-sha "$BASE_SHA" \ + --out /tmp/engine-report.json + code=$? + set -e + if [ "$code" = "0" ]; then + echo "verdict=dethroned" >> "$GITHUB_OUTPUT" + elif [ "$code" = "3" ]; then + echo "verdict=held" >> "$GITHUB_OUTPUT" + else + exit "$code" + fi + + - name: post the scorecard + if: steps.classify.outputs.mode == 'engine' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + VERDICT: ${{ steps.score.outputs.verdict }} + run: | + { + echo "koth engine lane - ${VERDICT}" + echo + echo '```json' + cat /tmp/engine-report.json + echo '```' + echo + echo "engine code is NOT auto-merged. a winning strategy earns the" + echo "leaderboard place; a maintainer reviews the code and merges it" + echo "as a new default. the daily result is provisional (public seeds)" + echo "- payout rank is settled by the monthly sealed run." + } > /tmp/comment.md + gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --body-file /tmp/comment.md + + - name: report the verdict as the check result + if: steps.classify.outputs.mode == 'engine' + env: + VERDICT: ${{ steps.score.outputs.verdict }} + run: | + echo "verdict: ${VERDICT}" + # a held challenger is a green, informational result - nothing merges + # here regardless, so the gate never blocks. a scoring error already + # failed the job above. + exit 0 diff --git a/contrib/strategies/README.md b/contrib/strategies/README.md new file mode 100644 index 00000000..fa4abbb8 --- /dev/null +++ b/contrib/strategies/README.md @@ -0,0 +1,58 @@ +# contrib/strategies - the engine-submission lane + +this is the ditto-equivalent lane: you submit **real ranking code**, not a +config file. a strategy decides the order the reader sees retrieved +candidates in - the place where a new fusion, a learned reranker, or a novel +signal actually lives. + +## the contract + +your file exposes a `rank` function (or a `STRATEGY` object with a `.rank` +method): + +```python +from vouch.strategy import Candidate + +def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]: + # return candidate ids, best first + ... +``` + +- a `Candidate` has `kind`, `id`, `summary`, `score` - **data only**. your + code never gets the KB, the filesystem, or the network. +- ordering is authoritative but bounded: ids you invent are ignored, and any + candidate you omit is appended in its original order. you can reorder and + de-prioritise; you cannot fabricate or hide a result. +- [`baseline.py`](./baseline.py) is the reigning champion (returns the + backend order unchanged). [`example_lexical.py`](./example_lexical.py) is a + worked example you can study and beat. + +## how it is scored + +open a PR that touches **only** your new file under `contrib/strategies/`. +the `koth-engine-gate` workflow: + +1. runs your code in a locked-down `python -I` sandbox (resource limits + an + audit hook that blocks network, subprocess, and filesystem writes); +2. scores vouchbench with your strategy vs the reigning champion, paired over + the day's seeds; +3. posts the scorecard and updates the engine leaderboard on a win. + +## the one hard rule: engine code is never auto-merged + +the config (kit) lane auto-merges because a kit cannot execute. **strategy +code can**, and vouch is a library people install - so a winning strategy is +never merged automatically. it earns the leaderboard place and the payout, +and ships only after a human reviews the code and merges it as a new default. +that human review is vouch's version of ditto's tee-plus-deployment gate: the +benchmark decides the *rank*, a person decides what *ships*. + +reproduce any score locally: + +```bash +pip install -e . +python .github/scripts/score_strategy.py \ + --champion contrib/strategies/baseline.py \ + --challenger contrib/strategies/example_lexical.py \ + --base-sha "$(git rev-parse origin/main)" --date "$(date -u +%F)" +``` diff --git a/contrib/strategies/baseline.py b/contrib/strategies/baseline.py new file mode 100644 index 00000000..e74d7d09 --- /dev/null +++ b/contrib/strategies/baseline.py @@ -0,0 +1,19 @@ +"""The reigning engine-lane champion: trust the backend's fused order. + +This is the strategy every submission must beat. It returns the candidates in +exactly the order retrieval handed them over - i.e. it does nothing - so a +challenger only dethrones it by making vouch's benchmark score genuinely +higher, not by accident. + +A strategy is real ranking code. It receives the query and the retrieved +candidates (data only - no KB, no disk, no network) and returns the ids in the +order the reader should see them. Ordering is authoritative but bounded: ids +you invent are ignored, and any candidate you drop is appended at the tail, so +you can reorder and de-prioritise but never fabricate or hide a result. +""" + +from vouch.strategy import Candidate + + +def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]: + return [c.id for c in candidates] diff --git a/contrib/strategies/example_lexical.py b/contrib/strategies/example_lexical.py new file mode 100644 index 00000000..aead101c --- /dev/null +++ b/contrib/strategies/example_lexical.py @@ -0,0 +1,34 @@ +"""A worked example: re-rank by lexical overlap with the query. + +Not necessarily a winner - it exists to show the shape of a real submission. +It boosts candidates whose summary shares more words with the query, blended +with the backend's own score so a strong retrieval signal is not thrown away. + +Copy this file, change the scoring, and open a PR that touches only your new +file under contrib/strategies/. The engine gate scores it in a sandbox against +the reigning champion; you never edit the engine itself. +""" + +import re + +from vouch.strategy import Candidate + +_WORD = re.compile(r"[a-z0-9]+") + + +def _tokens(text: str) -> set[str]: + return set(_WORD.findall(text.lower())) + + +def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]: + q = _tokens(query) + if not q: + return [c.id for c in candidates] + + def blended(c: Candidate) -> float: + overlap = len(q & _tokens(c.summary)) / len(q) + # keep the backend score in the mix; overlap only tips ties and + # rescues a lexically-obvious hit the fusion under-ranked. + return 0.7 * c.score + 0.3 * overlap + + return [c.id for c in sorted(candidates, key=blended, reverse=True)] diff --git a/docs/koth-strategy-lane.md b/docs/koth-strategy-lane.md new file mode 100644 index 00000000..3adf35b4 --- /dev/null +++ b/docs/koth-strategy-lane.md @@ -0,0 +1,96 @@ +# the engine lane - submit ranking code, not config + +the kit ladder (docs/koth-ladder.md) lets contributors tune coefficients. +this lane is the ditto-equivalent: contributors submit **real engine code** - +a retrieval strategy that decides the order the reader sees candidates in - +and the benchmark scores it. it is the place a new fusion, a learned reranker, +or a novel signal actually competes. + +## why a second lane exists + +a kit can only move dials that already exist, so its ceiling is low - most +single knobs are no-ops on the current bench. real gains come from new ranking +logic. ditto accepts exactly this (miners submit the retrieval harness). the +difference is that ditto is a hosted service scored in a tee, while vouch is a +library people install - so the two lanes draw the trust boundary in different +places: + +| | kit lane | engine lane | +|---|---|---| +| what you submit | `competition/kits/current/kit.yaml` (data) | `contrib/strategies/.py` (code) | +| scored by | vouchbench, config arm | vouchbench, sandboxed strategy arm | +| on a win | **auto-merges** (data cannot execute) | leaderboard + payout; **never auto-merges** | +| how it ships | promoted to defaults by a human PR | reviewed and merged by a human | + +## the interface + +```python +from vouch.strategy import Candidate + +def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]: + # return candidate ids, best first + ... +``` + +a `Candidate` is `kind`, `id`, `summary`, `score` - **data only**. the +strategy never receives the KB, the filesystem, or a socket. ordering is +authoritative but bounded (ids you invent are ignored; candidates you omit are +appended), so a strategy can reorder and de-prioritise but cannot fabricate or +hide a result. see `contrib/strategies/baseline.py` (the champion) and +`example_lexical.py` (a worked example). + +## how a submission is scored + +open a PR touching only your new `contrib/strategies/.py`. the +`koth-engine-gate` workflow runs your code through +`vouch.strategy.run_sandboxed` and scores vouchbench with it, paired against +the reigning champion over the day's seeds, then posts the scorecard. + +### the sandbox + +each `rank` call runs in a fresh `python -I` child that, before importing your +file, installs: + +- **resource limits** (`RLIMIT_CPU`, `RLIMIT_AS`, `RLIMIT_NOFILE`) and a + parent-side wall-clock timeout - a runaway or memory-hungry strategy is + killed, not the run; +- **an audit hook** (`sys.addaudithook`) that refuses network + (`socket.*`/`urllib.*`), process spawning (`subprocess`, `os.exec`/`fork`, + native `ctypes.dlopen`), and filesystem writes (any `open` with write + intent). reads are allowed so numpy and friends still import. + +a crash, a timeout, or a blocked call yields "no reordering" - a broken +strategy simply fails to improve; it cannot take down scoring. + +### the honest boundary + +an in-process python guard cannot stop a determined native-code escape. it is +defence in depth, not the trust root. the real boundary is the same one ditto +relies on: the scoring job runs on an ephemeral CI runner with a **read-only +token and no secrets**, and **engine code is never auto-merged**. the benchmark +decides the *rank*; a human reviewing the diff decides what *ships*. if you +ever score submissions off ephemeral CI, add OS-level isolation (a container, +nsjail, or gVisor) before trusting the audit hook alone. + +## reproduce locally + +```bash +pip install -e . +python .github/scripts/score_strategy.py \ + --champion contrib/strategies/baseline.py \ + --challenger contrib/strategies/example_lexical.py \ + --base-sha "$(git rev-parse origin/main)" --date "$(date -u +%F)" +``` + +## shipping a winner (maintainer) + +1. read the strategy code - it is about to run on every user's machine. +2. move it into `src/vouch/strategies/` and point `retrieval.strategy` (a + dotted import path) at it in the starter config, or wire it as the default. + the in-engine hook loads a shipped strategy in-process (no sandbox - it is + now trusted, reviewed code). +3. record the dethrone in the engine ladder's `LEADERBOARD`. + +the config knob `retrieval.strategy` is how a merged strategy actually changes +retrieval; until one is merged it is unset and retrieval is byte-identical to +today. diff --git a/src/vouch/context.py b/src/vouch/context.py index 089701ac..a715ca7e 100644 --- a/src/vouch/context.py +++ b/src/vouch/context.py @@ -20,6 +20,7 @@ import yaml from . import graph, hot_memory, index_db, retrieval_events +from . import strategy as strategy_mod from .embeddings.fusion import rrf_fuse from .models import ClaimStatus, ContextItem, ContextPack, ContextQuality from .scoping import ( @@ -308,6 +309,71 @@ def _maybe_rerank( return ordered + hits[window_size:] +def _configured_strategy(store: KBStore) -> str | None: + """Resolve ``retrieval.strategy`` - a dotted import path to a shipped, + human-merged strategy - from config.yaml. Off (None) by default.""" + try: + loaded = yaml.safe_load(store.config_path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError): + return None + if not isinstance(loaded, dict): + return None + retrieval = loaded.get("retrieval") + if not isinstance(retrieval, dict): + return None + dotted = retrieval.get("strategy") + return dotted if isinstance(dotted, str) and dotted else None + + +def _maybe_strategy( + store: KBStore, + *, + query: str, + hits: list[tuple[str, str, str, float, str]], + limit: int, + strategy: strategy_mod.RetrievalStrategy | None = None, +) -> list[tuple[str, str, str, float, str]]: + """Apply a pluggable ranking strategy as the final reorder stage. + + ``strategy`` is passed explicitly by the benchmark (an untrusted + submission, wrapped in a sandbox proxy); otherwise a shipped strategy is + resolved from ``retrieval.strategy`` config and loaded in-process. With + neither, hits pass through byte-identical. A strategy that raises or + returns nothing usable leaves the order untouched - retrieval never fails + because a ranking plugin misbehaved. + """ + strat = strategy + if strat is None: + dotted = _configured_strategy(store) + if not dotted: + return hits + try: + strat = strategy_mod.load_dotted(dotted) + except Exception: + return hits + if not hits: + return hits + # the strategy addresses hits by id; if two hits somehow share one (a + # cross-kind slug collision), a reorder-by-id would be ambiguous, so skip + # the stage rather than risk attaching an order to the wrong artifact. + ids = [h[1] for h in hits] + if len(set(ids)) != len(ids): + return hits + candidates = [ + strategy_mod.Candidate(kind=k, id=i, summary=s, score=sc) + for k, i, s, sc, _b in hits + ] + try: + ordered_ids = strat.rank(query, candidates, limit=limit) + except Exception: + return hits + by_id = {h[1]: h for h in hits} + reordered = strategy_mod.apply_ordering( + list(ordered_ids), [(k, i, s, sc) for k, i, s, sc, _b in hits] + ) + return [by_id[h4[1]] for h4 in reordered] + + def _retrieve( store: KBStore, query: str, @@ -602,6 +668,7 @@ def build_context_pack( graph_depth: int = 1, graph_limit: int = 20, graph_rel_types: list[str] | None = None, + strategy: strategy_mod.RetrievalStrategy | None = None, ) -> ContextPack | dict[str, Any]: viewer = viewer_from( config_path=store.config_path, @@ -609,6 +676,9 @@ def build_context_pack( agent=agent, ) hits = _retrieve(store, query, limit, viewer) + hits = _maybe_strategy( + store, query=query, hits=hits, limit=limit, strategy=strategy + ) items: list[ContextItem] = [] for kind, hid, summary, score, backend in hits: cites: list[str] = [] diff --git a/src/vouch/strategy.py b/src/vouch/strategy.py new file mode 100644 index 00000000..35d0a090 --- /dev/null +++ b/src/vouch/strategy.py @@ -0,0 +1,356 @@ +"""Pluggable retrieval strategies - the engine-submission lane of the koth +ladder. + +A *strategy* is real ranking code: given the query and the candidate hits a +backend retrieved, it returns the ids in the order the reader should see them. +This is where a contributor competes on algorithm (a new fusion, a learned +reranker, a novel signal) rather than on config coefficients (the kit lane). + +Two ways a strategy runs, with very different trust: + +- **shipped / merged** (trusted): resolved from ``retrieval.strategy`` in + config.yaml as a dotted import path, loaded in-process. It is trusted + because a human reviewed and merged it - the review gate, applied to code. +- **a competition submission** (untrusted): a file under ``contrib/strategies/`` + loaded by the benchmark and run via :func:`run_sandboxed`, which executes it + in a separate ``python -I`` process under resource limits and an audit hook + that blocks network, subprocess, and filesystem writes. + +Honesty about the sandbox boundary: the audit hook + rlimits stop casual and +accidental misbehaviour and raise the bar against a hostile one, but no +in-process Python guard is a true boundary - a determined attacker who can +introspect the interpreter (native ctypes/cffi, or pure-Python gc/frame walking +to reach and neutralise the hook object) can defeat it. The *real* boundary is +the same one ditto leans on: the scoring job runs on an ephemeral CI runner +with a read-only token and no secrets, and **strategy code is never +auto-merged** - it earns a leaderboard place and is shipped only through human +review. So the worst a full escape achieves during scoring is misbehaviour on a +throwaway runner that can reach nothing and merge nothing. The sandbox is +defence in depth; the trust root is the runner's least privilege plus the human +merge gate. +""" + +from __future__ import annotations + +import importlib +import importlib.util +import json +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol, runtime_checkable + +# a retrieval hit as the reorder stages pass it: (kind, id, summary, score). +Hit = tuple[str, str, str, float] + +DEFAULT_TIMEOUT_S = 20.0 +DEFAULT_MEM_MB = 2048 +DEFAULT_CPU_S = 15 + +# audit events refused inside the sandbox child. network, process spawning, +# and native dlopen are blocked outright; filesystem writes are blocked by +# inspecting the open mode (reads - needed to import numpy etc. - are allowed). +_BLOCKED_EXACT = frozenset({ + "os.system", + "os.exec", + "os.fork", + "os.forkpty", + "os.posix_spawn", + "os.spawn", + "subprocess.Popen", + "ctypes.dlopen", + "ctypes.dlsym", + "ctypes.call_function", + "ctypes.cdata", +}) +_BLOCKED_PREFIXES = ("socket.", "urllib.", "ftplib.", "http.") + + +@dataclass(frozen=True) +class Candidate: + """What a strategy sees for one retrieved artifact - data only. + + A strategy never receives the KB, the filesystem, or a network handle; + just these fields, so it cannot reach anything outside the ranking task. + """ + + kind: str + id: str + summary: str + score: float + + +@runtime_checkable +class RetrievalStrategy(Protocol): + """The interface a submission implements. + + ``rank`` returns candidate ids best-first. It is treated as *ordering + only*: ids not in the input are ignored, and any input id the strategy + omits is appended in its original order, so a strategy can reorder and + drop-from-the-top but can never fabricate a result or smuggle one in. + """ + + def rank( + self, query: str, candidates: list[Candidate], *, limit: int + ) -> list[str]: ... + + +def apply_ordering(ordered_ids: list[str], hits: list[Hit]) -> list[Hit]: + """Reorder ``hits`` by ``ordered_ids``, preserving the hit set. + + Mirrors the discipline of the built-in rerank stage: the strategy may move + artifacts but not add or remove them. Unknown ids are dropped; hits the + strategy did not mention keep their relative order at the tail. + """ + by_id: dict[str, Hit] = {} + for hit in hits: + by_id.setdefault(hit[1], hit) + seen: set[str] = set() + ordered: list[Hit] = [] + for hid in ordered_ids: + match = by_id.get(hid) + if match is not None and hid not in seen: + ordered.append(match) + seen.add(hid) + for hit in hits: + if hit[1] not in seen: + ordered.append(hit) + seen.add(hit[1]) + return ordered + + +def load_from_path(path: str | Path) -> RetrievalStrategy: + """Import a strategy from a .py file and return its strategy object. + + The module must expose either a ``STRATEGY`` object with a ``rank`` method + or a module-level ``rank`` function. Loading runs module top-level code, so + only call this on trusted (merged) files or inside the sandbox child. + """ + path = Path(path) + spec = importlib.util.spec_from_file_location(f"vouch_strategy_{path.stem}", path) + if spec is None or spec.loader is None: + raise ValueError(f"cannot load strategy from {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return _strategy_from_module(module) + + +def load_dotted(dotted: str) -> RetrievalStrategy: + """Import a trusted, shipped strategy by dotted module path (config hook).""" + module = importlib.import_module(dotted) + return _strategy_from_module(module) + + +def _strategy_from_module(module: Any) -> RetrievalStrategy: + candidate = getattr(module, "STRATEGY", None) + if candidate is not None and hasattr(candidate, "rank"): + return candidate # type: ignore[no-any-return] + fn = getattr(module, "rank", None) + if callable(fn): + rank_fn = fn + + class _FnStrategy: + def rank( + self, query: str, candidates: list[Candidate], *, limit: int + ) -> list[str]: + return list(rank_fn(query, candidates, limit=limit)) + + return _FnStrategy() + raise ValueError( + "strategy module must define STRATEGY (with .rank) or a rank() function" + ) + + +class SandboxProxy: + """A strategy handle whose ``rank`` runs the real code in a sandbox child. + + Built by the benchmark for an untrusted submission. Each ``rank`` call + spawns a fresh ``python -I`` process, so a submission cannot keep state + between calls or wear down the limits over time. A crash, a timeout, or a + limit breach yields an empty ordering, which :func:`apply_ordering` turns + into "keep the backend's order" - a broken strategy simply fails to + improve rather than taking down the run. + """ + + def __init__( + self, + path: str | Path, + *, + timeout_s: float = DEFAULT_TIMEOUT_S, + mem_mb: int = DEFAULT_MEM_MB, + cpu_s: int = DEFAULT_CPU_S, + ) -> None: + self.path = str(Path(path).resolve()) + self.timeout_s = timeout_s + self.mem_mb = mem_mb + self.cpu_s = cpu_s + self.failures = 0 + + def rank( + self, query: str, candidates: list[Candidate], *, limit: int + ) -> list[str]: + ordered = run_sandboxed( + self.path, + query, + candidates, + limit=limit, + timeout_s=self.timeout_s, + mem_mb=self.mem_mb, + cpu_s=self.cpu_s, + ) + if ordered is None: + self.failures += 1 + return [] + return ordered + + +def run_sandboxed( + path: str, + query: str, + candidates: list[Candidate], + *, + limit: int, + timeout_s: float = DEFAULT_TIMEOUT_S, + mem_mb: int = DEFAULT_MEM_MB, + cpu_s: int = DEFAULT_CPU_S, +) -> list[str] | None: + """Run a strategy file in a locked-down child; return ordered ids or None. + + None means the child failed (crash, timeout, limit, or malformed output) - + the caller treats that as "no reordering", never as an error that aborts + scoring. + """ + payload = { + "path": path, + "query": query, + "limit": limit, + "mem_bytes": mem_mb * 1024 * 1024, + "cpu_seconds": cpu_s, + "candidates": [ + {"kind": c.kind, "id": c.id, "summary": c.summary, "score": c.score} + for c in candidates + ], + } + try: + proc = subprocess.run( + [sys.executable, "-I", "-m", "vouch.strategy", "--child"], + input=json.dumps(payload), + capture_output=True, + text=True, + timeout=timeout_s, + # a minimal env; -I already ignores PYTHON* vars and the cwd. + env={"PATH": "/usr/bin:/bin"}, + ) + except (subprocess.TimeoutExpired, OSError): + return None + if proc.returncode != 0: + return None + try: + result = json.loads(proc.stdout) + ordered = result["ordered"] + except (json.JSONDecodeError, KeyError, TypeError): + return None + if not isinstance(ordered, list): + return None + return [str(x) for x in ordered] + + +# --- sandbox child --------------------------------------------------------- + + +def _install_audit_hook() -> None: + import os + + # Bind the guard sets into closure cells that hold the original frozenset + # objects. The hook reads them via LOAD_DEREF, not from module globals, so + # untrusted top-level code cannot disarm the block by reassigning + # ``__main__._BLOCKED_EXACT`` (the child runs as __main__). This is still + # in-process defence in depth - see the module docstring: a determined + # attacker who introspects the interpreter can defeat any pure-Python + # hook, which is why the real boundary is the ephemeral runner and the + # no-auto-merge rule, not this function. + blocked_exact = _BLOCKED_EXACT + blocked_prefixes = _BLOCKED_PREFIXES + + def _hook(event: str, args: tuple[Any, ...]) -> None: + if event in blocked_exact or event.startswith(blocked_prefixes): + raise PermissionError(f"blocked in strategy sandbox: {event}") + if event == "open": + # args = (path, mode, flags); refuse any write/append/create intent. + mode = args[1] if len(args) > 1 else "" + if isinstance(mode, str) and any(c in mode for c in "wax+"): + raise PermissionError("filesystem writes are blocked in the sandbox") + flags = args[2] if len(args) > 2 else 0 + if isinstance(flags, int) and ( + flags & (os.O_WRONLY | os.O_RDWR | os.O_CREAT | os.O_APPEND) + ): + raise PermissionError("filesystem writes are blocked in the sandbox") + + sys.addaudithook(_hook) + + +def _apply_rlimits(mem_bytes: int, cpu_seconds: int) -> None: + try: + import resource + except ImportError: # non-unix; the audit hook still applies + return + # cpu seconds is a hard ceiling backing the parent's wall-clock timeout; + # nofile caps fd pressure. RLIMIT_AS is deliberately generous - numpy and + # friends reserve large virtual space even at small RSS, and a too-tight + # AS limit kills honest strategies, so wall-clock + cpu are the real caps. + for res, soft in ( + (resource.RLIMIT_CPU, cpu_seconds), + (resource.RLIMIT_AS, mem_bytes), + (resource.RLIMIT_NOFILE, 64), + ): + try: + hard = resource.getrlimit(res)[1] + cap = soft if hard == resource.RLIM_INFINITY else min(soft, hard) + resource.setrlimit(res, (cap, hard)) + except (ValueError, OSError): + pass + + +def _child_main() -> int: + import os + + payload = json.load(sys.stdin) + mem_bytes = int(payload["mem_bytes"]) + cpu_seconds = int(payload["cpu_seconds"]) + path = payload["path"] + query = payload["query"] + limit = int(payload["limit"]) + candidates = [ + Candidate(kind=c["kind"], id=c["id"], summary=c["summary"], score=c["score"]) + for c in payload["candidates"] + ] + # Isolate the result channel from the strategy's stdout: a strategy that + # prints (a debug line, a library banner, a progress bar) must not corrupt + # the JSON the parent parses. Keep a private dup of the real stdout for the + # result, then point fd 1 at /dev/null so anything the strategy writes is + # discarded. Both opens happen before the write-blocking audit hook arms. + result_fd = os.dup(1) + devnull_fd = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull_fd, 1) + # limits and the audit hook are armed BEFORE the untrusted module is loaded. + _apply_rlimits(mem_bytes, cpu_seconds) + _install_audit_hook() + strategy = load_from_path(path) + ordered = strategy.rank(query, candidates, limit=limit) + valid = {c.id for c in candidates} + ordered_ids = [str(x) for x in ordered if str(x) in valid] + os.write(result_fd, json.dumps({"ordered": ordered_ids}).encode("utf-8")) + return 0 + + +def main(argv: list[str] | None = None) -> int: + argv = sys.argv[1:] if argv is None else argv + if argv and argv[0] == "--child": + return _child_main() + print("usage: python -I -m vouch.strategy --child (reads a job on stdin)") + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_strategy.py b/tests/test_strategy.py new file mode 100644 index 00000000..103ffecf --- /dev/null +++ b/tests/test_strategy.py @@ -0,0 +1,202 @@ +"""The pluggable-strategy engine lane, with the sandbox as the load-bearing +part. Every escape test here guards the boundary that lets vouch score +untrusted ranking code automatically.""" + +from pathlib import Path + +import pytest + +from vouch.strategy import ( + Candidate, + SandboxProxy, + apply_ordering, + load_from_path, + run_sandboxed, +) + +REPO = Path(__file__).resolve().parents[1] +BASELINE = str(REPO / "contrib" / "strategies" / "baseline.py") +EXAMPLE = str(REPO / "contrib" / "strategies" / "example_lexical.py") + +CANDS = [ + Candidate("claim", "a", "alpha beta", 0.9), + Candidate("claim", "b", "gamma delta", 0.5), + Candidate("page", "c", "beta gamma query", 0.3), +] + + +def _write(tmp_path: Path, body: str) -> str: + p = tmp_path / "challenger.py" + p.write_text(body, encoding="utf-8") + return str(p) + + +# --- interface + ordering discipline -------------------------------------- + + +def test_baseline_is_identity() -> None: + strat = load_from_path(BASELINE) + assert strat.rank("beta", CANDS, limit=10) == ["a", "b", "c"] + + +def test_apply_ordering_drops_unknown_and_appends_missing() -> None: + hits = [(c.kind, c.id, c.summary, c.score) for c in CANDS] + out = apply_ordering(["c", "invented", "a"], hits) + # 'invented' dropped, 'b' (unmentioned) appended at the tail. + assert [h[1] for h in out] == ["c", "a", "b"] + + +def test_apply_ordering_cannot_grow_the_set() -> None: + hits = [(c.kind, c.id, c.summary, c.score) for c in CANDS] + out = apply_ordering(["a", "a", "b", "c", "c"], hits) + assert sorted(h[1] for h in out) == ["a", "b", "c"] + + +# --- sandbox happy path ---------------------------------------------------- + + +def test_sandbox_runs_and_matches_in_process() -> None: + assert run_sandboxed(BASELINE, "beta", CANDS, limit=10) == ["a", "b", "c"] + + +def test_sandbox_is_deterministic() -> None: + a = run_sandboxed(EXAMPLE, "beta gamma query", CANDS, limit=10) + b = run_sandboxed(EXAMPLE, "beta gamma query", CANDS, limit=10) + assert a == b and a is not None + + +def test_sandbox_only_returns_known_ids() -> None: + body = """ +from vouch.strategy import Candidate +def rank(query, candidates, *, limit): + return ["a", "ghost", "c"] # 'ghost' is not a candidate +""" + # the child filters to valid ids before returning. + import tempfile + + with tempfile.TemporaryDirectory() as d: + path = _write(Path(d), body) + assert run_sandboxed(path, "q", CANDS, limit=10) == ["a", "c"] + + +# --- sandbox blocks (the security contract) ------------------------------- + + +def test_sandbox_blocks_network(tmp_path: Path) -> None: + body = """ +from vouch.strategy import Candidate +def rank(query, candidates, *, limit): + import socket + socket.socket().connect(("1.1.1.1", 80)) + return [c.id for c in candidates] +""" + assert run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10) is None + + +def test_sandbox_blocks_file_write(tmp_path: Path) -> None: + target = tmp_path / "pwned" + body = f""" +from vouch.strategy import Candidate +def rank(query, candidates, *, limit): + open({str(target)!r}, "w").write("x") + return [c.id for c in candidates] +""" + assert run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10) is None + assert not target.exists() + + +def test_sandbox_blocks_subprocess(tmp_path: Path) -> None: + body = """ +from vouch.strategy import Candidate +def rank(query, candidates, *, limit): + import subprocess + subprocess.Popen(["/bin/echo", "hi"]) + return [c.id for c in candidates] +""" + assert run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10) is None + + +def test_sandbox_kills_infinite_loop(tmp_path: Path) -> None: + body = """ +from vouch.strategy import Candidate +def rank(query, candidates, *, limit): + while True: + pass +""" + assert ( + run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10, timeout_s=5) + is None + ) + + +def test_audit_hook_survives_module_global_disarm(tmp_path: Path) -> None: + # the child runs as __main__; reassigning the guard globals must NOT + # re-enable network/exec, because the hook reads them from closure cells + # holding the original frozensets. + body = """ +import sys +_m = sys.modules["__main__"] +_m._BLOCKED_EXACT = frozenset() +_m._BLOCKED_PREFIXES = () +def rank(query, candidates, *, limit): + import socket + socket.socket().connect(("1.1.1.1", 80)) + return [c.id for c in candidates] +""" + assert run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10) is None + + +def test_strategy_stdout_does_not_corrupt_result(tmp_path: Path) -> None: + # a strategy that prints debug output must still have its reordering + # respected - the result travels on a channel the strategy cannot dirty. + body = """ +import sys +def rank(query, candidates, *, limit): + print("noisy debug line") + sys.stdout.write("more noise\\n") + return list(reversed([c.id for c in candidates])) +""" + assert run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10) == [ + "c", + "b", + "a", + ] + + +def test_sandbox_survives_a_crashing_strategy(tmp_path: Path) -> None: + body = """ +def rank(query, candidates, *, limit): + raise RuntimeError("boom") +""" + assert run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10) is None + + +def test_proxy_counts_failures_and_returns_empty(tmp_path: Path) -> None: + body = """ +def rank(query, candidates, *, limit): + raise RuntimeError("boom") +""" + proxy = SandboxProxy(_write(tmp_path, body)) + assert proxy.rank("q", CANDS, limit=10) == [] + assert proxy.failures == 1 + + +# --- retrieval integration ------------------------------------------------- + + +def test_build_context_pack_strategy_none_is_default(tmp_path: Path) -> None: + # strategy=None must not change retrieval - exercised in bench parity, but + # assert the config hook returns None cleanly on a bare KB here. + from vouch.context import _configured_strategy + from vouch.storage import KBStore + + store = KBStore.init(tmp_path / "kb") + assert _configured_strategy(store) is None + + +@pytest.mark.parametrize("path", [BASELINE, EXAMPLE]) +def test_shipped_examples_load(path: str) -> None: + strat = load_from_path(path) + assert hasattr(strat, "rank") + result = strat.rank("beta gamma", CANDS, limit=10) + assert sorted(result) == ["a", "b", "c"] From bc5f23b64f5ea2d53878e4255e80dd50d4730ea6 Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:13:06 +0900 Subject: [PATCH 06/13] feat(bench): local strategy scoring via --strategy/--against vouch bench run gains --strategy (score a sandboxed ranking strategy exactly like ci) and --against (paired dethrone verdict vs a champion strategy before pushing a pr). run_seeds threads the strategy through. the dethrone test now lives once, in bench.paired_verdict, with the floor and z constants exported; both ci scorers call it instead of carrying their own copies, so the margin math cannot drift between the local practice loop and the gates. --- .github/scripts/koth_score.py | 33 ++++------------- .github/scripts/score_strategy.py | 30 ++++------------ src/vouch/bench.py | 48 +++++++++++++++++++++++++ src/vouch/cli.py | 59 ++++++++++++++++++++++++++++--- tests/test_bench.py | 43 ++++++++++++++++++++++ 5 files changed, 160 insertions(+), 53 deletions(-) diff --git a/.github/scripts/koth_score.py b/.github/scripts/koth_score.py index 1b5af781..a32e8668 100644 --- a/.github/scripts/koth_score.py +++ b/.github/scripts/koth_score.py @@ -31,7 +31,6 @@ import datetime as dt import hashlib import json -import statistics from pathlib import Path from vouch import bench @@ -42,8 +41,8 @@ # the paired diffs are not perfectly normal, so the floor below still does # the real gatekeeping when the SE collapses on a deterministic overfit. N_SEEDS = 12 -FLOOR = 0.007 -Z = 1.96 +FLOOR = bench.DETHRONE_FLOOR +Z = bench.DETHRONE_Z # bench sleeps this long between session ingests to spread created_at # timestamps; keep it small — it is wall-clock, not simulated time. the # 3600s that lived here briefly would have out-slept the CI job timeout @@ -87,40 +86,22 @@ def main() -> int: champion_scores = score(champion_text, seeds) challenger_scores = score(challenger_text, seeds) - diffs = [ - c - r for c, r in zip(challenger_scores, champion_scores, strict=True) - ] - mean_diff = statistics.mean(diffs) - se = ( - statistics.stdev(diffs) / (len(diffs) ** 0.5) - if len(diffs) > 1 - else 0.0 + verdict = bench.paired_verdict( + champion_scores, challenger_scores, floor=FLOOR, z=Z, ) - band = max(FLOOR, Z * se) - dethroned = mean_diff >= band report = { "date": date, "base_sha": args.base_sha, "seeds": seeds, - "champion": { - "scores": champion_scores, - "mean": statistics.mean(champion_scores), - }, - "challenger": { - "scores": challenger_scores, - "mean": statistics.mean(challenger_scores), - }, - "mean_diff": mean_diff, - "se": se, - "band": band, - "dethroned": dethroned, + "lane": "kit", + **verdict, } text = json.dumps(report, indent=1) print(text) if args.out: Path(args.out).write_text(text + "\n", encoding="utf-8") - return 0 if dethroned else 3 + return 0 if report["dethroned"] else 3 if __name__ == "__main__": diff --git a/.github/scripts/score_strategy.py b/.github/scripts/score_strategy.py index b3d16d5e..128993fb 100644 --- a/.github/scripts/score_strategy.py +++ b/.github/scripts/score_strategy.py @@ -17,15 +17,14 @@ import datetime as dt import hashlib import json -import statistics from pathlib import Path from vouch import bench from vouch.strategy import SandboxProxy N_SEEDS = 8 -FLOOR = 0.007 -Z = 1.96 +FLOOR = bench.DETHRONE_FLOOR +Z = bench.DETHRONE_Z SESSION_GAP_SECONDS = 2.0 @@ -61,37 +60,22 @@ def main() -> int: champion_scores = score(args.champion, seeds) challenger_scores = score(args.challenger, seeds) - diffs = [ - c - r for c, r in zip(challenger_scores, champion_scores, strict=True) - ] - mean_diff = statistics.mean(diffs) - se = statistics.stdev(diffs) / (len(diffs) ** 0.5) if len(diffs) > 1 else 0.0 - band = max(FLOOR, Z * se) - dethroned = mean_diff >= band + verdict = bench.paired_verdict( + champion_scores, challenger_scores, floor=FLOOR, z=Z, + ) report = { "date": date, "base_sha": args.base_sha, "seeds": seeds, "lane": "engine", - "champion": { - "scores": champion_scores, - "mean": statistics.mean(champion_scores), - }, - "challenger": { - "scores": challenger_scores, - "mean": statistics.mean(challenger_scores), - }, - "mean_diff": mean_diff, - "se": se, - "band": band, - "dethroned": dethroned, + **verdict, } text = json.dumps(report, indent=1) print(text) if args.out: Path(args.out).write_text(text + "\n", encoding="utf-8") - return 0 if dethroned else 3 + return 0 if report["dethroned"] else 3 if __name__ == "__main__": diff --git a/src/vouch/bench.py b/src/vouch/bench.py index 1955d968..11905e8d 100644 --- a/src/vouch/bench.py +++ b/src/vouch/bench.py @@ -619,6 +619,7 @@ def run_seeds( sessions: int = DEFAULT_SESSIONS, extra_config: str | None = None, session_gap_seconds: float = 0.0, + strategy: Any = None, ) -> dict[str, Any]: """Run several seeds; report mean composite with a standard error. @@ -630,6 +631,7 @@ def run_seeds( run( s, budget_chars=budget_chars, limit=limit, sessions=sessions, extra_config=extra_config, session_gap_seconds=session_gap_seconds, + strategy=strategy, ) for s in seeds ] @@ -658,6 +660,52 @@ def run_seeds( } +# The dethrone test from docs/vouchbench-seasons.md. FLOOR does the real +# gatekeeping when a deterministic overfit collapses the SE to zero; Z is the +# two-sided 95% normal quantile. Both CI scorers and the local --against loop +# call paired_verdict, so the margin math cannot drift between them. +DETHRONE_FLOOR = 0.007 +DETHRONE_Z = 1.96 + + +def paired_verdict( + champion_scores: list[float], + challenger_scores: list[float], + *, + floor: float = DETHRONE_FLOOR, + z: float = DETHRONE_Z, +) -> dict[str, Any]: + """Apply the paired dethrone test to two same-seed score lists. + + dethroned iff mean(challenger - champion) >= max(floor, z * SE) where SE + is the standard error of the per-seed paired differences (common random + numbers cancel seed-to-seed variance). + """ + diffs = [ + c - r for c, r in zip(challenger_scores, champion_scores, strict=True) + ] + mean_diff = statistics.mean(diffs) + se = ( + statistics.stdev(diffs) / (len(diffs) ** 0.5) + if len(diffs) > 1 else 0.0 + ) + band = max(floor, z * se) + return { + "champion": { + "scores": champion_scores, + "mean": statistics.mean(champion_scores), + }, + "challenger": { + "scores": challenger_scores, + "mean": statistics.mean(challenger_scores), + }, + "mean_diff": mean_diff, + "se": se, + "band": band, + "dethroned": mean_diff >= band, + } + + def format_report(report: dict[str, Any]) -> str: """Render one run() report as an aligned text table.""" lines = [ diff --git a/src/vouch/cli.py b/src/vouch/cli.py index aba0730c..129727bb 100644 --- a/src/vouch/cli.py +++ b/src/vouch/cli.py @@ -5401,10 +5401,21 @@ def bench() -> None: ) @click.option("--budget-chars", default=None, type=int, help="Context pack budget.") @click.option("--limit", default=None, type=int, help="Max context items per query.") +@click.option( + "--strategy", "strategy_path", default=None, + type=click.Path(exists=True, dir_okay=False), + help="Score with this ranking strategy file, sandboxed exactly like CI.", +) +@click.option( + "--against", "against_path", default=None, + type=click.Path(exists=True, dir_okay=False), + help="Champion strategy file to pair against; prints the dethrone verdict.", +) @click.option("--json", "as_json", is_flag=True, help="Emit the full report as JSON.") def bench_run( seed: int, seeds: str | None, budget_chars: int | None, - limit: int | None, as_json: bool, + limit: int | None, strategy_path: str | None, against_path: str | None, + as_json: bool, ) -> None: """Score retrieval on a generated dataset. @@ -5412,14 +5423,54 @@ def bench_run( Examples: vouch bench run --seed 7 vouch bench run --seeds 1,2,3,4,5 --json + vouch bench run --seeds 1,2,3 --strategy contrib/strategies/mine.py \\ + --against contrib/strategies/baseline.py """ from . import bench as bench_mod budget = budget_chars if budget_chars is not None else bench_mod.DEFAULT_BUDGET_CHARS top_k = limit if limit is not None else bench_mod.DEFAULT_LIMIT + if against_path and not strategy_path: + raise click.UsageError("--against requires --strategy") + strategy = None + if strategy_path: + from .strategy import SandboxProxy + + strategy = SandboxProxy(strategy_path) + seed_list = ( + [int(s) for s in seeds.replace(" ", "").split(",") if s] + if seeds else [seed] + ) + if against_path: + champion = SandboxProxy(against_path) + champion_scores = [ + bench_mod.run( + s, budget_chars=budget, limit=top_k, strategy=champion, + )["composite"] + for s in seed_list + ] + challenger_scores = [ + bench_mod.run( + s, budget_chars=budget, limit=top_k, strategy=strategy, + )["composite"] + for s in seed_list + ] + verdict = bench_mod.paired_verdict(champion_scores, challenger_scores) + verdict["seeds"] = seed_list + if as_json: + click.echo(json.dumps(verdict, indent=2)) + return + click.echo( + f"challenger {verdict['challenger']['mean']:.4f} " + f"champion {verdict['champion']['mean']:.4f} " + f"diff {verdict['mean_diff']:+.4f} band {verdict['band']:.4f}" + ) + click.echo("DETHRONED" if verdict["dethroned"] else "held") + return if seeds: - seed_list = [int(s) for s in seeds.replace(" ", "").split(",") if s] - report = bench_mod.run_seeds(seed_list, budget_chars=budget, limit=top_k) + report = bench_mod.run_seeds( + seed_list, budget_chars=budget, limit=top_k, strategy=strategy, + ) if as_json: click.echo(json.dumps(report, indent=2)) return @@ -5430,7 +5481,7 @@ def bench_run( for name, mean in report["categories"].items(): click.echo(f" {name:<24} {mean:>5.2f}") return - report = bench_mod.run(seed, budget_chars=budget, limit=top_k) + report = bench_mod.run(seed, budget_chars=budget, limit=top_k, strategy=strategy) if as_json: click.echo(json.dumps(report, indent=2)) return diff --git a/tests/test_bench.py b/tests/test_bench.py index 676c706e..25554512 100644 --- a/tests/test_bench.py +++ b/tests/test_bench.py @@ -5,6 +5,7 @@ import json from pathlib import Path +import pytest from click.testing import CliRunner from vouch import bench, health, lifecycle @@ -188,3 +189,45 @@ def test_run_verifiability_stock_scores() -> None: assert report["categories"]["citation-correctness"]["mean"] == 1.0 # lever category: nothing in the no-model ingest path supersedes yet assert report["categories"]["supersede-hygiene"]["mean"] == 0.0 + + +def test_paired_verdict_floor_gatekeeps_zero_se() -> None: + # identical per-seed diffs collapse the SE; the floor still gates + verdict = bench.paired_verdict([0.5, 0.5, 0.5], [0.505, 0.505, 0.505]) + assert verdict["se"] == 0.0 + assert verdict["band"] == bench.DETHRONE_FLOOR + assert not verdict["dethroned"] + clear = bench.paired_verdict([0.5, 0.5, 0.5], [0.51, 0.51, 0.51]) + assert clear["dethroned"] + + +def test_paired_verdict_requires_same_seed_count() -> None: + with pytest.raises(ValueError): + bench.paired_verdict([0.5, 0.5], [0.5]) + + +def test_run_seeds_threads_strategy( + monkeypatch: pytest.MonkeyPatch, +) -> None: + seen: list[object] = [] + + def fake_run(seed: int, **kwargs: object) -> dict: + seen.append(kwargs.get("strategy")) + return { + "seed": seed, + "composite": 0.5, + "categories": {name: {"mean": 0.5} for name in CATEGORIES}, + } + + monkeypatch.setattr(bench, "run", fake_run) + sentinel = object() + bench.run_seeds([1, 2], strategy=sentinel) + assert seen == [sentinel, sentinel] + + +def test_cli_bench_run_against_requires_strategy(tmp_path: Path) -> None: + champion = tmp_path / "champ.py" + champion.write_text("def rank(query, candidates, *, limit):\n return []\n") + result = CliRunner().invoke(cli, ["bench", "run", "--against", str(champion)]) + assert result.exit_code != 0 + assert "--against requires --strategy" in result.output From fe7d1e9c0eb29a23ffb0fa369dc1a8717a7822a5 Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:16:57 +0900 Subject: [PATCH 07/13] docs(competition): engine lane is the competition, kit lane the warm-up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mining-on-vouch.md is the canonical from-fork-to-shipped walkthrough: fork, practice locally with the exact ci scoring loop, open a pr, read the scorecard, get paid by the season split — and the carrot ditto cannot offer, a reviewed winner ships as the default in every install. the kit ladder is repositioned as the 10-minute on-ramp (its ceiling is low by construction; the bench's zero categories are only reachable from code), the strategy lane doc gets the headline and the local practice command, and the readme points at the walkthrough. --- README.md | 1 + docs/koth-ladder.md | 10 +++- docs/koth-strategy-lane.md | 15 ++++-- docs/mining-on-vouch.md | 95 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 docs/mining-on-vouch.md diff --git a/README.md b/README.md index 4e8ec5f2..4281a20e 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,7 @@ Pending drafts (`proposed/`) and the derived search index (`state.db`) are gitig * `vouch install-mcp ` also wires cursor, codex, zed, windsurf, openclaw and friends ([adapters/](adapters/)) * [vouch webapp](https://github.com/vouchdev/webApp) — the chat-first browser console from the video; [vouch-desktop](https://github.com/vouchdev/vouch-desktop) wraps the same loop as a desktop app * [CONTRIBUTING.md](CONTRIBUTING.md) — development setup and the test gate +* [docs/mining-on-vouch.md](docs/mining-on-vouch.md) — get paid to improve the retrieval engine: submit ranking code, scored by a reproducible benchmark; the best verified strategy ships as the default ## Incubated by Gittensor diff --git a/docs/koth-ladder.md b/docs/koth-ladder.md index 9a7f5bc8..b8cd3dc3 100644 --- a/docs/koth-ladder.md +++ b/docs/koth-ladder.md @@ -1,4 +1,12 @@ -# the koth ladder — mine on vouch, through a pull request +# the kit ladder — the 10-minute warm-up lane + +> **start here instead: [mining-on-vouch.md](./mining-on-vouch.md).** +> the main competition is the engine lane +> ([koth-strategy-lane.md](./koth-strategy-lane.md)) — contributors +> submit ranking *code* and the best verified strategy ships as the +> default. this kit ladder is the on-ramp: the same paired scorer over a +> bounded yaml file, useful for learning the loop, but its ceiling is +> low by construction — most single knobs cannot move the bench. ditto onboards contributors by letting miners tweak a retrieval starter kit, scoring the result, and paying whoever holds the throne. the vouch diff --git a/docs/koth-strategy-lane.md b/docs/koth-strategy-lane.md index 3adf35b4..c044790f 100644 --- a/docs/koth-strategy-lane.md +++ b/docs/koth-strategy-lane.md @@ -1,10 +1,19 @@ # the engine lane - submit ranking code, not config -the kit ladder (docs/koth-ladder.md) lets contributors tune coefficients. -this lane is the ditto-equivalent: contributors submit **real engine code** - +**this is the main competition.** (new? start with the walkthrough in +[mining-on-vouch.md](./mining-on-vouch.md).) the kit ladder +(docs/koth-ladder.md) is the warm-up lane for tuning coefficients; this +lane is the ditto-equivalent: contributors submit **real engine code** - a retrieval strategy that decides the order the reader sees candidates in - and the benchmark scores it. it is the place a new fusion, a learned reranker, -or a novel signal actually competes. +or a novel signal actually competes. practice locally with the exact CI +scoring loop: + +```bash +vouch bench run --seeds 1,2,3,4,5,6 \ + --strategy contrib/strategies/mine.py \ + --against contrib/strategies/baseline.py +``` ## why a second lane exists diff --git a/docs/mining-on-vouch.md b/docs/mining-on-vouch.md new file mode 100644 index 00000000..75d51a60 --- /dev/null +++ b/docs/mining-on-vouch.md @@ -0,0 +1,95 @@ +# mining on vouch — from fork to shipped + +vouch pays open contributors to make its retrieval engine better, scored +by a benchmark anyone can reproduce on any machine. no wallet, no subnet, +no tarball uploads: your mining rig is a fork and a text editor, and the +scoreboard is a github workflow. + +the thing you submit is **code** — a ranking strategy the benchmark runs +in a sandbox. and the prize is bigger than the payout: a winning strategy +that passes review ships as the default in the next release. your ranking +code runs on every vouch install. + +## the loop, end to end + +```bash +# 1. fork + clone, then: +pip install -e '.[dev]' + +# 2. see where the money is — the zeros are the levers +vouch bench run --seeds 1,2,3,4,5,6 + +# 3. start from the worked example +cp contrib/strategies/example_lexical.py contrib/strategies/.py + +# 4. practice locally with the EXACT scoring loop CI uses — +# same sandbox, same paired seeds, same margin math +vouch bench run --seeds 1,2,3,4,5,6 \ + --strategy contrib/strategies/.py \ + --against contrib/strategies/baseline.py + +# 5. when it says DETHRONED, open a PR touching only your strategy file +``` + +the `koth-engine-gate` workflow scores your PR against the reigning +champion over the day's seeds and posts the full scorecard as a comment. +win or hold, you see exactly why. + +## what a strategy is + +one function, data in, order out: + +```python +from vouch.strategy import Candidate + +def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]: + ... # return candidate ids, best first +``` + +a `Candidate` is `kind`, `id`, `summary`, `score` — data only. the +sandbox blocks network, subprocesses, and file writes; ordering is +authoritative but bounded (invented ids are ignored, omitted candidates +are appended). see `docs/koth-strategy-lane.md` for the full contract and +sandbox details. + +## how you get paid + +* **daily throne (provisional).** the scorecard names the day's champion; + days-on-throne accrue between dethrones. +* **monthly season (settled).** a sealed commit-reveal run on seeds that + did not exist before the cutoff settles the standings; rank shares are + 65/14/10/7/4 (`docs/vouchbench-seasons.md`). payouts go through a + PR-native bounty platform or github sponsors within a week. +* **flat bounties.** issues labeled `bounty:$X` pay on merge for + benchmark hardening, new categories, and adapters — that ladder is how + the benchmark itself keeps improving. +* **the ledger** lives in `competition/LEADERBOARD.md`; every scorecard's + inputs (seeds, commit, command) are public, so any row can be + recomputed by anyone. + +## the merge rule + +**highest verified score merges.** engine code never auto-merges: the +sandbox stops code from cheating the scorer, and pre-merge human review +stops code that overfits the benchmark — lookup tables, category-pattern +dispatch, generator-template matching are the disqualifiers +(`docs/vouchbench-seasons.md`). review is a veto for cheating, never a +taste test. a clean win merges, ships in the next release, and is +credited in the changelog. + +## warming up: the kit lane + +not ready to write ranking code? the kit ladder +(`docs/koth-ladder.md`) is the 10-minute on-ramp: PR a change to one +bounded yaml file of retrieval knobs and the same paired scorer runs. it +auto-merges on a win because data cannot execute. treat it as the +tutorial — its ceiling is low by construction (most single knobs cannot +move the bench), and the real levers, the benchmark's zero-score +categories, are only reachable from code. + +## why trust the scores + +no TEE, no oracle: the score is a pure function of (seed, code). seeds +derive from the champion sha and the utc date; season seeds come from +public drand randomness at the cutoff. every scorecard can be recomputed +offline with one command. reproducibility is the whole trust model. From 6a992bac78d8502484050ef0199f8e00b8de47d1 Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:17:11 +0900 Subject: [PATCH 08/13] ci(competition): koth-ledger appends the throne row on dethrone every dethrone that lands on the ladder branch now writes its own LEADERBOARD.md row instead of waiting for the maintainer at season close. the scoring gates stay read-only by design; the ledger workflow runs only on push to the ladder branch, after the merge, from base-branch code, so an untrusted pr can never reach its write token. the row comes from the gate's own scorecard comment (comments by any other author are ignored, so a scorecard-shaped comment cannot spoof a row), and update_leaderboard.py is idempotent per pr, so the workflow triggering on its own ledger push converges instead of looping. --- .github/scripts/update_leaderboard.py | 84 ++++++++++++++++++++++++ .github/workflows/koth-ledger.yml | 92 +++++++++++++++++++++++++++ competition/LEADERBOARD.md | 10 +-- tests/test_update_leaderboard.py | 62 ++++++++++++++++++ 4 files changed, 244 insertions(+), 4 deletions(-) create mode 100644 .github/scripts/update_leaderboard.py create mode 100644 .github/workflows/koth-ledger.yml create mode 100644 tests/test_update_leaderboard.py diff --git a/.github/scripts/update_leaderboard.py b/.github/scripts/update_leaderboard.py new file mode 100644 index 00000000..3ee1e9e0 --- /dev/null +++ b/.github/scripts/update_leaderboard.py @@ -0,0 +1,84 @@ +"""Append a dethrone row to competition/LEADERBOARD.md from a scorecard. + +The ledger row was appended by hand at season close; every dethrone that +lands on the ladder branch now writes its own row. Input is the report +json both scorers emit (koth_score.py for kits, score_strategy.py for +strategies — same shape, ``lane`` distinguishes them). + +Idempotent on the PR number: if the ledger already cites the PR, the +script exits 0 without writing, so the ledger workflow re-running on its +own push converges instead of looping. + +Usage: + python update_leaderboard.py --report report.json --pr 123 \ + --author somebody [--ledger competition/LEADERBOARD.md] + +Exit code 0 = row appended or already present, 1 = error. +""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + + +def next_row_number(ledger_text: str) -> int: + rows = re.findall(r"^\| (\d+) \|", ledger_text, flags=re.M) + return max((int(r) for r in rows), default=-1) + 1 + + +def format_row( + number: int, report: dict, pr: int, author: str +) -> str: + lane = report.get("lane", "kit") + date = report.get("date", "") + mean = report["challenger"]["mean"] + margin = report["mean_diff"] + return ( + f"| {number} | {author} ({lane}) | #{pr} | {date} " + f"| {mean:.4f} | +{margin:.4f} |" + ) + + +def append_row(ledger: Path, report: dict, pr: int, author: str) -> bool: + """Add the row unless the PR is already cited. True = file changed.""" + text = ledger.read_text(encoding="utf-8") + if re.search(rf"\| #{pr} \|", text): + return False + row = format_row(next_row_number(text), report, pr, author) + if not text.endswith("\n"): + text += "\n" + # the table is the last block before the payout footer; append right + # after the final table row so the footer prose stays at the bottom + lines = text.splitlines(keepends=True) + last_row = max( + i for i, line in enumerate(lines) if line.startswith("| ") + ) + lines.insert(last_row + 1, row + "\n") + ledger.write_text("".join(lines), encoding="utf-8") + return True + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--report", required=True) + parser.add_argument("--pr", required=True, type=int) + parser.add_argument("--author", required=True) + parser.add_argument( + "--ledger", default="competition/LEADERBOARD.md", + ) + args = parser.parse_args() + + report = json.loads(Path(args.report).read_text(encoding="utf-8")) + if not report.get("dethroned"): + print("report is not a dethrone; nothing to append") + return 0 + changed = append_row(Path(args.ledger), report, args.pr, args.author) + print("row appended" if changed else f"pr #{args.pr} already in ledger") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/koth-ledger.yml b/.github/workflows/koth-ledger.yml new file mode 100644 index 00000000..e025d627 --- /dev/null +++ b/.github/workflows/koth-ledger.yml @@ -0,0 +1,92 @@ +# appends the dethrone row to competition/LEADERBOARD.md after a win lands +# on the ladder branch. the scoring gates stay read-only by design (the +# engine gate holds no write token at all); this workflow never runs on a +# PR event — only on push to the ladder branch, after the merge, from +# base-branch code — so an untrusted PR can never reach its write token. +# idempotent: update_leaderboard.py no-ops when the PR is already cited, +# so the workflow triggering on its own ledger push converges. +name: koth-ledger +on: + push: +permissions: {} +concurrency: + group: koth-ledger-${{ github.ref_name }} + cancel-in-progress: false +jobs: + ledger: + if: >- + vars.KOTH_LADDER_BASE != '' && + github.ref_name == vars.KOTH_LADDER_BASE && + !startsWith(github.event.head_commit.message, 'docs(competition): ledger row') + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: read + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref_name }} + + - name: find the merged pr and its scorecard comment + id: scorecard + env: + GH_TOKEN: ${{ github.token }} + SHA: ${{ github.sha }} + run: | + pr_json="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${SHA}/pulls" \ + --jq '[.[] | select(.merged_at != null)][0] // empty')" + if [ -z "$pr_json" ]; then + echo "no merged pr for ${SHA}; nothing to ledger" + echo "found=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + pr_number="$(printf '%s' "$pr_json" | jq -r '.number')" + pr_author="$(printf '%s' "$pr_json" | jq -r '.user.login')" + # only the gate's own comments are trusted: a scorecard-shaped + # comment from anyone else must never reach the ledger + gh api "repos/${GITHUB_REPOSITORY}/issues/${pr_number}/comments" \ + --paginate \ + --jq '.[] | select(.user.login == "github-actions[bot]") | .body' \ + > /tmp/comments.txt + python3 - <<'PYEOF' + import re + + body = open("/tmp/comments.txt", encoding="utf-8").read() + blocks = re.findall(r"```json\n(.*?)\n```", body, flags=re.S) + reports = [b for b in blocks if '"dethroned": true' in b] + if reports: + with open("/tmp/report.json", "w", encoding="utf-8") as fh: + fh.write(reports[-1]) + PYEOF + if [ ! -f /tmp/report.json ]; then + echo "no gate scorecard with a dethrone on pr ${pr_number}" + echo "found=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "found=true" >> "$GITHUB_OUTPUT" + echo "pr=${pr_number}" >> "$GITHUB_OUTPUT" + echo "author=${pr_author}" >> "$GITHUB_OUTPUT" + + - name: append the ledger row + if: steps.scorecard.outputs.found == 'true' + env: + PR: ${{ steps.scorecard.outputs.pr }} + AUTHOR: ${{ steps.scorecard.outputs.author }} + run: | + python3 .github/scripts/update_leaderboard.py \ + --report /tmp/report.json --pr "$PR" --author "$AUTHOR" + + - name: commit the row + if: steps.scorecard.outputs.found == 'true' + env: + PR: ${{ steps.scorecard.outputs.pr }} + run: | + if git diff --quiet -- competition/LEADERBOARD.md; then + echo "ledger unchanged (row already present)" + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add competition/LEADERBOARD.md + git commit -m "docs(competition): ledger row for #${PR}" + git push diff --git a/competition/LEADERBOARD.md b/competition/LEADERBOARD.md index f6e40405..c9517c78 100644 --- a/competition/LEADERBOARD.md +++ b/competition/LEADERBOARD.md @@ -1,9 +1,11 @@ # koth ladder — throne history -every row is a dethrone: a kit-only PR that beat the reigning kit by the -margin band and auto-merged on the ladder branch. the merged PR is the -authoritative record; this file is the human-readable ledger, appended by -the maintainer when the monthly season closes and payouts are computed. +every row is a dethrone: a challenger (kit or engine strategy) that beat +the reigning champion by the margin band and landed on the ladder branch. +the merged PR is the authoritative record; this file is the human-readable +ledger, appended automatically by the `koth-ledger` workflow from the +gate's scorecard when a win lands (a maintainer can also run +`.github/scripts/update_leaderboard.py` by hand). the daily throne is provisional (public seeds — see docs/koth-ladder.md); payout rank is settled by the monthly sealed commit-reveal run. diff --git a/tests/test_update_leaderboard.py b/tests/test_update_leaderboard.py new file mode 100644 index 00000000..a30685ad --- /dev/null +++ b/tests/test_update_leaderboard.py @@ -0,0 +1,62 @@ +"""The ledger appender — every dethrone that lands on the ladder branch +writes its own LEADERBOARD.md row. These tests guard the row math and the +idempotency that lets the ledger workflow re-trigger on its own push.""" + +import importlib.util +from pathlib import Path + +_SPEC = importlib.util.spec_from_file_location( + "update_leaderboard", + Path(__file__).resolve().parents[1] + / ".github" / "scripts" / "update_leaderboard.py", +) +assert _SPEC and _SPEC.loader +update_leaderboard = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(update_leaderboard) + +BASELINE_ROW = "| 0 | baseline kit (repo defaults) | - | 2026-07-28 | 0.52 | - |" + +LEDGER = f"""# koth ladder — throne history + +prose above the table. + +| # | champion | PR | dethroned on | scored mean | margin over prior | +|---|----------|----|--------------|-------------|-------------------| +{BASELINE_ROW} + +payout footer prose stays at the bottom. +""" + +REPORT = { + "date": "2026-07-29", + "lane": "engine", + "challenger": {"scores": [0.6, 0.62], "mean": 0.61}, + "mean_diff": 0.09, + "dethroned": True, +} + + +def test_next_row_number_continues_the_table() -> None: + assert update_leaderboard.next_row_number(LEDGER) == 1 + assert update_leaderboard.next_row_number("no table yet") == 0 + + +def test_append_row_lands_after_last_row(tmp_path: Path) -> None: + ledger = tmp_path / "LEADERBOARD.md" + ledger.write_text(LEDGER, encoding="utf-8") + changed = update_leaderboard.append_row(ledger, REPORT, 123, "alice-example") + assert changed + lines = ledger.read_text(encoding="utf-8").splitlines() + row = "| 1 | alice-example (engine) | #123 | 2026-07-29 | 0.6100 | +0.0900 |" + assert row in lines + # the new row sits directly under the baseline row, above the footer + assert lines.index(row) == lines.index(BASELINE_ROW) + 1 + + +def test_append_row_is_idempotent_per_pr(tmp_path: Path) -> None: + ledger = tmp_path / "LEADERBOARD.md" + ledger.write_text(LEDGER, encoding="utf-8") + assert update_leaderboard.append_row(ledger, REPORT, 123, "alice-example") + before = ledger.read_text(encoding="utf-8") + assert not update_leaderboard.append_row(ledger, REPORT, 123, "alice-example") + assert ledger.read_text(encoding="utf-8") == before From bc0f12061ab3be506dc95418a7e94de75b58fe1e Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:48:21 +0900 Subject: [PATCH 09/13] ci(competition): ledger sweeps merged prs - auto-merge pushes fire no events round 1 (pr #566) proved the gap live: the gate arms auto-merge with the workflow GITHUB_TOKEN, and github suppresses workflow triggers for pushes caused by that token, so the merge push never reached the ledger. the workflow now sweeps recently merged ladder prs (schedule + manual dispatch + the still-working push path for human merges); the appender stays idempotent per pr, so overlapping sweeps converge. --- .github/workflows/koth-ledger.yml | 99 ++++++++++++++----------------- 1 file changed, 46 insertions(+), 53 deletions(-) diff --git a/.github/workflows/koth-ledger.yml b/.github/workflows/koth-ledger.yml index e025d627..fbd43770 100644 --- a/.github/workflows/koth-ledger.yml +++ b/.github/workflows/koth-ledger.yml @@ -1,23 +1,32 @@ -# appends the dethrone row to competition/LEADERBOARD.md after a win lands -# on the ladder branch. the scoring gates stay read-only by design (the -# engine gate holds no write token at all); this workflow never runs on a -# PR event — only on push to the ladder branch, after the merge, from -# base-branch code — so an untrusted PR can never reach its write token. -# idempotent: update_leaderboard.py no-ops when the PR is already cited, -# so the workflow triggering on its own ledger push converges. +# appends dethrone rows to competition/LEADERBOARD.md for merged ladder +# PRs. the scoring gates stay read-only by design (the engine gate holds +# no write token at all); this workflow never runs on a PR event, so an +# untrusted PR can never reach its write token. +# +# it is a SWEEP, not a per-push hook: the gate arms auto-merge with the +# workflow GITHUB_TOKEN, and github's recursion guard suppresses workflow +# triggers for pushes caused by that token — so an auto-merged dethrone +# never fires a push event here (round 1, PR #566, proved it live). the +# push trigger still catches human-merged rows immediately; the schedule +# and manual dispatch pick up auto-merged ones. update_leaderboard.py is +# idempotent per PR, so overlapping sweeps converge instead of duplicating. name: koth-ledger on: push: + schedule: + - cron: "17 */6 * * *" + workflow_dispatch: permissions: {} concurrency: - group: koth-ledger-${{ github.ref_name }} + group: koth-ledger cancel-in-progress: false jobs: ledger: if: >- vars.KOTH_LADDER_BASE != '' && - github.ref_name == vars.KOTH_LADDER_BASE && - !startsWith(github.event.head_commit.message, 'docs(competition): ledger row') + (github.event_name != 'push' || + (github.ref_name == vars.KOTH_LADDER_BASE && + !startsWith(github.event.head_commit.message, 'docs(competition): ledger row'))) runs-on: ubuntu-latest permissions: contents: write @@ -25,30 +34,27 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: ${{ github.ref_name }} + ref: ${{ vars.KOTH_LADDER_BASE }} - - name: find the merged pr and its scorecard comment - id: scorecard + - name: append rows for recently merged ladder prs env: GH_TOKEN: ${{ github.token }} - SHA: ${{ github.sha }} + LADDER: ${{ vars.KOTH_LADDER_BASE }} run: | - pr_json="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${SHA}/pulls" \ - --jq '[.[] | select(.merged_at != null)][0] // empty')" - if [ -z "$pr_json" ]; then - echo "no merged pr for ${SHA}; nothing to ledger" - echo "found=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - pr_number="$(printf '%s' "$pr_json" | jq -r '.number')" - pr_author="$(printf '%s' "$pr_json" | jq -r '.user.login')" - # only the gate's own comments are trusted: a scorecard-shaped - # comment from anyone else must never reach the ledger - gh api "repos/${GITHUB_REPOSITORY}/issues/${pr_number}/comments" \ - --paginate \ - --jq '.[] | select(.user.login == "github-actions[bot]") | .body' \ - > /tmp/comments.txt - python3 - <<'PYEOF' + gh pr list --repo "$GITHUB_REPOSITORY" --base "$LADDER" \ + --state merged --limit 15 --json number,author \ + --jq '.[] | "\(.number) \(.author.login)"' > /tmp/merged.txt + appended=0 + while read -r pr author; do + [ -n "$pr" ] || continue + # only the gate's own comments are trusted: a scorecard-shaped + # comment from anyone else must never reach the ledger + gh api "repos/${GITHUB_REPOSITORY}/issues/${pr}/comments" \ + --paginate \ + --jq '.[] | select(.user.login == "github-actions[bot]") | .body' \ + > /tmp/comments.txt || continue + rm -f /tmp/report.json + python3 - <<'PYEOF' import re body = open("/tmp/comments.txt", encoding="utf-8").read() @@ -58,35 +64,22 @@ jobs: with open("/tmp/report.json", "w", encoding="utf-8") as fh: fh.write(reports[-1]) PYEOF - if [ ! -f /tmp/report.json ]; then - echo "no gate scorecard with a dethrone on pr ${pr_number}" - echo "found=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - echo "found=true" >> "$GITHUB_OUTPUT" - echo "pr=${pr_number}" >> "$GITHUB_OUTPUT" - echo "author=${pr_author}" >> "$GITHUB_OUTPUT" + if [ -f /tmp/report.json ]; then + python3 .github/scripts/update_leaderboard.py \ + --report /tmp/report.json --pr "$pr" --author "$author" \ + && appended=1 + fi + done < /tmp/merged.txt + echo "sweep done (appended=$appended)" - - name: append the ledger row - if: steps.scorecard.outputs.found == 'true' - env: - PR: ${{ steps.scorecard.outputs.pr }} - AUTHOR: ${{ steps.scorecard.outputs.author }} - run: | - python3 .github/scripts/update_leaderboard.py \ - --report /tmp/report.json --pr "$PR" --author "$AUTHOR" - - - name: commit the row - if: steps.scorecard.outputs.found == 'true' - env: - PR: ${{ steps.scorecard.outputs.pr }} + - name: commit the rows run: | if git diff --quiet -- competition/LEADERBOARD.md; then - echo "ledger unchanged (row already present)" + echo "ledger unchanged" exit 0 fi git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add competition/LEADERBOARD.md - git commit -m "docs(competition): ledger row for #${PR}" + git commit -m "docs(competition): ledger rows from sweep" git push From 1fbfe1a424beeec8d84abd4b81b6e5c9740e4644 Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:52:44 +0900 Subject: [PATCH 10/13] ci(competition): ledger pushes with an owner token past the ruleset the ladder's required gate check applies to direct pushes too, so the bot's row append was rejected (round 1 proved it). protection moved from classic branch protection to the koth-ladder-gate ruleset (same gate requirement, strict), and the ledger checkout uses an owner token with admin bypass. the workflow never runs on pr events and never executes untrusted code, so the token stays out of untrusted reach. --- .github/workflows/koth-ledger.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/koth-ledger.yml b/.github/workflows/koth-ledger.yml index fbd43770..acae6c61 100644 --- a/.github/workflows/koth-ledger.yml +++ b/.github/workflows/koth-ledger.yml @@ -32,9 +32,15 @@ jobs: contents: write pull-requests: read steps: + # the push token: the ladder ruleset requires the gate check on every + # push, and the workflow GITHUB_TOKEN holds no bypass, so rows are + # pushed with an owner token (repo secret). safe because this workflow + # never runs on PR events and never executes untrusted code — the only + # inputs are gate-authored comments parsed as json. - uses: actions/checkout@v4 with: ref: ${{ vars.KOTH_LADDER_BASE }} + token: ${{ secrets.KOTH_LEDGER_TOKEN || github.token }} - name: append rows for recently merged ladder prs env: From 15139124a23bf80a52d24b70356ff1c08ed40565 Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:13:41 +0900 Subject: [PATCH 11/13] feat(retrieval): strategy ranks an over-fetched pool - exclusion is real with a ranking strategy active, retrieval fetched exactly `limit` hits before the strategy ran, so ordering could never change pack membership - and the bench grades presence, not order, which made every strategy score byte-identical to baseline (measured: 12 seeds, diff 0.0000). retrieval now over-fetches a bounded pool (5x limit, floor 50) when a strategy is active and the top `limit` of the strategy's order survive the cut. de-prioritising below the window excludes the candidate; a strategy still cannot fabricate a result or shrink the pack. contract language in the lane doc and baseline docstring updated to match, plus a regression test that a demoted candidate leaves the pack. --- contrib/strategies/baseline.py | 12 ++++++----- docs/koth-strategy-lane.md | 9 ++++++--- src/vouch/context.py | 17 ++++++++++++++-- tests/test_strategy.py | 37 ++++++++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 10 deletions(-) diff --git a/contrib/strategies/baseline.py b/contrib/strategies/baseline.py index e74d7d09..08723892 100644 --- a/contrib/strategies/baseline.py +++ b/contrib/strategies/baseline.py @@ -5,11 +5,13 @@ challenger only dethrones it by making vouch's benchmark score genuinely higher, not by accident. -A strategy is real ranking code. It receives the query and the retrieved -candidates (data only - no KB, no disk, no network) and returns the ids in the -order the reader should see them. Ordering is authoritative but bounded: ids -you invent are ignored, and any candidate you drop is appended at the tail, so -you can reorder and de-prioritise but never fabricate or hide a result. +A strategy is real ranking code. It receives the query and an over-fetched +candidate pool (data only - no KB, no disk, no network) and returns the ids in +the order the reader should see them; the top ``limit`` survive the cut. +Ordering is authoritative but bounded: ids you invent are ignored, and any +candidate you drop is appended at the tail before the cut - so you can +de-prioritise a candidate out of the pack, but never fabricate a result or +shrink the pack. """ from vouch.strategy import Candidate diff --git a/docs/koth-strategy-lane.md b/docs/koth-strategy-lane.md index c044790f..19bb38ab 100644 --- a/docs/koth-strategy-lane.md +++ b/docs/koth-strategy-lane.md @@ -44,9 +44,12 @@ def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]: a `Candidate` is `kind`, `id`, `summary`, `score` - **data only**. the strategy never receives the KB, the filesystem, or a socket. ordering is authoritative but bounded (ids you invent are ignored; candidates you omit are -appended), so a strategy can reorder and de-prioritise but cannot fabricate or -hide a result. see `contrib/strategies/baseline.py` (the champion) and -`example_lexical.py` (a worked example). +appended). with a strategy active, retrieval over-fetches a bounded candidate +pool and the top `limit` of your order survive the cut - so de-prioritising a +candidate below the window genuinely excludes it from the pack. a strategy +still cannot fabricate a result or shrink the pack. see +`contrib/strategies/baseline.py` (the champion) and `example_lexical.py` (a +worked example). ## how a submission is scored diff --git a/src/vouch/context.py b/src/vouch/context.py index a715ca7e..f2e5e1d3 100644 --- a/src/vouch/context.py +++ b/src/vouch/context.py @@ -45,6 +45,12 @@ ContextItemKind = Literal["claim", "page", "entity", "relation", "source"] +# Candidate-pool sizing when a ranking strategy is active: the strategy +# ranks pool candidates and the top ``limit`` survive, so exclusion (not +# just order) is in its hands. Factor/floor keep the pool a shortlist. +_STRATEGY_POOL_FACTOR = 5 +_STRATEGY_POOL_MIN = 50 + _VALID_BACKENDS = ("auto", "hybrid", "embedding", "fts5", "substring") _RERANKER_CACHE: Any | None = None @@ -675,10 +681,17 @@ def build_context_pack( project=project, agent=agent, ) - hits = _retrieve(store, query, limit, viewer) + # with a ranking strategy active, retrieval over-fetches a bounded pool + # and the strategy's order decides which ``limit`` survive the cut — + # de-prioritising a candidate below the window excludes it from the + # pack. without one, the pool IS the limit and nothing changes. the + # pool is bounded so a strategy ranks a shortlist, never the whole kb. + strategy_active = strategy is not None or _configured_strategy(store) + pool = max(limit * _STRATEGY_POOL_FACTOR, _STRATEGY_POOL_MIN) if strategy_active else limit + hits = _retrieve(store, query, pool, viewer) hits = _maybe_strategy( store, query=query, hits=hits, limit=limit, strategy=strategy - ) + )[:limit] items: list[ContextItem] = [] for kind, hid, summary, score, backend in hits: cites: list[str] = [] diff --git a/tests/test_strategy.py b/tests/test_strategy.py index 103ffecf..fb907cc2 100644 --- a/tests/test_strategy.py +++ b/tests/test_strategy.py @@ -200,3 +200,40 @@ def test_shipped_examples_load(path: str) -> None: assert hasattr(strat, "rank") result = strat.rank("beta gamma", CANDS, limit=10) assert sorted(result) == ["a", "b", "c"] + + +def test_strategy_demotion_excludes_from_pack(tmp_path: Path) -> None: + # with a strategy active, retrieval over-fetches a pool and the top + # ``limit`` of the strategy's order survive - de-prioritising below the + # window must EXCLUDE the candidate, not just reorder it. + from vouch import health + from vouch.context import build_context_pack + from vouch.models import Claim + from vouch.storage import KBStore + + store = KBStore.init(tmp_path / "kb") + src = store.put_source(b"raw") + for i in range(4): + store.put_claim(Claim( + id=f"kite-{i}", text=f"kite fact number {i}", evidence=[src.id], + )) + store.put_claim(Claim( + id="kite-noisy", text="kite fact but noisy hearsay", evidence=[src.id], + )) + health.rebuild_index(store) + + class DemoteNoisy: + def rank(self, query, candidates, *, limit): + keep = [c.id for c in candidates if "noisy" not in c.summary] + tail = [c.id for c in candidates if "noisy" in c.summary] + return keep + tail + + pack = dict(build_context_pack( + store, query="kite fact", limit=4, strategy=DemoteNoisy(), + )) + ids = [item["id"] for item in pack["items"]] + assert len(ids) == 4 + assert "kite-noisy" not in ids + + baseline = dict(build_context_pack(store, query="kite fact", limit=4)) + assert len(baseline["items"]) == 4 From 01a7c037e387f68c0da33e6e7fabc9f9d0dbadd7 Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:18:48 +0900 Subject: [PATCH 12/13] docs(contributing): competition pr shapes for both koth lanes open contributors kept needing the lane rules from three scattered docs. contributing.md now states the two pr shapes the gates enforce: engine lane = one new file under contrib/strategies/ against koth-ladder, sandboxed, scored, never auto-merged, human review for benchmark-keyed logic; kit lane = kit.yaml only, auto-merge on dethrone, low ceiling by construction. includes the local ci-exact practice command and the reproducibility and payout pointers. --- CONTRIBUTING.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 33a4bf68..45e5315a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,6 +31,55 @@ touch behavior. path to the capture → approve → compile → recall loop. - Tests, fixtures, CI hardening, and developer-experience improvements. +## Competition PRs (the koth ladders) + +The competition has its own PR shapes, and the gates enforce them +mechanically — a PR that doesn't match is classified as a normal PR and +just gets human review. Full walkthrough: +[docs/mining-on-vouch.md](docs/mining-on-vouch.md). + +**Engine lane — the main competition (submit ranking code):** + +- The PR adds **exactly one new file**: `contrib/strategies/.py` + (letters, digits, underscores). Nothing else in the diff — not + `baseline.py`, not the engine, not tests. +- Base branch: **`koth-ladder`**, not `main`. +- The file implements `rank(query, candidates, *, limit) -> list[str]` + (see [docs/koth-strategy-lane.md](docs/koth-strategy-lane.md)). It runs + in a sandbox: no network, no subprocesses, no file writes, hard + resource limits. A crashing strategy scores as the baseline, silently — + test locally first. +- Practice with the exact CI loop before pushing: + + ```bash + vouch bench run --seeds 1,2,3,4,5,6,7,8,9,10,11,12 \ + --strategy contrib/strategies/.py \ + --against contrib/strategies/baseline.py + ``` + +- The gate scores your PR against the reigning champion over the day's + seeds and posts the scorecard. **Engine code never auto-merges**: the + highest verified score merges after a human checks for benchmark-keyed + logic (lookup tables, category-pattern dispatch, generator-template + matching — instant disqualifiers). A merged winner ships as a default + and is credited in the changelog. + +**Kit lane — the 10-minute warm-up (config only):** + +- The PR touches **exactly one file**: + `competition/kits/current/kit.yaml`. Allowed keys and bounds are + enforced by `.github/scripts/validate_kit.py`. +- Base branch: **`koth-ladder`**. On a dethrone the PR auto-merges (data + cannot execute); against `main` it is scored and commented only. +- Know the ceiling: most kit knobs cannot move the bench — this lane is + for learning the loop, not for winning the season. + +**Both lanes:** seeds derive from the champion sha and the utc date, so +every scorecard is reproducible offline — the comment includes the exact +command. Payouts and season rules: +[docs/vouchbench-seasons.md](docs/vouchbench-seasons.md). Titles follow +the same conventional-commit format as everything else. + ## What we won't merge - Anything that bypasses the review gate from the agent side From 36e4b8fa9c48a7c5ebf7ac1935764d03b5184058 Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:53:26 +0900 Subject: [PATCH 13/13] ci(competition): zizmor annotations and shellcheck note for the gates workflow lint only runs on prs that touch workflows, so the koth gates' pull_request_target findings never surfaced until the engine-gate registration pr woke the linter. both gates now carry the ignore annotation with the justification inline (base-branch code only, pr content handled as data, read-only tokens), and the scorecard comment script declares its literal backticks for shellcheck. --- .github/workflows/koth-engine-gate.yml | 2 +- .github/workflows/koth-gate.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/koth-engine-gate.yml b/.github/workflows/koth-engine-gate.yml index 6eb8a56f..e3f6fc9e 100644 --- a/.github/workflows/koth-engine-gate.yml +++ b/.github/workflows/koth-engine-gate.yml @@ -19,7 +19,7 @@ name: koth-engine-gate on: - pull_request_target: + pull_request_target: # zizmor: ignore[dangerous-triggers] workflow + grader + champion all come from the base branch; the PR's .py runs only inside the sandbox child, and the job token is read-only with no secrets types: [opened, synchronize, reopened, ready_for_review] concurrency: diff --git a/.github/workflows/koth-gate.yml b/.github/workflows/koth-gate.yml index b6e3b243..b13e0172 100644 --- a/.github/workflows/koth-gate.yml +++ b/.github/workflows/koth-gate.yml @@ -17,7 +17,7 @@ name: koth-gate on: - pull_request_target: + pull_request_target: # zizmor: ignore[dangerous-triggers] no PR code ever executes: the kit is fetched as data, validated against a closed-world allowlist, and scored by base-branch code with a read-only checkout types: [opened, synchronize, reopened, ready_for_review] concurrency: @@ -131,6 +131,7 @@ jobs: BASE_REF: ${{ github.event.pull_request.base.ref }} LADDER_BASE: ${{ vars.KOTH_LADDER_BASE }} run: | + # shellcheck disable=SC2016 # the posted markdown carries literal backticks if [ "$BASE_REF" = "$LADDER_BASE" ] && [ -n "$LADDER_BASE" ]; then gate_note="auto-merge on dethrone (provisional ladder branch)" else