diff --git a/.github/scripts/koth_score.py b/.github/scripts/koth_score.py new file mode 100644 index 00000000..a32e8668 --- /dev/null +++ b/.github/scripts/koth_score.py @@ -0,0 +1,108 @@ +"""Paired champion-vs-challenger scoring for the koth ladder. + +Runs vouchbench twice per seed — once with the reigning kit, once with the +challenger kit — over seeds derived from the base sha and the utc date, and +applies the dethrone test from docs/vouchbench-seasons.md: + + dethroned iff mean(challenger - champion) >= max(0.007, 1.96 x SE) + +where SE is the standard error of the per-seed paired differences (common +random numbers: both arms see identical generated sessions per seed, so the +difference cancels seed-to-seed variance). + +Seeds are deterministic for a given (base sha, utc day): anyone can recompute +the day's seeds and reproduce the run locally. Same-day overfitting to known +seeds is bounded by the margin band and settled monthly by the season's +commit-reveal scored run, which uses seeds that do not exist until the +cutoff. + +Usage: + python koth_score.py --champion kit.yaml --challenger kit-pr.yaml \ + --base-sha [--date YYYY-MM-DD] [--out report.json] + +Exit code 0 = dethroned (challenger wins), 3 = held (champion stands), +1 = error. The distinct win/hold codes let the workflow branch without +parsing the report. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +from pathlib import Path + +from vouch import bench + +# 12 paired seeds, not 4: the band is built from the standard error of the +# paired differences, and n=4 makes that SE noisy and its normal-z reading +# optimistic. z=1.96 is the two-sided 95% normal quantile; with a small n +# the paired diffs are not perfectly normal, so the floor below still does +# the real gatekeeping when the SE collapses on a deterministic overfit. +N_SEEDS = 12 +FLOOR = bench.DETHRONE_FLOOR +Z = bench.DETHRONE_Z +# bench sleeps this long between session ingests to spread created_at +# timestamps; keep it small — it is wall-clock, not simulated time. the +# 3600s that lived here briefly would have out-slept the CI job timeout +# many times over. +SESSION_GAP_SECONDS = 2.0 + + +def day_seeds(base_sha: str, date: str, n: int = N_SEEDS) -> list[int]: + """Derive the day's seed list from the champion sha and the utc date.""" + seeds = [] + for i in range(n): + digest = hashlib.sha256(f"{base_sha}:{date}:{i}".encode()).hexdigest() + seeds.append(int(digest[:12], 16)) + return seeds + + +def score(kit_text: str, seeds: list[int]) -> list[float]: + extra = kit_text if kit_text.strip() else None + return [ + bench.run( + s, extra_config=extra, session_gap_seconds=SESSION_GAP_SECONDS + )["composite"] + for s in seeds + ] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--champion", required=True) + parser.add_argument("--challenger", required=True) + parser.add_argument("--base-sha", required=True) + parser.add_argument("--date", default=None) + parser.add_argument("--out", default=None) + args = parser.parse_args() + + date = args.date or dt.datetime.now(dt.UTC).strftime("%Y-%m-%d") + seeds = day_seeds(args.base_sha, date) + + champion_text = Path(args.champion).read_text(encoding="utf-8") + challenger_text = Path(args.challenger).read_text(encoding="utf-8") + + champion_scores = score(champion_text, seeds) + challenger_scores = score(challenger_text, seeds) + verdict = bench.paired_verdict( + champion_scores, challenger_scores, floor=FLOOR, z=Z, + ) + + report = { + "date": date, + "base_sha": args.base_sha, + "seeds": seeds, + "lane": "kit", + **verdict, + } + text = json.dumps(report, indent=1) + print(text) + if args.out: + Path(args.out).write_text(text + "\n", encoding="utf-8") + return 0 if report["dethroned"] else 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/score_strategy.py b/.github/scripts/score_strategy.py new file mode 100644 index 00000000..128993fb --- /dev/null +++ b/.github/scripts/score_strategy.py @@ -0,0 +1,82 @@ +"""Paired champion-vs-challenger scoring for the engine (strategy) lane. + +Same dethrone test as the kit lane (docs/vouchbench-seasons.md), but the arm +is a pluggable ranking strategy instead of a config fragment: each submission +is loaded as an untrusted file and run through vouch.strategy.SandboxProxy, so +the scored code executes only inside the sandbox child. + + dethroned iff mean(challenger - champion) >= max(0.007, 1.96 x SE) + +Exit code 0 = dethroned, 3 = held, 1 = error. Unlike the kit lane, a winning +verdict here does NOT auto-merge: engine code ships only through human review. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +from pathlib import Path + +from vouch import bench +from vouch.strategy import SandboxProxy + +N_SEEDS = 8 +FLOOR = bench.DETHRONE_FLOOR +Z = bench.DETHRONE_Z +SESSION_GAP_SECONDS = 2.0 + + +def day_seeds(base_sha: str, date: str, n: int = N_SEEDS) -> list[int]: + seeds = [] + for i in range(n): + digest = hashlib.sha256(f"{base_sha}:{date}:{i}".encode()).hexdigest() + seeds.append(int(digest[:12], 16)) + return seeds + + +def score(path: str, seeds: list[int]) -> list[float]: + proxy = SandboxProxy(path) + return [ + bench.run(s, strategy=proxy, session_gap_seconds=SESSION_GAP_SECONDS)[ + "composite" + ] + for s in seeds + ] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--champion", required=True) + parser.add_argument("--challenger", required=True) + parser.add_argument("--base-sha", required=True) + parser.add_argument("--date", default=None) + parser.add_argument("--out", default=None) + args = parser.parse_args() + + date = args.date or dt.datetime.now(dt.UTC).strftime("%Y-%m-%d") + seeds = day_seeds(args.base_sha, date) + + champion_scores = score(args.champion, seeds) + challenger_scores = score(args.challenger, seeds) + verdict = bench.paired_verdict( + champion_scores, challenger_scores, floor=FLOOR, z=Z, + ) + + report = { + "date": date, + "base_sha": args.base_sha, + "seeds": seeds, + "lane": "engine", + **verdict, + } + text = json.dumps(report, indent=1) + print(text) + if args.out: + Path(args.out).write_text(text + "\n", encoding="utf-8") + return 0 if report["dethroned"] else 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/update_leaderboard.py b/.github/scripts/update_leaderboard.py new file mode 100644 index 00000000..3ee1e9e0 --- /dev/null +++ b/.github/scripts/update_leaderboard.py @@ -0,0 +1,84 @@ +"""Append a dethrone row to competition/LEADERBOARD.md from a scorecard. + +The ledger row was appended by hand at season close; every dethrone that +lands on the ladder branch now writes its own row. Input is the report +json both scorers emit (koth_score.py for kits, score_strategy.py for +strategies — same shape, ``lane`` distinguishes them). + +Idempotent on the PR number: if the ledger already cites the PR, the +script exits 0 without writing, so the ledger workflow re-running on its +own push converges instead of looping. + +Usage: + python update_leaderboard.py --report report.json --pr 123 \ + --author somebody [--ledger competition/LEADERBOARD.md] + +Exit code 0 = row appended or already present, 1 = error. +""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + + +def next_row_number(ledger_text: str) -> int: + rows = re.findall(r"^\| (\d+) \|", ledger_text, flags=re.M) + return max((int(r) for r in rows), default=-1) + 1 + + +def format_row( + number: int, report: dict, pr: int, author: str +) -> str: + lane = report.get("lane", "kit") + date = report.get("date", "") + mean = report["challenger"]["mean"] + margin = report["mean_diff"] + return ( + f"| {number} | {author} ({lane}) | #{pr} | {date} " + f"| {mean:.4f} | +{margin:.4f} |" + ) + + +def append_row(ledger: Path, report: dict, pr: int, author: str) -> bool: + """Add the row unless the PR is already cited. True = file changed.""" + text = ledger.read_text(encoding="utf-8") + if re.search(rf"\| #{pr} \|", text): + return False + row = format_row(next_row_number(text), report, pr, author) + if not text.endswith("\n"): + text += "\n" + # the table is the last block before the payout footer; append right + # after the final table row so the footer prose stays at the bottom + lines = text.splitlines(keepends=True) + last_row = max( + i for i, line in enumerate(lines) if line.startswith("| ") + ) + lines.insert(last_row + 1, row + "\n") + ledger.write_text("".join(lines), encoding="utf-8") + return True + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--report", required=True) + parser.add_argument("--pr", required=True, type=int) + parser.add_argument("--author", required=True) + parser.add_argument( + "--ledger", default="competition/LEADERBOARD.md", + ) + args = parser.parse_args() + + report = json.loads(Path(args.report).read_text(encoding="utf-8")) + if not report.get("dethroned"): + print("report is not a dethrone; nothing to append") + return 0 + changed = append_row(Path(args.ledger), report, args.pr, args.author) + print("row appended" if changed else f"pr #{args.pr} already in ledger") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/validate_kit.py b/.github/scripts/validate_kit.py new file mode 100644 index 00000000..7edbab58 --- /dev/null +++ b/.github/scripts/validate_kit.py @@ -0,0 +1,107 @@ +"""Validate a ladder kit file against the strict allowlist. + +The kit is the one file an untrusted PR may change, and it is applied as +extra_config inside a write-token CI context — so the surface must be data +only. In particular, config keys that name executables (compile.llm_cmd, +enrich.llm_cmd) must never be reachable from a kit: a kit that could set a +command string would execute it on the runner. The allowlist below is +therefore closed-world: unknown keys are an error, not a warning. + +Usage: python validate_kit.py +Exits 0 and prints the canonical yaml on success; exits 1 with a reason +on any violation. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +import yaml + +MAX_KIT_BYTES = 4096 + +BACKENDS = {"auto", "fts5", "embedding", "hybrid", "substring"} + +# key path -> (type, validator) +BOUNDS: dict[tuple[str, ...], Any] = { + ("retrieval", "backend"): lambda v: isinstance(v, str) and v in BACKENDS, + ("retrieval", "default_limit"): lambda v: ( + isinstance(v, int) and not isinstance(v, bool) and 1 <= v <= 100 + ), + ("retrieval", "prompt_gate", "enabled"): lambda v: isinstance(v, bool), + ("retrieval", "recency", "enabled"): lambda v: isinstance(v, bool), + ("retrieval", "recency", "half_life_days"): lambda v: ( + isinstance(v, (int, float)) + and not isinstance(v, bool) + and 0.01 <= float(v) <= 3650.0 + ), + ("retrieval", "rerank", "enabled"): lambda v: isinstance(v, bool), + ("retrieval", "rerank", "top_k"): lambda v: ( + isinstance(v, int) and not isinstance(v, bool) and 1 <= v <= 500 + ), + ("retrieval", "pages_first", "enabled"): lambda v: isinstance(v, bool), + ("retrieval", "pages_first", "boost"): lambda v: ( + isinstance(v, (int, float)) + and not isinstance(v, bool) + and 0.1 <= float(v) <= 10.0 + ), +} + +ALLOWED_BRANCHES = {path[:i] for path in BOUNDS for i in range(1, len(path))} + + +def _walk(node: Any, path: tuple[str, ...], errors: list[str]) -> None: + if isinstance(node, dict): + for key, value in node.items(): + if not isinstance(key, str): + errors.append(f"non-string key at {'.'.join(path) or ''}") + continue + child = (*path, key) + if child in BOUNDS: + if not BOUNDS[child](value): + errors.append(f"{'.'.join(child)}: value {value!r} out of bounds") + elif child in ALLOWED_BRANCHES: + _walk(value, child, errors) + else: + errors.append(f"{'.'.join(child)}: key not in allowlist") + else: + errors.append(f"{'.'.join(path) or ''}: expected a mapping") + + +def validate(text: str) -> list[str]: + # an empty kit must NOT validate: over the contents-api, a file larger + # than the api's inline limit comes back with empty content, which would + # otherwise decode to "" and pass vacuously — the PR would then be scored + # as engine defaults rather than its real, oversized contents. require a + # real retrieval mapping so "empty" can never mean "champion defaults". + encoded = text.encode("utf-8") + if not encoded.strip(): + return ["empty kit (nothing to score — did the fetch truncate?)"] + if len(encoded) > MAX_KIT_BYTES: + return [f"kit larger than {MAX_KIT_BYTES} bytes"] + try: + data = yaml.safe_load(text) + except yaml.YAMLError as exc: + return [f"not valid yaml: {exc}"] + if not isinstance(data, dict) or "retrieval" not in data: + return ["kit must be a mapping with a top-level 'retrieval' section"] + errors: list[str] = [] + _walk(data, (), errors) + return errors + + +def main() -> int: + text = Path(sys.argv[1]).read_text(encoding="utf-8") + errors = validate(text) + if errors: + for err in errors: + print(f"kit rejected: {err}", file=sys.stderr) + return 1 + print(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/koth-engine-gate.yml b/.github/workflows/koth-engine-gate.yml new file mode 100644 index 00000000..e3f6fc9e --- /dev/null +++ b/.github/workflows/koth-engine-gate.yml @@ -0,0 +1,152 @@ +# koth engine gate - scores strategy (ranking-code) submissions. +# +# the deliberate contrast with koth-gate.yml (the kit lane): a kit is data +# and auto-merges; a STRATEGY is code and NEVER auto-merges. this job has no +# write token, holds no secrets, and merges nothing. it scores the submission +# in a sandbox and posts a scorecard + leaderboard note; a human reviews the +# code and merges it as a new default if it wins. that human review is the +# review gate applied to engine code. +# +# security model: +# - pull_request_target: the workflow, the grader (score_strategy.py), and +# the champion strategy all come from the BASE branch. only the challenger +# .py is read from the PR, and it is executed ONLY inside the sandbox +# child (vouch.strategy.run_sandboxed: rlimits + an audit hook blocking +# network/subprocess/writes). +# - permissions: contents: read only. even a full sandbox escape lands on +# an ephemeral runner with a read-only token and no secrets, and cannot +# merge itself. +name: koth-engine-gate + +on: + pull_request_target: # zizmor: ignore[dangerous-triggers] workflow + grader + champion all come from the base branch; the PR's .py runs only inside the sandbox child, and the job token is read-only with no secrets + types: [opened, synchronize, reopened, ready_for_review] + +concurrency: + group: koth-engine-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + gate: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + pull-requests: write # scorecard comment only - no merge + steps: + - name: checkout base branch (trusted code only) + uses: actions/checkout@v4 + + - name: classify the PR + id: classify + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \ + --paginate --jq '.[].filename' > /tmp/changed.txt + count=$(wc -l < /tmp/changed.txt) + only=$(head -1 /tmp/changed.txt) + case "$only" in + contrib/strategies/baseline.py|contrib/strategies/README.md) only="" ;; + esac + if [ "$count" = "1" ] && \ + printf '%s' "$only" | grep -qE '^contrib/strategies/[A-Za-z0-9_]+\.py$'; then + echo "mode=engine" >> "$GITHUB_OUTPUT" + echo "path=$only" >> "$GITHUB_OUTPUT" + else + echo "mode=normal" >> "$GITHUB_OUTPUT" + fi + + - name: pass through (not a strategy PR) + if: steps.classify.outputs.mode == 'normal' + run: echo "not a single-strategy PR - engine gate does not apply." + + - name: fetch challenger strategy from the PR (as data) + if: steps.classify.outputs.mode == 'engine' + env: + GH_TOKEN: ${{ github.token }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + KIT_PATH: ${{ steps.classify.outputs.path }} + run: | + gh api "repos/${HEAD_REPO}/contents/${KIT_PATH}?ref=${HEAD_SHA}" \ + > /tmp/strat-meta.json + encoding=$(jq -r '.encoding // ""' /tmp/strat-meta.json) + if [ "$encoding" != "base64" ]; then + echo "strategy not returned as an inlined blob (encoding=$encoding)" >&2 + exit 1 + fi + jq -r '.content' /tmp/strat-meta.json | base64 -d > /tmp/challenger.py + if [ ! -s /tmp/challenger.py ]; then + echo "fetched strategy is empty" >&2 + exit 1 + fi + + - name: set up python + if: steps.classify.outputs.mode == 'engine' + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: install vouch (base branch code) + if: steps.classify.outputs.mode == 'engine' + run: python -m pip install -e . + + - name: paired scoring - challenger vs baseline champion (sandboxed) + if: steps.classify.outputs.mode == 'engine' + id: score + env: + BASE_SHA: ${{ github.sha }} + run: | + set +e + python .github/scripts/score_strategy.py \ + --champion contrib/strategies/baseline.py \ + --challenger /tmp/challenger.py \ + --base-sha "$BASE_SHA" \ + --out /tmp/engine-report.json + code=$? + set -e + if [ "$code" = "0" ]; then + echo "verdict=dethroned" >> "$GITHUB_OUTPUT" + elif [ "$code" = "3" ]; then + echo "verdict=held" >> "$GITHUB_OUTPUT" + else + exit "$code" + fi + + - name: post the scorecard + if: steps.classify.outputs.mode == 'engine' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + VERDICT: ${{ steps.score.outputs.verdict }} + run: | + { + echo "koth engine lane - ${VERDICT}" + echo + echo '```json' + cat /tmp/engine-report.json + echo '```' + echo + echo "engine code is NOT auto-merged. a winning strategy earns the" + echo "leaderboard place; a maintainer reviews the code and merges it" + echo "as a new default. the daily result is provisional (public seeds)" + echo "- payout rank is settled by the monthly sealed run." + } > /tmp/comment.md + gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --body-file /tmp/comment.md + + - name: report the verdict as the check result + if: steps.classify.outputs.mode == 'engine' + env: + VERDICT: ${{ steps.score.outputs.verdict }} + run: | + echo "verdict: ${VERDICT}" + # a held challenger is a green, informational result - nothing merges + # here regardless, so the gate never blocks. a scoring error already + # failed the job above. + exit 0 diff --git a/.github/workflows/koth-gate.yml b/.github/workflows/koth-gate.yml new file mode 100644 index 00000000..b13e0172 --- /dev/null +++ b/.github/workflows/koth-gate.yml @@ -0,0 +1,182 @@ +# koth ladder gate - scores kit-only PRs against the reigning kit and +# enables auto-merge on a dethrone. the ditto mining loop, rebuilt on +# github primitives: the bench is the validator, branch protection is the +# chain, auto-merge is the emission. +# +# security model (do not weaken): +# - pull_request_target: this definition and every line of executed code +# come from the BASE branch. PR code is never checked out or executed. +# - the only thing read from the PR is competition/kits/current/kit.yaml, +# fetched over the api as data and schema-validated (closed allowlist) +# before it is passed to the bench as extra_config. +# - a PR qualifies for the ladder only if the kit file is the ONLY file +# it touches. anything else is a normal PR: the gate passes without +# scoring and a human reviews as usual. +# - untrusted strings (branch names, repo names, titles) reach scripts +# via env, never by interpolation into run bodies. +name: koth-gate + +on: + pull_request_target: # zizmor: ignore[dangerous-triggers] no PR code ever executes: the kit is fetched as data, validated against a closed-world allowlist, and scored by base-branch code with a read-only checkout + types: [opened, synchronize, reopened, ready_for_review] + +concurrency: + group: koth-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + gate: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write # auto-merge (squash) needs it + pull-requests: write # enable auto-merge + scorecard comment + steps: + - name: checkout base branch (trusted code only) + uses: actions/checkout@v4 + # no ref: pull_request_target checks out the base branch tip + + - name: classify the PR + id: classify + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \ + --paginate --jq '.[].filename' > /tmp/changed.txt + count=$(wc -l < /tmp/changed.txt) + if [ "$count" = "1" ] && \ + grep -qx 'competition/kits/current/kit.yaml' /tmp/changed.txt; then + echo "mode=ladder" >> "$GITHUB_OUTPUT" + else + echo "mode=normal" >> "$GITHUB_OUTPUT" + fi + + - name: pass through (not a ladder PR) + if: steps.classify.outputs.mode == 'normal' + run: echo "not a kit-only PR - koth gate does not apply, humans review." + + - name: fetch challenger kit from the PR (data only) + if: steps.classify.outputs.mode == 'ladder' + env: + GH_TOKEN: ${{ github.token }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + KIT_PATH: competition/kits/current/kit.yaml + run: | + # the contents api inlines base64 only for files under its size + # limit (~1MB); a larger file returns empty content and a download + # url instead. read size + content together and refuse anything + # that is not a small inlined blob, so an oversized kit can never + # decode to "" and be scored as champion defaults. + gh api "repos/${HEAD_REPO}/contents/${KIT_PATH}?ref=${HEAD_SHA}" \ + > /tmp/kit-meta.json + size=$(jq -r '.size // 0' /tmp/kit-meta.json) + encoding=$(jq -r '.encoding // ""' /tmp/kit-meta.json) + if [ "$encoding" != "base64" ]; then + echo "kit not returned as an inlined base64 blob (encoding=$encoding, size=$size)" >&2 + exit 1 + fi + jq -r '.content' /tmp/kit-meta.json | base64 -d > /tmp/kit-challenger.yaml + if [ ! -s /tmp/kit-challenger.yaml ]; then + echo "fetched kit is empty" >&2 + exit 1 + fi + + - name: set up python + if: steps.classify.outputs.mode == 'ladder' + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: install vouch (base branch code) + if: steps.classify.outputs.mode == 'ladder' + run: python -m pip install -e . + + - name: validate challenger kit against the allowlist + if: steps.classify.outputs.mode == 'ladder' + run: python .github/scripts/validate_kit.py /tmp/kit-challenger.yaml + + - name: paired scoring - challenger vs reigning kit + if: steps.classify.outputs.mode == 'ladder' + id: score + env: + BASE_SHA: ${{ github.sha }} + run: | + set +e + python .github/scripts/koth_score.py \ + --champion competition/kits/current/kit.yaml \ + --challenger /tmp/kit-challenger.yaml \ + --base-sha "$BASE_SHA" \ + --out /tmp/koth-report.json + code=$? + set -e + if [ "$code" = "0" ]; then + echo "verdict=dethroned" >> "$GITHUB_OUTPUT" + elif [ "$code" = "3" ]; then + echo "verdict=held" >> "$GITHUB_OUTPUT" + else + exit "$code" + fi + + - name: post the scorecard + if: steps.classify.outputs.mode == 'ladder' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + VERDICT: ${{ steps.score.outputs.verdict }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + LADDER_BASE: ${{ vars.KOTH_LADDER_BASE }} + run: | + # shellcheck disable=SC2016 # the posted markdown carries literal backticks + if [ "$BASE_REF" = "$LADDER_BASE" ] && [ -n "$LADDER_BASE" ]; then + gate_note="auto-merge on dethrone (provisional ladder branch)" + else + gate_note="scored only - auto-merge is disabled for base '${BASE_REF}'; a maintainer promotes proven champions to shipped defaults by hand" + fi + { + echo "koth ladder - ${VERDICT}" + echo + echo '```json' + cat /tmp/koth-report.json + echo '```' + echo + echo "${gate_note}." + echo + echo "seeds derive from base sha + utc date; rerun locally with" + echo '`python .github/scripts/koth_score.py --date ` to reproduce.' + echo "the daily result is provisional and overfittable - payout rank is" + echo "settled monthly on sealed commit-reveal seeds (docs/vouchbench-seasons.md)." + } > /tmp/comment.md + gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --body-file /tmp/comment.md + + # auto-merge fires ONLY on a dedicated, non-shipped ladder branch + # (repo variable KOTH_LADDER_BASE). the trunk is never auto-written: + # a kit-only PR against main is scored and commented, then a human + # merges. this keeps the review gate load-bearing for everything that + # ships - a beatable benchmark must never be the sole writer to code + # users install. when KOTH_LADDER_BASE is unset, nothing auto-merges. + - name: enable auto-merge on dethrone (ladder branch only) + if: >- + steps.classify.outputs.mode == 'ladder' && + steps.score.outputs.verdict == 'dethroned' && + vars.KOTH_LADDER_BASE != '' && + github.event.pull_request.base.ref == vars.KOTH_LADDER_BASE + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + gh pr merge "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --auto --squash + + - name: hold the throne (challenger did not clear the band) + if: >- + steps.classify.outputs.mode == 'ladder' && + steps.score.outputs.verdict == 'held' + run: | + echo "champion holds - challenger did not clear max(0.007, 1.96 x paired SE)." + exit 1 diff --git a/.github/workflows/koth-ledger.yml b/.github/workflows/koth-ledger.yml new file mode 100644 index 00000000..acae6c61 --- /dev/null +++ b/.github/workflows/koth-ledger.yml @@ -0,0 +1,91 @@ +# appends dethrone rows to competition/LEADERBOARD.md for merged ladder +# PRs. the scoring gates stay read-only by design (the engine gate holds +# no write token at all); this workflow never runs on a PR event, so an +# untrusted PR can never reach its write token. +# +# it is a SWEEP, not a per-push hook: the gate arms auto-merge with the +# workflow GITHUB_TOKEN, and github's recursion guard suppresses workflow +# triggers for pushes caused by that token — so an auto-merged dethrone +# never fires a push event here (round 1, PR #566, proved it live). the +# push trigger still catches human-merged rows immediately; the schedule +# and manual dispatch pick up auto-merged ones. update_leaderboard.py is +# idempotent per PR, so overlapping sweeps converge instead of duplicating. +name: koth-ledger +on: + push: + schedule: + - cron: "17 */6 * * *" + workflow_dispatch: +permissions: {} +concurrency: + group: koth-ledger + cancel-in-progress: false +jobs: + ledger: + if: >- + vars.KOTH_LADDER_BASE != '' && + (github.event_name != 'push' || + (github.ref_name == vars.KOTH_LADDER_BASE && + !startsWith(github.event.head_commit.message, 'docs(competition): ledger row'))) + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: read + steps: + # the push token: the ladder ruleset requires the gate check on every + # push, and the workflow GITHUB_TOKEN holds no bypass, so rows are + # pushed with an owner token (repo secret). safe because this workflow + # never runs on PR events and never executes untrusted code — the only + # inputs are gate-authored comments parsed as json. + - uses: actions/checkout@v4 + with: + ref: ${{ vars.KOTH_LADDER_BASE }} + token: ${{ secrets.KOTH_LEDGER_TOKEN || github.token }} + + - name: append rows for recently merged ladder prs + env: + GH_TOKEN: ${{ github.token }} + LADDER: ${{ vars.KOTH_LADDER_BASE }} + run: | + gh pr list --repo "$GITHUB_REPOSITORY" --base "$LADDER" \ + --state merged --limit 15 --json number,author \ + --jq '.[] | "\(.number) \(.author.login)"' > /tmp/merged.txt + appended=0 + while read -r pr author; do + [ -n "$pr" ] || continue + # only the gate's own comments are trusted: a scorecard-shaped + # comment from anyone else must never reach the ledger + gh api "repos/${GITHUB_REPOSITORY}/issues/${pr}/comments" \ + --paginate \ + --jq '.[] | select(.user.login == "github-actions[bot]") | .body' \ + > /tmp/comments.txt || continue + rm -f /tmp/report.json + python3 - <<'PYEOF' + import re + + body = open("/tmp/comments.txt", encoding="utf-8").read() + blocks = re.findall(r"```json\n(.*?)\n```", body, flags=re.S) + reports = [b for b in blocks if '"dethroned": true' in b] + if reports: + with open("/tmp/report.json", "w", encoding="utf-8") as fh: + fh.write(reports[-1]) + PYEOF + if [ -f /tmp/report.json ]; then + python3 .github/scripts/update_leaderboard.py \ + --report /tmp/report.json --pr "$pr" --author "$author" \ + && appended=1 + fi + done < /tmp/merged.txt + echo "sweep done (appended=$appended)" + + - name: commit the rows + run: | + if git diff --quiet -- competition/LEADERBOARD.md; then + echo "ledger unchanged" + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add competition/LEADERBOARD.md + git commit -m "docs(competition): ledger rows from sweep" + git push diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 33a4bf68..45e5315a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,6 +31,55 @@ touch behavior. path to the capture → approve → compile → recall loop. - Tests, fixtures, CI hardening, and developer-experience improvements. +## Competition PRs (the koth ladders) + +The competition has its own PR shapes, and the gates enforce them +mechanically — a PR that doesn't match is classified as a normal PR and +just gets human review. Full walkthrough: +[docs/mining-on-vouch.md](docs/mining-on-vouch.md). + +**Engine lane — the main competition (submit ranking code):** + +- The PR adds **exactly one new file**: `contrib/strategies/.py` + (letters, digits, underscores). Nothing else in the diff — not + `baseline.py`, not the engine, not tests. +- Base branch: **`koth-ladder`**, not `main`. +- The file implements `rank(query, candidates, *, limit) -> list[str]` + (see [docs/koth-strategy-lane.md](docs/koth-strategy-lane.md)). It runs + in a sandbox: no network, no subprocesses, no file writes, hard + resource limits. A crashing strategy scores as the baseline, silently — + test locally first. +- Practice with the exact CI loop before pushing: + + ```bash + vouch bench run --seeds 1,2,3,4,5,6,7,8,9,10,11,12 \ + --strategy contrib/strategies/.py \ + --against contrib/strategies/baseline.py + ``` + +- The gate scores your PR against the reigning champion over the day's + seeds and posts the scorecard. **Engine code never auto-merges**: the + highest verified score merges after a human checks for benchmark-keyed + logic (lookup tables, category-pattern dispatch, generator-template + matching — instant disqualifiers). A merged winner ships as a default + and is credited in the changelog. + +**Kit lane — the 10-minute warm-up (config only):** + +- The PR touches **exactly one file**: + `competition/kits/current/kit.yaml`. Allowed keys and bounds are + enforced by `.github/scripts/validate_kit.py`. +- Base branch: **`koth-ladder`**. On a dethrone the PR auto-merges (data + cannot execute); against `main` it is scored and commented only. +- Know the ceiling: most kit knobs cannot move the bench — this lane is + for learning the loop, not for winning the season. + +**Both lanes:** seeds derive from the champion sha and the utc date, so +every scorecard is reproducible offline — the comment includes the exact +command. Payouts and season rules: +[docs/vouchbench-seasons.md](docs/vouchbench-seasons.md). Titles follow +the same conventional-commit format as everything else. + ## What we won't merge - Anything that bypasses the review gate from the agent side diff --git a/README.md b/README.md index 4e8ec5f2..4281a20e 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,7 @@ Pending drafts (`proposed/`) and the derived search index (`state.db`) are gitig * `vouch install-mcp ` also wires cursor, codex, zed, windsurf, openclaw and friends ([adapters/](adapters/)) * [vouch webapp](https://github.com/vouchdev/webApp) — the chat-first browser console from the video; [vouch-desktop](https://github.com/vouchdev/vouch-desktop) wraps the same loop as a desktop app * [CONTRIBUTING.md](CONTRIBUTING.md) — development setup and the test gate +* [docs/mining-on-vouch.md](docs/mining-on-vouch.md) — get paid to improve the retrieval engine: submit ranking code, scored by a reproducible benchmark; the best verified strategy ships as the default ## Incubated by Gittensor diff --git a/competition/LEADERBOARD.md b/competition/LEADERBOARD.md new file mode 100644 index 00000000..c9517c78 --- /dev/null +++ b/competition/LEADERBOARD.md @@ -0,0 +1,19 @@ +# koth ladder — throne history + +every row is a dethrone: a challenger (kit or engine strategy) that beat +the reigning champion by the margin band and landed on the ladder branch. +the merged PR is the authoritative record; this file is the human-readable +ledger, appended automatically by the `koth-ledger` workflow from the +gate's scorecard when a win lands (a maintainer can also run +`.github/scripts/update_leaderboard.py` by hand). + +the daily throne is provisional (public seeds — see docs/koth-ladder.md); +payout rank is settled by the monthly sealed commit-reveal run. + +| # | champion | PR | dethroned on | scored mean | margin over prior | +|---|----------|----|--------------|-------------|-------------------| +| 0 | baseline kit (repo defaults) | — | 2026-07-28 | 0.52 ± 0.03 (seeds 1–6) | — | + +payouts follow the season shares in docs/vouchbench-seasons.md +(65/14/10/7/4). days-on-throne accrue between dethrones; the monthly +commit-reveal scored run settles the season standings. diff --git a/competition/kits/current/kit.yaml b/competition/kits/current/kit.yaml new file mode 100644 index 00000000..59fb05dc --- /dev/null +++ b/competition/kits/current/kit.yaml @@ -0,0 +1,28 @@ +# the reigning retrieval kit - vouch's answer to ditto's starter-kit crate. +# ascii only in this file: it crosses tools and locales. +# +# this file is the ONLY thing a ladder PR may change, and it is the WHOLE +# retrieval config of the bench kb: vouchbench writes `review:` plus this +# file verbatim as the throwaway kb's config.yaml. omitting a key does not +# inherit anything - it means the engine's code default for that knob. +# the reigning kit spells out every knob it relies on, and the current +# values reproduce the repo baseline exactly. +# +# dethrone these values by max(0.007, 1.96 * paired SE) over the day's +# seeds and your PR auto-merges on the ladder branch. allowed keys and +# bounds are enforced by .github/scripts/validate_kit.py - anything else +# fails before scoring. +retrieval: + backend: auto + default_limit: 10 + recency: + enabled: true + half_life_days: 90.0 + prompt_gate: + enabled: true + rerank: + enabled: false + top_k: 50 + pages_first: + enabled: false + boost: 1.25 diff --git a/contrib/strategies/README.md b/contrib/strategies/README.md new file mode 100644 index 00000000..fa4abbb8 --- /dev/null +++ b/contrib/strategies/README.md @@ -0,0 +1,58 @@ +# contrib/strategies - the engine-submission lane + +this is the ditto-equivalent lane: you submit **real ranking code**, not a +config file. a strategy decides the order the reader sees retrieved +candidates in - the place where a new fusion, a learned reranker, or a novel +signal actually lives. + +## the contract + +your file exposes a `rank` function (or a `STRATEGY` object with a `.rank` +method): + +```python +from vouch.strategy import Candidate + +def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]: + # return candidate ids, best first + ... +``` + +- a `Candidate` has `kind`, `id`, `summary`, `score` - **data only**. your + code never gets the KB, the filesystem, or the network. +- ordering is authoritative but bounded: ids you invent are ignored, and any + candidate you omit is appended in its original order. you can reorder and + de-prioritise; you cannot fabricate or hide a result. +- [`baseline.py`](./baseline.py) is the reigning champion (returns the + backend order unchanged). [`example_lexical.py`](./example_lexical.py) is a + worked example you can study and beat. + +## how it is scored + +open a PR that touches **only** your new file under `contrib/strategies/`. +the `koth-engine-gate` workflow: + +1. runs your code in a locked-down `python -I` sandbox (resource limits + an + audit hook that blocks network, subprocess, and filesystem writes); +2. scores vouchbench with your strategy vs the reigning champion, paired over + the day's seeds; +3. posts the scorecard and updates the engine leaderboard on a win. + +## the one hard rule: engine code is never auto-merged + +the config (kit) lane auto-merges because a kit cannot execute. **strategy +code can**, and vouch is a library people install - so a winning strategy is +never merged automatically. it earns the leaderboard place and the payout, +and ships only after a human reviews the code and merges it as a new default. +that human review is vouch's version of ditto's tee-plus-deployment gate: the +benchmark decides the *rank*, a person decides what *ships*. + +reproduce any score locally: + +```bash +pip install -e . +python .github/scripts/score_strategy.py \ + --champion contrib/strategies/baseline.py \ + --challenger contrib/strategies/example_lexical.py \ + --base-sha "$(git rev-parse origin/main)" --date "$(date -u +%F)" +``` diff --git a/contrib/strategies/baseline.py b/contrib/strategies/baseline.py new file mode 100644 index 00000000..08723892 --- /dev/null +++ b/contrib/strategies/baseline.py @@ -0,0 +1,21 @@ +"""The reigning engine-lane champion: trust the backend's fused order. + +This is the strategy every submission must beat. It returns the candidates in +exactly the order retrieval handed them over - i.e. it does nothing - so a +challenger only dethrones it by making vouch's benchmark score genuinely +higher, not by accident. + +A strategy is real ranking code. It receives the query and an over-fetched +candidate pool (data only - no KB, no disk, no network) and returns the ids in +the order the reader should see them; the top ``limit`` survive the cut. +Ordering is authoritative but bounded: ids you invent are ignored, and any +candidate you drop is appended at the tail before the cut - so you can +de-prioritise a candidate out of the pack, but never fabricate a result or +shrink the pack. +""" + +from vouch.strategy import Candidate + + +def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]: + return [c.id for c in candidates] diff --git a/contrib/strategies/example_lexical.py b/contrib/strategies/example_lexical.py new file mode 100644 index 00000000..aead101c --- /dev/null +++ b/contrib/strategies/example_lexical.py @@ -0,0 +1,34 @@ +"""A worked example: re-rank by lexical overlap with the query. + +Not necessarily a winner - it exists to show the shape of a real submission. +It boosts candidates whose summary shares more words with the query, blended +with the backend's own score so a strong retrieval signal is not thrown away. + +Copy this file, change the scoring, and open a PR that touches only your new +file under contrib/strategies/. The engine gate scores it in a sandbox against +the reigning champion; you never edit the engine itself. +""" + +import re + +from vouch.strategy import Candidate + +_WORD = re.compile(r"[a-z0-9]+") + + +def _tokens(text: str) -> set[str]: + return set(_WORD.findall(text.lower())) + + +def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]: + q = _tokens(query) + if not q: + return [c.id for c in candidates] + + def blended(c: Candidate) -> float: + overlap = len(q & _tokens(c.summary)) / len(q) + # keep the backend score in the mix; overlap only tips ties and + # rescues a lexically-obvious hit the fusion under-ranked. + return 0.7 * c.score + 0.3 * overlap + + return [c.id for c in sorted(candidates, key=blended, reverse=True)] diff --git a/docs/koth-ladder.md b/docs/koth-ladder.md new file mode 100644 index 00000000..b8cd3dc3 --- /dev/null +++ b/docs/koth-ladder.md @@ -0,0 +1,149 @@ +# the kit ladder — the 10-minute warm-up lane + +> **start here instead: [mining-on-vouch.md](./mining-on-vouch.md).** +> the main competition is the engine lane +> ([koth-strategy-lane.md](./koth-strategy-lane.md)) — contributors +> submit ranking *code* and the best verified strategy ships as the +> default. this kit ladder is the on-ramp: the same paired scorer over a +> bounded yaml file, useful for learning the loop, but its ceiling is +> low by construction — most single knobs cannot move the bench. + +ditto onboards contributors by letting miners tweak a retrieval starter +kit, scoring the result, and paying whoever holds the throne. the vouch +ladder is the same loop rebuilt on github primitives — no wallet, no +subnet, no tarball uploads. your mining rig is a fork and a text editor. + +## the loop + +1. **fork the repo** and edit exactly one file: + [`competition/kits/current/kit.yaml`](../competition/kits/current/kit.yaml) + — the reigning retrieval kit. it is the whole retrieval config of the + benchmark kb: backend, result limit, recency half-life, prompt-gate, + rerank, and pages-first knobs, all bounded by a strict allowlist + (`.github/scripts/validate_kit.py`). it is pure data — a kit carries + no code. +2. **open a PR** that touches only that file, **against the ladder + branch** (see setup below). the `koth-gate` workflow classifies it as + a ladder entry, validates your kit, and runs vouchbench across the + day's seeds — reigning kit vs yours, paired on identical generated + sessions (common random numbers). +3. **dethrone** if your mean improvement clears the band + `max(0.007, 1.96 x paired SE)`. the gate posts the full scorecard as a + PR comment either way. +4. **auto-merge on the ladder branch.** on a dethrone against the ladder + branch the workflow enables github auto-merge; once required checks + are green your PR lands with no human in the loop. your kit is now the + champion every later challenger must beat, and the merge commit is + your permanent, public receipt. +5. **earn.** days-on-throne accrue until someone dethrones you. the daily + ladder is provisional (see "why the daily throne is only practice"); + seasons close monthly with a sealed commit-reveal scored run + (docs/vouchbench-seasons.md), and that is what settles payouts + (shares 65/14/10/7/4). the ledger lives in `competition/LEADERBOARD.md`. + +## reproduce any score locally + +```bash +pip install -e . +python .github/scripts/koth_score.py \ + --champion competition/kits/current/kit.yaml \ + --challenger my-kit.yaml \ + --base-sha "$(git rev-parse origin/main)" \ + --date "$(date -u +%F)" +``` + +seeds derive from the champion sha and the utc date, so every scorecard +in ci can be recomputed by anyone — pass the same `--date` the run used +(it is printed in the scorecard) to reproduce a past result exactly. no +judge model, no hidden eval set, no api key. + +## why the daily throne is only practice + +the daily seeds are a deterministic function of public inputs (the base +sha and the utc date), and the whole scorer is offline-reproducible — by +design, so anyone can audit a score. the cost is that a contributor can +grid-search the knob space against today's exact seeds and submit only a +kit that already wins. the daily ladder therefore behaves like a public +practice leaderboard: fast, transparent, and overfittable. + +that is fine, because **money is never tied to the daily throne.** payout +rank is decided by the monthly season's scored run, whose seeds come from +a commit-reveal drand round at the cutoff and do not exist until entries +are frozen (docs/vouchbench-seasons.md). the daily loop is where you +iterate and get instant feedback; the sealed monthly run is where a kit +has to generalise to seeds nobody could tune against. + +the band (`max(0.007, 1.96 x paired SE)`) still matters daily: it stops a +kit that merely ties the champion from taking the throne, and the 0.007 +floor holds the line when a deterministic overfit collapses the SE toward +zero. + +## why only a data file auto-merges, and only on the ladder branch + +the review gate is vouch's load-bearing invariant, and it stays load +bearing here in two ways: + +- **code never auto-merges.** the ladder surface is a schema-validated + config fragment; the worst a malicious entry can do is score badly. a + merged ladder PR provably contains exactly one bounded data file. +- **the trunk is never auto-written.** auto-merge fires only for PRs + whose base is the dedicated ladder branch (repo variable + `KOTH_LADDER_BASE`). a kit-only PR opened against `main` is scored and + commented, then a human merges — so a beatable benchmark is never the + sole gate to code that users install. promoting the reigning kit into + the shipped starter-config defaults is a periodic, human-reviewed PR. + +engine improvements — new retrieval stages, new signals, bench +extensions — are welcome as normal PRs with human review, and the +maintainer may then expose a new knob in the kit allowlist so the ladder +can tune it. that is the division of labour: humans review capabilities, +the ladder optimises coefficients. miners on ditto never write to ditto's +core either — they submit kits that a validator scores. same boundary, +expressed as a path allowlist and a branch boundary instead of a tarball +contract. + +## anti-cheat, mapped from ditto + +| threat | ditto's answer | the ladder's answer | +|---|---|---| +| overfitting the eval | hidden benchmark versions | conceded for the daily ladder (public seeds) — it is explicitly practice; the monthly settlement uses commit-reveal seeds that do not exist until cutoff, so the paying rank cannot be pre-fit | +| lookup tables | ast minhash scan | impossible by construction: kits carry no code | +| self-reported scores | tee-locked validator | scoring runs from base-branch code under `pull_request_target`; a PR cannot alter the grader, the workflow, or the seeds it is scored with | +| copy the champion + epsilon | first-seen protection | the band forces a real margin over the reigning kit, and a tie loses to the throne | +| oversized / empty kit | — | the fetch refuses any file not returned as a small inlined blob, and the validator rejects an empty or non-`retrieval` kit, so "empty" can never be scored as champion defaults | +| stale wins racing a new champion | — | branch protection in strict mode on the ladder branch: a dethrone that lands invalidates every open challenger's check until it rebases and re-scores against the new champion | +| benchmark as the only gate to shipped code | tee isolation | auto-merge is confined to the ladder branch; the trunk keeps human review, and champions reach shipped defaults only through a human PR | + +## one-time repo setup (maintainer) + +```bash +# 1. create the dedicated ladder branch off the trunk and seed the champion kit +git switch -c ladder origin/main && git push -u origin ladder + +# 2. tell the gate which base is the auto-merge ladder branch +gh variable set KOTH_LADDER_BASE --body ladder + +# 3. allow the auto-merge button repo-wide +gh repo edit --enable-auto-merge + +# 4. protect the ladder branch: gate is a required check, strict mode. +# (json body via --input so booleans stay booleans — a bare -f sends +# the string "true" and the api 422s.) +cat > /tmp/ladder-protection.json <<'JSON' +{ + "required_status_checks": { "strict": true, "contexts": ["gate"] }, + "enforce_admins": false, + "required_pull_request_reviews": null, + "restrictions": null +} +JSON +gh api -X PUT repos/OWNER/REPO/branches/ladder/protection \ + --input /tmp/ladder-protection.json +``` + +strict mode is what closes the race window: after any merge to the ladder +branch, every open ladder PR must update its branch, which re-triggers +scoring against the new champion. `koth-gate` runs on every PR and passes +trivially for non-kit PRs, so requiring it never blocks normal +development. leave `main` protected with your usual human-review rules — +the gate never auto-merges there. diff --git a/docs/koth-strategy-lane.md b/docs/koth-strategy-lane.md new file mode 100644 index 00000000..19bb38ab --- /dev/null +++ b/docs/koth-strategy-lane.md @@ -0,0 +1,108 @@ +# the engine lane - submit ranking code, not config + +**this is the main competition.** (new? start with the walkthrough in +[mining-on-vouch.md](./mining-on-vouch.md).) the kit ladder +(docs/koth-ladder.md) is the warm-up lane for tuning coefficients; this +lane is the ditto-equivalent: contributors submit **real engine code** - +a retrieval strategy that decides the order the reader sees candidates in - +and the benchmark scores it. it is the place a new fusion, a learned reranker, +or a novel signal actually competes. practice locally with the exact CI +scoring loop: + +```bash +vouch bench run --seeds 1,2,3,4,5,6 \ + --strategy contrib/strategies/mine.py \ + --against contrib/strategies/baseline.py +``` + +## why a second lane exists + +a kit can only move dials that already exist, so its ceiling is low - most +single knobs are no-ops on the current bench. real gains come from new ranking +logic. ditto accepts exactly this (miners submit the retrieval harness). the +difference is that ditto is a hosted service scored in a tee, while vouch is a +library people install - so the two lanes draw the trust boundary in different +places: + +| | kit lane | engine lane | +|---|---|---| +| what you submit | `competition/kits/current/kit.yaml` (data) | `contrib/strategies/.py` (code) | +| scored by | vouchbench, config arm | vouchbench, sandboxed strategy arm | +| on a win | **auto-merges** (data cannot execute) | leaderboard + payout; **never auto-merges** | +| how it ships | promoted to defaults by a human PR | reviewed and merged by a human | + +## the interface + +```python +from vouch.strategy import Candidate + +def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]: + # return candidate ids, best first + ... +``` + +a `Candidate` is `kind`, `id`, `summary`, `score` - **data only**. the +strategy never receives the KB, the filesystem, or a socket. ordering is +authoritative but bounded (ids you invent are ignored; candidates you omit are +appended). with a strategy active, retrieval over-fetches a bounded candidate +pool and the top `limit` of your order survive the cut - so de-prioritising a +candidate below the window genuinely excludes it from the pack. a strategy +still cannot fabricate a result or shrink the pack. see +`contrib/strategies/baseline.py` (the champion) and `example_lexical.py` (a +worked example). + +## how a submission is scored + +open a PR touching only your new `contrib/strategies/.py`. the +`koth-engine-gate` workflow runs your code through +`vouch.strategy.run_sandboxed` and scores vouchbench with it, paired against +the reigning champion over the day's seeds, then posts the scorecard. + +### the sandbox + +each `rank` call runs in a fresh `python -I` child that, before importing your +file, installs: + +- **resource limits** (`RLIMIT_CPU`, `RLIMIT_AS`, `RLIMIT_NOFILE`) and a + parent-side wall-clock timeout - a runaway or memory-hungry strategy is + killed, not the run; +- **an audit hook** (`sys.addaudithook`) that refuses network + (`socket.*`/`urllib.*`), process spawning (`subprocess`, `os.exec`/`fork`, + native `ctypes.dlopen`), and filesystem writes (any `open` with write + intent). reads are allowed so numpy and friends still import. + +a crash, a timeout, or a blocked call yields "no reordering" - a broken +strategy simply fails to improve; it cannot take down scoring. + +### the honest boundary + +an in-process python guard cannot stop a determined native-code escape. it is +defence in depth, not the trust root. the real boundary is the same one ditto +relies on: the scoring job runs on an ephemeral CI runner with a **read-only +token and no secrets**, and **engine code is never auto-merged**. the benchmark +decides the *rank*; a human reviewing the diff decides what *ships*. if you +ever score submissions off ephemeral CI, add OS-level isolation (a container, +nsjail, or gVisor) before trusting the audit hook alone. + +## reproduce locally + +```bash +pip install -e . +python .github/scripts/score_strategy.py \ + --champion contrib/strategies/baseline.py \ + --challenger contrib/strategies/example_lexical.py \ + --base-sha "$(git rev-parse origin/main)" --date "$(date -u +%F)" +``` + +## shipping a winner (maintainer) + +1. read the strategy code - it is about to run on every user's machine. +2. move it into `src/vouch/strategies/` and point `retrieval.strategy` (a + dotted import path) at it in the starter config, or wire it as the default. + the in-engine hook loads a shipped strategy in-process (no sandbox - it is + now trusted, reviewed code). +3. record the dethrone in the engine ladder's `LEADERBOARD`. + +the config knob `retrieval.strategy` is how a merged strategy actually changes +retrieval; until one is merged it is unset and retrieval is byte-identical to +today. diff --git a/docs/mining-on-vouch.md b/docs/mining-on-vouch.md new file mode 100644 index 00000000..75d51a60 --- /dev/null +++ b/docs/mining-on-vouch.md @@ -0,0 +1,95 @@ +# mining on vouch — from fork to shipped + +vouch pays open contributors to make its retrieval engine better, scored +by a benchmark anyone can reproduce on any machine. no wallet, no subnet, +no tarball uploads: your mining rig is a fork and a text editor, and the +scoreboard is a github workflow. + +the thing you submit is **code** — a ranking strategy the benchmark runs +in a sandbox. and the prize is bigger than the payout: a winning strategy +that passes review ships as the default in the next release. your ranking +code runs on every vouch install. + +## the loop, end to end + +```bash +# 1. fork + clone, then: +pip install -e '.[dev]' + +# 2. see where the money is — the zeros are the levers +vouch bench run --seeds 1,2,3,4,5,6 + +# 3. start from the worked example +cp contrib/strategies/example_lexical.py contrib/strategies/.py + +# 4. practice locally with the EXACT scoring loop CI uses — +# same sandbox, same paired seeds, same margin math +vouch bench run --seeds 1,2,3,4,5,6 \ + --strategy contrib/strategies/.py \ + --against contrib/strategies/baseline.py + +# 5. when it says DETHRONED, open a PR touching only your strategy file +``` + +the `koth-engine-gate` workflow scores your PR against the reigning +champion over the day's seeds and posts the full scorecard as a comment. +win or hold, you see exactly why. + +## what a strategy is + +one function, data in, order out: + +```python +from vouch.strategy import Candidate + +def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]: + ... # return candidate ids, best first +``` + +a `Candidate` is `kind`, `id`, `summary`, `score` — data only. the +sandbox blocks network, subprocesses, and file writes; ordering is +authoritative but bounded (invented ids are ignored, omitted candidates +are appended). see `docs/koth-strategy-lane.md` for the full contract and +sandbox details. + +## how you get paid + +* **daily throne (provisional).** the scorecard names the day's champion; + days-on-throne accrue between dethrones. +* **monthly season (settled).** a sealed commit-reveal run on seeds that + did not exist before the cutoff settles the standings; rank shares are + 65/14/10/7/4 (`docs/vouchbench-seasons.md`). payouts go through a + PR-native bounty platform or github sponsors within a week. +* **flat bounties.** issues labeled `bounty:$X` pay on merge for + benchmark hardening, new categories, and adapters — that ladder is how + the benchmark itself keeps improving. +* **the ledger** lives in `competition/LEADERBOARD.md`; every scorecard's + inputs (seeds, commit, command) are public, so any row can be + recomputed by anyone. + +## the merge rule + +**highest verified score merges.** engine code never auto-merges: the +sandbox stops code from cheating the scorer, and pre-merge human review +stops code that overfits the benchmark — lookup tables, category-pattern +dispatch, generator-template matching are the disqualifiers +(`docs/vouchbench-seasons.md`). review is a veto for cheating, never a +taste test. a clean win merges, ships in the next release, and is +credited in the changelog. + +## warming up: the kit lane + +not ready to write ranking code? the kit ladder +(`docs/koth-ladder.md`) is the 10-minute on-ramp: PR a change to one +bounded yaml file of retrieval knobs and the same paired scorer runs. it +auto-merges on a win because data cannot execute. treat it as the +tutorial — its ceiling is low by construction (most single knobs cannot +move the bench), and the real levers, the benchmark's zero-score +categories, are only reachable from code. + +## why trust the scores + +no TEE, no oracle: the score is a pure function of (seed, code). seeds +derive from the champion sha and the utc date; season seeds come from +public drand randomness at the cutoff. every scorecard can be recomputed +offline with one command. reproducibility is the whole trust model. diff --git a/docs/vouchbench-seasons.md b/docs/vouchbench-seasons.md index d74f352f..7e61164a 100644 --- a/docs/vouchbench-seasons.md +++ b/docs/vouchbench-seasons.md @@ -29,7 +29,7 @@ machine; that reproducibility is the whole trust model. around `proposals.approve()`, plaintext storage, no baked model deps). Near-identical entries: the earlier-opened PR wins (first-seen). 6. **Pay and merge.** Every entry that beats `main`'s composite by the - margin band — `max(0.007, 1.64 x SE_paired)` over the scored seeds — + margin band — `max(0.007, 1.96 x SE_paired)` over the scored seeds — earns its rank share of the pool (65 / 14 / 10 / 7 / 4 while the field is small). The winner merges and becomes the champion the next season must dethrone. diff --git a/src/vouch/bench.py b/src/vouch/bench.py index 7aca5b4d..11905e8d 100644 --- a/src/vouch/bench.py +++ b/src/vouch/bench.py @@ -22,6 +22,15 @@ rebuilds the index, and retrieves through ``context.build_context_pack``. A score means vouch-as-shipped retrieved it, under the same review-gate invariants as always. +* **Verifiability axes.** Three categories grade the store's receipts and + lifecycle state, not the pack text: citation-correctness (the surfaced + answer must be spelled by a quote whose byte-offset receipt verifies), + receipt-coverage (fraction of surfaced claim items carrying a verifying + receipt), and supersede-hygiene (once an update landed, the stale value + must not survive as a live claim). Recall-only engines cannot score here + by construction — the axis receipts make measurable. The shared-contract + half of a head-to-head lives in ``memory_contract.MemoryContract``, the + five-tool (Ditto-contract) adapter over the same store. No model, no network, no wall-clock dependence: `vouch bench run --seed 7` gives the same number on every machine, which is what makes scores comparable @@ -30,17 +39,21 @@ Reference baseline (update when retrieval changes; the levers are the zeros): ====================== ===================================================== -run ``vouch bench run --seeds 1,2,3,4,5,6`` @ 2026-07-27 -composite 0.57 ± 0.04 (SE) +run ``vouch bench run --seeds 1,2,3,4,5,6`` @ 2026-07-28 +composite 0.52 ± 0.03 (SE) single-session-recall 1.00 — verbatim receipts + FTS: plain recall is won -multi-session 0.83 +multi-session 0.50 knowledge-update 0.00 — superseded value stays in the pack; needs lifecycle-driven supersession, not reranking (recency reorders, dump-guard still zeroes) -point-in-time 1.00 +point-in-time 0.83 decoy-discrimination 0.00 — same-attribute other-person value outranks injection-resistance 0.83 -abstention 0.33 — cross-person leak under lexical match +abstention 0.00 — cross-person leak under lexical match +citation-correctness 1.00 — guard: the surfaced answer is receipt-quoted +receipt-coverage 1.00 — guard: surfaced claims are receipt-backed +supersede-hygiene 0.00 — stale value stays live; the lifecycle lever + (same root cause as knowledge-update's zero) ====================== ===================================================== For calibration only (different benchmarks, not directly comparable): @@ -57,7 +70,8 @@ from pathlib import Path from typing import Any -from .storage import KBStore +from .models import ClaimStatus +from .storage import ArtifactNotFoundError, KBStore BENCH_ACTOR = "vouch-bench" DEFAULT_BUDGET_CHARS = 2000 @@ -74,6 +88,19 @@ "decoy-discrimination", "injection-resistance", "abstention", + # The verifiability axes — graded against the store's receipts and + # lifecycle state, not the pack text alone. A recall-only benchmark + # (DittoBench included) cannot measure any of these: they require the + # engine to carry byte-offset receipts in the first place. + "citation-correctness", + "receipt-coverage", + "supersede-hygiene", +) + +# A superseded, archived, or redacted claim is not a live memory (mirrors +# the set the context pack excludes). +_RETIRED_STATUSES = frozenset( + (ClaimStatus.SUPERSEDED, ClaimStatus.ARCHIVED, ClaimStatus.REDACTED) ) _DECOY_PERSON = "alice-example" @@ -116,6 +143,18 @@ "what is my parking spot?", "which parking spot is mine?", )), + ("test runner", ( + "which test runner do we use?", + "what runner executes the test suite?", + )), + ("backup cadence", ( + "how often do backups run?", + "what is the backup cadence?", + )), + ("standup time", ( + "when is the daily standup?", + "what time is standup?", + )), ) _STATEMENT_TEMPLATES = ( @@ -201,6 +240,10 @@ def _coin_value(rng: random.Random, attr: str) -> str: return f"spot {rng.randrange(11, 99)}{rng.choice('bcdfg')}" if attr == "staging region": return f"{_coin_word(rng, 2)}-{rng.randrange(2, 9)}" + if attr == "backup cadence": + return f"every {rng.randrange(3, 9)} hours" + if attr == "standup time": + return f"{rng.randrange(8, 12)}:{rng.choice(('05', '15', '35', '45'))}" return _coin_word(rng) @@ -293,6 +336,36 @@ def question(phrasings: tuple[str, ...]) -> str: )) cases.append(MemoryCase("abstention", question(asks), None, (decoy_value,))) + # 8. citation-correctness: stated once, but graded on the receipt — the + # surfaced answer must be provably quoted from the source bytes. + attr, asks = attrs.pop() + value = _coin_value(rng, attr) + docs.add(spot(), statement(attr, value)) + cases.append(MemoryCase("citation-correctness", question(asks), value)) + + # 9. receipt-coverage: every claim item in the answering pack must carry + # a verifying receipt. A guard category: stock scores 1.0, and a change + # that starts surfacing unbacked content pays for it here. + attr, asks = attrs.pop() + value = _coin_value(rng, attr) + docs.add(spot(), statement(attr, value)) + cases.append(MemoryCase("receipt-coverage", question(asks), value)) + + # 10. supersede-hygiene: v1 then an update to v2, graded on the store — + # the stale value must not survive as a live claim while a live claim + # holds the current one. The lifecycle lever knowledge-update's zero + # points at, made a scored axis of its own. + attr, asks = attrs.pop() + v1 = _coin_value(rng, attr) + v2 = _coin_value(rng, attr) + while v2 == v1: + v2 = _coin_value(rng, attr) + early = rng.randrange(sessions - 1) + late = rng.randrange(early + 1, sessions) + docs.add(early, statement(attr, v1)) + docs.add(late, rng.choice(_UPDATE_TEMPLATES).format(attr=attr, value=v2)) + cases.append(MemoryCase("supersede-hygiene", question(asks), v2, (v1,))) + # Filler prose in every session, shuffled placement. for idx in range(sessions): for _ in range(rng.randrange(2, 5)): @@ -334,6 +407,101 @@ def grade_case(case: MemoryCase, pack_text: str) -> tuple[float, str | None]: return 0.0, "expected value not surfaced" +def _verified_quotes(store: KBStore, citation_ids: list[str]) -> list[str]: + """Quotes of the citations whose byte-offset receipts verify. + + A bare source-id citation, a dangling id, or a forged span contributes + nothing — only a receipt that verifies by string comparison counts. + """ + from .receipts import verify_receipt + + quotes: list[str] = [] + for cid in citation_ids: + try: + ev = store.get_evidence(cid) + raw = store.read_source_content(ev.source_id) + except (ArtifactNotFoundError, OSError): + continue + if verify_receipt(ev, raw).verified and ev.quote: + quotes.append(ev.quote) + return quotes + + +def _item_citations(item: dict[str, Any]) -> list[str]: + return [str(c) for c in item.get("citations", [])] + + +def grade_citation_correctness( + store: KBStore, case: MemoryCase, pack: dict[str, Any] +) -> tuple[float, str | None]: + """The answer must be receipt-backed, not merely present. + + Some claim item surfacing the expected value has to carry a citation + whose *verified* quote spells that value — proof the answer was quoted + from real source bytes rather than drifted or fabricated en route. + """ + expected = (case.expected or "").lower() + if expected not in _pack_text(pack).lower(): + return 0.0, "expected value not surfaced" + for item in pack.get("items", []): + if item.get("type") != "claim": + continue + if expected not in str(item.get("summary", "")).lower(): + continue + quotes = _verified_quotes(store, _item_citations(item)) + if any(expected in q.lower() for q in quotes): + return 1.0, None + return 0.0, "answer surfaced without a verifying receipt" + + +def grade_receipt_coverage( + store: KBStore, case: MemoryCase, pack: dict[str, Any] +) -> tuple[float, str | None]: + """Fraction of the pack's claim items backed by a verifying receipt. + + Deliberately not gated on the expected value surfacing — recall misses + are priced by the recall categories. This axis measures only that what + the pack *does* surface is mechanically backed; an empty pack surfaces + nothing unbacked and scores full. + """ + claim_items = [ + i for i in pack.get("items", []) if i.get("type") == "claim" + ] + if not claim_items: + return 1.0, None + backed = sum( + 1 for i in claim_items + if _verified_quotes(store, _item_citations(i)) + ) + if backed == len(claim_items): + return 1.0, None + return ( + round(backed / len(claim_items), 4), + f"{len(claim_items) - backed} of {len(claim_items)} claim items " + "lack a verifying receipt", + ) + + +def grade_supersede_hygiene( + store: KBStore, case: MemoryCase +) -> tuple[float, str | None]: + """Graded on the store, not the pack: after ingest the stale value must + not survive as a live claim while a live claim holds the current one — + the KB tells one truth, enforced by lifecycle state.""" + stale = [f.lower() for f in case.forbidden] + current = (case.expected or "").lower() + live = [ + c for c in store.list_claims() if c.status not in _RETIRED_STATUSES + ] + for claim in live: + text = claim.text.lower() + if any(s in text for s in stale): + return 0.0, f"stale value still live in claim {claim.id!r}" + if not any(current in c.text.lower() for c in live): + return 0.0, "no live claim carries the current value" + return 1.0, None + + def _pack_text(pack: dict[str, Any]) -> str: # build_context_pack returns a ContextPack.model_dump() dict (plus # transport extras); the graded surface is what an agent would read. @@ -351,6 +519,7 @@ def run( workdir: Path | None = None, extra_config: str | None = None, session_gap_seconds: float = 0.0, + strategy: Any = None, ) -> dict[str, Any]: """Generate, ingest through the real pipeline, retrieve, and grade. @@ -367,6 +536,11 @@ def run( timestamps carry the sessions' temporal order — the structure a recency-aware arm ranks by. Zero (the default) keeps runs fast; the generated dataset is identical either way. + + ``strategy`` is an optional pluggable ranking strategy (see + ``vouch.strategy``) — the engine-lane arm. For a competition submission + it is a ``SandboxProxy`` wrapping the untrusted file; the same generated + dataset scored with vs without it isolates the strategy's contribution. """ import tempfile import time as time_mod @@ -396,8 +570,17 @@ def run( for case in dataset.cases: pack = build_context_pack( store, query=case.question, limit=limit, max_chars=budget_chars, + strategy=strategy, ) - score, reason = grade_case(case, _pack_text(dict(pack))) + pack_dict = dict(pack) + if case.category == "citation-correctness": + score, reason = grade_citation_correctness(store, case, pack_dict) + elif case.category == "receipt-coverage": + score, reason = grade_receipt_coverage(store, case, pack_dict) + elif case.category == "supersede-hygiene": + score, reason = grade_supersede_hygiene(store, case) + else: + score, reason = grade_case(case, _pack_text(pack_dict)) per_category[case.category].append(score) if reason is not None: failures.append({ @@ -436,6 +619,7 @@ def run_seeds( sessions: int = DEFAULT_SESSIONS, extra_config: str | None = None, session_gap_seconds: float = 0.0, + strategy: Any = None, ) -> dict[str, Any]: """Run several seeds; report mean composite with a standard error. @@ -447,6 +631,7 @@ def run_seeds( run( s, budget_chars=budget_chars, limit=limit, sessions=sessions, extra_config=extra_config, session_gap_seconds=session_gap_seconds, + strategy=strategy, ) for s in seeds ] @@ -475,6 +660,52 @@ def run_seeds( } +# The dethrone test from docs/vouchbench-seasons.md. FLOOR does the real +# gatekeeping when a deterministic overfit collapses the SE to zero; Z is the +# two-sided 95% normal quantile. Both CI scorers and the local --against loop +# call paired_verdict, so the margin math cannot drift between them. +DETHRONE_FLOOR = 0.007 +DETHRONE_Z = 1.96 + + +def paired_verdict( + champion_scores: list[float], + challenger_scores: list[float], + *, + floor: float = DETHRONE_FLOOR, + z: float = DETHRONE_Z, +) -> dict[str, Any]: + """Apply the paired dethrone test to two same-seed score lists. + + dethroned iff mean(challenger - champion) >= max(floor, z * SE) where SE + is the standard error of the per-seed paired differences (common random + numbers cancel seed-to-seed variance). + """ + diffs = [ + c - r for c, r in zip(challenger_scores, champion_scores, strict=True) + ] + mean_diff = statistics.mean(diffs) + se = ( + statistics.stdev(diffs) / (len(diffs) ** 0.5) + if len(diffs) > 1 else 0.0 + ) + band = max(floor, z * se) + return { + "champion": { + "scores": champion_scores, + "mean": statistics.mean(champion_scores), + }, + "challenger": { + "scores": challenger_scores, + "mean": statistics.mean(challenger_scores), + }, + "mean_diff": mean_diff, + "se": se, + "band": band, + "dethroned": mean_diff >= band, + } + + def format_report(report: dict[str, Any]) -> str: """Render one run() report as an aligned text table.""" lines = [ diff --git a/src/vouch/cli.py b/src/vouch/cli.py index aba0730c..129727bb 100644 --- a/src/vouch/cli.py +++ b/src/vouch/cli.py @@ -5401,10 +5401,21 @@ def bench() -> None: ) @click.option("--budget-chars", default=None, type=int, help="Context pack budget.") @click.option("--limit", default=None, type=int, help="Max context items per query.") +@click.option( + "--strategy", "strategy_path", default=None, + type=click.Path(exists=True, dir_okay=False), + help="Score with this ranking strategy file, sandboxed exactly like CI.", +) +@click.option( + "--against", "against_path", default=None, + type=click.Path(exists=True, dir_okay=False), + help="Champion strategy file to pair against; prints the dethrone verdict.", +) @click.option("--json", "as_json", is_flag=True, help="Emit the full report as JSON.") def bench_run( seed: int, seeds: str | None, budget_chars: int | None, - limit: int | None, as_json: bool, + limit: int | None, strategy_path: str | None, against_path: str | None, + as_json: bool, ) -> None: """Score retrieval on a generated dataset. @@ -5412,14 +5423,54 @@ def bench_run( Examples: vouch bench run --seed 7 vouch bench run --seeds 1,2,3,4,5 --json + vouch bench run --seeds 1,2,3 --strategy contrib/strategies/mine.py \\ + --against contrib/strategies/baseline.py """ from . import bench as bench_mod budget = budget_chars if budget_chars is not None else bench_mod.DEFAULT_BUDGET_CHARS top_k = limit if limit is not None else bench_mod.DEFAULT_LIMIT + if against_path and not strategy_path: + raise click.UsageError("--against requires --strategy") + strategy = None + if strategy_path: + from .strategy import SandboxProxy + + strategy = SandboxProxy(strategy_path) + seed_list = ( + [int(s) for s in seeds.replace(" ", "").split(",") if s] + if seeds else [seed] + ) + if against_path: + champion = SandboxProxy(against_path) + champion_scores = [ + bench_mod.run( + s, budget_chars=budget, limit=top_k, strategy=champion, + )["composite"] + for s in seed_list + ] + challenger_scores = [ + bench_mod.run( + s, budget_chars=budget, limit=top_k, strategy=strategy, + )["composite"] + for s in seed_list + ] + verdict = bench_mod.paired_verdict(champion_scores, challenger_scores) + verdict["seeds"] = seed_list + if as_json: + click.echo(json.dumps(verdict, indent=2)) + return + click.echo( + f"challenger {verdict['challenger']['mean']:.4f} " + f"champion {verdict['champion']['mean']:.4f} " + f"diff {verdict['mean_diff']:+.4f} band {verdict['band']:.4f}" + ) + click.echo("DETHRONED" if verdict["dethroned"] else "held") + return if seeds: - seed_list = [int(s) for s in seeds.replace(" ", "").split(",") if s] - report = bench_mod.run_seeds(seed_list, budget_chars=budget, limit=top_k) + report = bench_mod.run_seeds( + seed_list, budget_chars=budget, limit=top_k, strategy=strategy, + ) if as_json: click.echo(json.dumps(report, indent=2)) return @@ -5430,7 +5481,7 @@ def bench_run( for name, mean in report["categories"].items(): click.echo(f" {name:<24} {mean:>5.2f}") return - report = bench_mod.run(seed, budget_chars=budget, limit=top_k) + report = bench_mod.run(seed, budget_chars=budget, limit=top_k, strategy=strategy) if as_json: click.echo(json.dumps(report, indent=2)) return diff --git a/src/vouch/context.py b/src/vouch/context.py index 089701ac..f2e5e1d3 100644 --- a/src/vouch/context.py +++ b/src/vouch/context.py @@ -20,6 +20,7 @@ import yaml from . import graph, hot_memory, index_db, retrieval_events +from . import strategy as strategy_mod from .embeddings.fusion import rrf_fuse from .models import ClaimStatus, ContextItem, ContextPack, ContextQuality from .scoping import ( @@ -44,6 +45,12 @@ ContextItemKind = Literal["claim", "page", "entity", "relation", "source"] +# Candidate-pool sizing when a ranking strategy is active: the strategy +# ranks pool candidates and the top ``limit`` survive, so exclusion (not +# just order) is in its hands. Factor/floor keep the pool a shortlist. +_STRATEGY_POOL_FACTOR = 5 +_STRATEGY_POOL_MIN = 50 + _VALID_BACKENDS = ("auto", "hybrid", "embedding", "fts5", "substring") _RERANKER_CACHE: Any | None = None @@ -308,6 +315,71 @@ def _maybe_rerank( return ordered + hits[window_size:] +def _configured_strategy(store: KBStore) -> str | None: + """Resolve ``retrieval.strategy`` - a dotted import path to a shipped, + human-merged strategy - from config.yaml. Off (None) by default.""" + try: + loaded = yaml.safe_load(store.config_path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError): + return None + if not isinstance(loaded, dict): + return None + retrieval = loaded.get("retrieval") + if not isinstance(retrieval, dict): + return None + dotted = retrieval.get("strategy") + return dotted if isinstance(dotted, str) and dotted else None + + +def _maybe_strategy( + store: KBStore, + *, + query: str, + hits: list[tuple[str, str, str, float, str]], + limit: int, + strategy: strategy_mod.RetrievalStrategy | None = None, +) -> list[tuple[str, str, str, float, str]]: + """Apply a pluggable ranking strategy as the final reorder stage. + + ``strategy`` is passed explicitly by the benchmark (an untrusted + submission, wrapped in a sandbox proxy); otherwise a shipped strategy is + resolved from ``retrieval.strategy`` config and loaded in-process. With + neither, hits pass through byte-identical. A strategy that raises or + returns nothing usable leaves the order untouched - retrieval never fails + because a ranking plugin misbehaved. + """ + strat = strategy + if strat is None: + dotted = _configured_strategy(store) + if not dotted: + return hits + try: + strat = strategy_mod.load_dotted(dotted) + except Exception: + return hits + if not hits: + return hits + # the strategy addresses hits by id; if two hits somehow share one (a + # cross-kind slug collision), a reorder-by-id would be ambiguous, so skip + # the stage rather than risk attaching an order to the wrong artifact. + ids = [h[1] for h in hits] + if len(set(ids)) != len(ids): + return hits + candidates = [ + strategy_mod.Candidate(kind=k, id=i, summary=s, score=sc) + for k, i, s, sc, _b in hits + ] + try: + ordered_ids = strat.rank(query, candidates, limit=limit) + except Exception: + return hits + by_id = {h[1]: h for h in hits} + reordered = strategy_mod.apply_ordering( + list(ordered_ids), [(k, i, s, sc) for k, i, s, sc, _b in hits] + ) + return [by_id[h4[1]] for h4 in reordered] + + def _retrieve( store: KBStore, query: str, @@ -602,13 +674,24 @@ def build_context_pack( graph_depth: int = 1, graph_limit: int = 20, graph_rel_types: list[str] | None = None, + strategy: strategy_mod.RetrievalStrategy | None = None, ) -> ContextPack | dict[str, Any]: viewer = viewer_from( config_path=store.config_path, project=project, agent=agent, ) - hits = _retrieve(store, query, limit, viewer) + # with a ranking strategy active, retrieval over-fetches a bounded pool + # and the strategy's order decides which ``limit`` survive the cut — + # de-prioritising a candidate below the window excludes it from the + # pack. without one, the pool IS the limit and nothing changes. the + # pool is bounded so a strategy ranks a shortlist, never the whole kb. + strategy_active = strategy is not None or _configured_strategy(store) + pool = max(limit * _STRATEGY_POOL_FACTOR, _STRATEGY_POOL_MIN) if strategy_active else limit + hits = _retrieve(store, query, pool, viewer) + hits = _maybe_strategy( + store, query=query, hits=hits, limit=limit, strategy=strategy + )[:limit] items: list[ContextItem] = [] for kind, hid, summary, score, backend in hits: cites: list[str] = [] diff --git a/src/vouch/memory_contract.py b/src/vouch/memory_contract.py new file mode 100644 index 00000000..90ca94b4 --- /dev/null +++ b/src/vouch/memory_contract.py @@ -0,0 +1,147 @@ +"""The five-tool memory contract, implemented over a vouch KB. + +Ditto's mining contract (SN118) scores a harness on exactly five tools: +save a memory, search memories, search subjects, fetch by id, and search +within a subject. This module is vouch's side of that shared contract — a +thin adapter over the real store, so a head-to-head can run both engines +on one task set instead of comparing incomparable benchmark numbers. + +Two properties are non-negotiable and differ from a plain memory store: + +* **Saves go through the review gate.** ``save_memory`` runs the same + receipt-gated capture loop production uses (``extract.ingest_source``). + With ``review.auto_approve_on_receipt`` off, a save files a proposal and + returns nothing durable — the gate is never silently bypassed, even + inside a benchmark harness. +* **Subjects are provenance, not labels.** A memory's subject is the title + of the source it cites; there is no free-floating subject field to + drift from the evidence. Subject search and subject-scoped search both + resolve through citations. +""" + +from __future__ import annotations + +from contextlib import suppress +from dataclasses import dataclass + +from .context import search_kb +from .extract import ingest_source +from .models import Claim, ClaimStatus, Evidence +from .receipts import verify_receipt +from .storage import ArtifactNotFoundError, KBStore + +CONTRACT_TOOLS = ( + "save_memory", + "search_memories", + "search_subjects", + "fetch_by_id", + "search_in_subject", +) + +# Mirrors the retracted set the context pack excludes: a superseded, +# archived, or redacted claim is not a live memory. +_RETRACTED = frozenset( + (ClaimStatus.SUPERSEDED, ClaimStatus.ARCHIVED, ClaimStatus.REDACTED) +) + + +@dataclass(frozen=True) +class MemoryHit: + """One memory as the contract returns it.""" + + id: str + text: str + score: float + subject: str | None + receipt_backed: bool + + +class MemoryContract: + """The five contract tools over one vouch KB.""" + + def __init__(self, store: KBStore, *, actor: str = "memory-contract") -> None: + self.store = store + self.actor = actor + + def save_memory(self, text: str, *, subject: str | None = None) -> list[str]: + """Store one memory; return the ids that became durable. + + An empty list means the receipt gate is off and the memory is + pending human review — filed, not durable. + """ + _, approved = ingest_source( + self.store, text.encode("utf-8"), + proposed_by=self.actor, title=subject, + ) + return [claim.id for claim in approved] + + def search_memories(self, query: str, *, limit: int = 10) -> list[MemoryHit]: + result = search_kb(self.store, query=query, limit=limit) + hits: list[MemoryHit] = [] + for row in result["hits"]: + if row["kind"] != "claim": + continue + try: + claim = self.store.get_claim(str(row["id"])) + except ArtifactNotFoundError: + continue + if claim.status in _RETRACTED: + continue + hits.append(self._hit(claim, score=float(row["score"]))) + return hits + + def search_subjects(self, query: str, *, limit: int = 10) -> list[str]: + needle = query.lower() + subjects = { + subject + for claim in self.store.list_claims() + if claim.status not in _RETRACTED + and (subject := self._subject_of(claim)) is not None + and needle in subject.lower() + } + return sorted(subjects)[:limit] + + def fetch_by_id(self, memory_id: str) -> MemoryHit: + return self._hit(self.store.get_claim(memory_id), score=1.0) + + def search_in_subject( + self, subject: str, query: str, *, limit: int = 10 + ) -> list[MemoryHit]: + wide = self.search_memories(query, limit=max(limit * 5, 25)) + return [hit for hit in wide if hit.subject == subject][:limit] + + def _hit(self, claim: Claim, *, score: float) -> MemoryHit: + return MemoryHit( + id=claim.id, + text=claim.text, + score=score, + subject=self._subject_of(claim), + receipt_backed=any( + verify_receipt( + ev, self.store.read_source_content(ev.source_id) + ).verified + for ev in self._evidence_of(claim) + ), + ) + + def _evidence_of(self, claim: Claim) -> list[Evidence]: + out: list[Evidence] = [] + for cid in claim.evidence: + try: + out.append(self.store.get_evidence(cid)) + except ArtifactNotFoundError: + continue # a bare source-id citation carries no receipt + return out + + def _subject_of(self, claim: Claim) -> str | None: + for cid in claim.evidence: + source_id = cid + with suppress(ArtifactNotFoundError): + source_id = self.store.get_evidence(cid).source_id + try: + title = self.store.get_source(source_id).title + except ArtifactNotFoundError: + continue + if title: + return title + return None diff --git a/src/vouch/strategy.py b/src/vouch/strategy.py new file mode 100644 index 00000000..35d0a090 --- /dev/null +++ b/src/vouch/strategy.py @@ -0,0 +1,356 @@ +"""Pluggable retrieval strategies - the engine-submission lane of the koth +ladder. + +A *strategy* is real ranking code: given the query and the candidate hits a +backend retrieved, it returns the ids in the order the reader should see them. +This is where a contributor competes on algorithm (a new fusion, a learned +reranker, a novel signal) rather than on config coefficients (the kit lane). + +Two ways a strategy runs, with very different trust: + +- **shipped / merged** (trusted): resolved from ``retrieval.strategy`` in + config.yaml as a dotted import path, loaded in-process. It is trusted + because a human reviewed and merged it - the review gate, applied to code. +- **a competition submission** (untrusted): a file under ``contrib/strategies/`` + loaded by the benchmark and run via :func:`run_sandboxed`, which executes it + in a separate ``python -I`` process under resource limits and an audit hook + that blocks network, subprocess, and filesystem writes. + +Honesty about the sandbox boundary: the audit hook + rlimits stop casual and +accidental misbehaviour and raise the bar against a hostile one, but no +in-process Python guard is a true boundary - a determined attacker who can +introspect the interpreter (native ctypes/cffi, or pure-Python gc/frame walking +to reach and neutralise the hook object) can defeat it. The *real* boundary is +the same one ditto leans on: the scoring job runs on an ephemeral CI runner +with a read-only token and no secrets, and **strategy code is never +auto-merged** - it earns a leaderboard place and is shipped only through human +review. So the worst a full escape achieves during scoring is misbehaviour on a +throwaway runner that can reach nothing and merge nothing. The sandbox is +defence in depth; the trust root is the runner's least privilege plus the human +merge gate. +""" + +from __future__ import annotations + +import importlib +import importlib.util +import json +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol, runtime_checkable + +# a retrieval hit as the reorder stages pass it: (kind, id, summary, score). +Hit = tuple[str, str, str, float] + +DEFAULT_TIMEOUT_S = 20.0 +DEFAULT_MEM_MB = 2048 +DEFAULT_CPU_S = 15 + +# audit events refused inside the sandbox child. network, process spawning, +# and native dlopen are blocked outright; filesystem writes are blocked by +# inspecting the open mode (reads - needed to import numpy etc. - are allowed). +_BLOCKED_EXACT = frozenset({ + "os.system", + "os.exec", + "os.fork", + "os.forkpty", + "os.posix_spawn", + "os.spawn", + "subprocess.Popen", + "ctypes.dlopen", + "ctypes.dlsym", + "ctypes.call_function", + "ctypes.cdata", +}) +_BLOCKED_PREFIXES = ("socket.", "urllib.", "ftplib.", "http.") + + +@dataclass(frozen=True) +class Candidate: + """What a strategy sees for one retrieved artifact - data only. + + A strategy never receives the KB, the filesystem, or a network handle; + just these fields, so it cannot reach anything outside the ranking task. + """ + + kind: str + id: str + summary: str + score: float + + +@runtime_checkable +class RetrievalStrategy(Protocol): + """The interface a submission implements. + + ``rank`` returns candidate ids best-first. It is treated as *ordering + only*: ids not in the input are ignored, and any input id the strategy + omits is appended in its original order, so a strategy can reorder and + drop-from-the-top but can never fabricate a result or smuggle one in. + """ + + def rank( + self, query: str, candidates: list[Candidate], *, limit: int + ) -> list[str]: ... + + +def apply_ordering(ordered_ids: list[str], hits: list[Hit]) -> list[Hit]: + """Reorder ``hits`` by ``ordered_ids``, preserving the hit set. + + Mirrors the discipline of the built-in rerank stage: the strategy may move + artifacts but not add or remove them. Unknown ids are dropped; hits the + strategy did not mention keep their relative order at the tail. + """ + by_id: dict[str, Hit] = {} + for hit in hits: + by_id.setdefault(hit[1], hit) + seen: set[str] = set() + ordered: list[Hit] = [] + for hid in ordered_ids: + match = by_id.get(hid) + if match is not None and hid not in seen: + ordered.append(match) + seen.add(hid) + for hit in hits: + if hit[1] not in seen: + ordered.append(hit) + seen.add(hit[1]) + return ordered + + +def load_from_path(path: str | Path) -> RetrievalStrategy: + """Import a strategy from a .py file and return its strategy object. + + The module must expose either a ``STRATEGY`` object with a ``rank`` method + or a module-level ``rank`` function. Loading runs module top-level code, so + only call this on trusted (merged) files or inside the sandbox child. + """ + path = Path(path) + spec = importlib.util.spec_from_file_location(f"vouch_strategy_{path.stem}", path) + if spec is None or spec.loader is None: + raise ValueError(f"cannot load strategy from {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return _strategy_from_module(module) + + +def load_dotted(dotted: str) -> RetrievalStrategy: + """Import a trusted, shipped strategy by dotted module path (config hook).""" + module = importlib.import_module(dotted) + return _strategy_from_module(module) + + +def _strategy_from_module(module: Any) -> RetrievalStrategy: + candidate = getattr(module, "STRATEGY", None) + if candidate is not None and hasattr(candidate, "rank"): + return candidate # type: ignore[no-any-return] + fn = getattr(module, "rank", None) + if callable(fn): + rank_fn = fn + + class _FnStrategy: + def rank( + self, query: str, candidates: list[Candidate], *, limit: int + ) -> list[str]: + return list(rank_fn(query, candidates, limit=limit)) + + return _FnStrategy() + raise ValueError( + "strategy module must define STRATEGY (with .rank) or a rank() function" + ) + + +class SandboxProxy: + """A strategy handle whose ``rank`` runs the real code in a sandbox child. + + Built by the benchmark for an untrusted submission. Each ``rank`` call + spawns a fresh ``python -I`` process, so a submission cannot keep state + between calls or wear down the limits over time. A crash, a timeout, or a + limit breach yields an empty ordering, which :func:`apply_ordering` turns + into "keep the backend's order" - a broken strategy simply fails to + improve rather than taking down the run. + """ + + def __init__( + self, + path: str | Path, + *, + timeout_s: float = DEFAULT_TIMEOUT_S, + mem_mb: int = DEFAULT_MEM_MB, + cpu_s: int = DEFAULT_CPU_S, + ) -> None: + self.path = str(Path(path).resolve()) + self.timeout_s = timeout_s + self.mem_mb = mem_mb + self.cpu_s = cpu_s + self.failures = 0 + + def rank( + self, query: str, candidates: list[Candidate], *, limit: int + ) -> list[str]: + ordered = run_sandboxed( + self.path, + query, + candidates, + limit=limit, + timeout_s=self.timeout_s, + mem_mb=self.mem_mb, + cpu_s=self.cpu_s, + ) + if ordered is None: + self.failures += 1 + return [] + return ordered + + +def run_sandboxed( + path: str, + query: str, + candidates: list[Candidate], + *, + limit: int, + timeout_s: float = DEFAULT_TIMEOUT_S, + mem_mb: int = DEFAULT_MEM_MB, + cpu_s: int = DEFAULT_CPU_S, +) -> list[str] | None: + """Run a strategy file in a locked-down child; return ordered ids or None. + + None means the child failed (crash, timeout, limit, or malformed output) - + the caller treats that as "no reordering", never as an error that aborts + scoring. + """ + payload = { + "path": path, + "query": query, + "limit": limit, + "mem_bytes": mem_mb * 1024 * 1024, + "cpu_seconds": cpu_s, + "candidates": [ + {"kind": c.kind, "id": c.id, "summary": c.summary, "score": c.score} + for c in candidates + ], + } + try: + proc = subprocess.run( + [sys.executable, "-I", "-m", "vouch.strategy", "--child"], + input=json.dumps(payload), + capture_output=True, + text=True, + timeout=timeout_s, + # a minimal env; -I already ignores PYTHON* vars and the cwd. + env={"PATH": "/usr/bin:/bin"}, + ) + except (subprocess.TimeoutExpired, OSError): + return None + if proc.returncode != 0: + return None + try: + result = json.loads(proc.stdout) + ordered = result["ordered"] + except (json.JSONDecodeError, KeyError, TypeError): + return None + if not isinstance(ordered, list): + return None + return [str(x) for x in ordered] + + +# --- sandbox child --------------------------------------------------------- + + +def _install_audit_hook() -> None: + import os + + # Bind the guard sets into closure cells that hold the original frozenset + # objects. The hook reads them via LOAD_DEREF, not from module globals, so + # untrusted top-level code cannot disarm the block by reassigning + # ``__main__._BLOCKED_EXACT`` (the child runs as __main__). This is still + # in-process defence in depth - see the module docstring: a determined + # attacker who introspects the interpreter can defeat any pure-Python + # hook, which is why the real boundary is the ephemeral runner and the + # no-auto-merge rule, not this function. + blocked_exact = _BLOCKED_EXACT + blocked_prefixes = _BLOCKED_PREFIXES + + def _hook(event: str, args: tuple[Any, ...]) -> None: + if event in blocked_exact or event.startswith(blocked_prefixes): + raise PermissionError(f"blocked in strategy sandbox: {event}") + if event == "open": + # args = (path, mode, flags); refuse any write/append/create intent. + mode = args[1] if len(args) > 1 else "" + if isinstance(mode, str) and any(c in mode for c in "wax+"): + raise PermissionError("filesystem writes are blocked in the sandbox") + flags = args[2] if len(args) > 2 else 0 + if isinstance(flags, int) and ( + flags & (os.O_WRONLY | os.O_RDWR | os.O_CREAT | os.O_APPEND) + ): + raise PermissionError("filesystem writes are blocked in the sandbox") + + sys.addaudithook(_hook) + + +def _apply_rlimits(mem_bytes: int, cpu_seconds: int) -> None: + try: + import resource + except ImportError: # non-unix; the audit hook still applies + return + # cpu seconds is a hard ceiling backing the parent's wall-clock timeout; + # nofile caps fd pressure. RLIMIT_AS is deliberately generous - numpy and + # friends reserve large virtual space even at small RSS, and a too-tight + # AS limit kills honest strategies, so wall-clock + cpu are the real caps. + for res, soft in ( + (resource.RLIMIT_CPU, cpu_seconds), + (resource.RLIMIT_AS, mem_bytes), + (resource.RLIMIT_NOFILE, 64), + ): + try: + hard = resource.getrlimit(res)[1] + cap = soft if hard == resource.RLIM_INFINITY else min(soft, hard) + resource.setrlimit(res, (cap, hard)) + except (ValueError, OSError): + pass + + +def _child_main() -> int: + import os + + payload = json.load(sys.stdin) + mem_bytes = int(payload["mem_bytes"]) + cpu_seconds = int(payload["cpu_seconds"]) + path = payload["path"] + query = payload["query"] + limit = int(payload["limit"]) + candidates = [ + Candidate(kind=c["kind"], id=c["id"], summary=c["summary"], score=c["score"]) + for c in payload["candidates"] + ] + # Isolate the result channel from the strategy's stdout: a strategy that + # prints (a debug line, a library banner, a progress bar) must not corrupt + # the JSON the parent parses. Keep a private dup of the real stdout for the + # result, then point fd 1 at /dev/null so anything the strategy writes is + # discarded. Both opens happen before the write-blocking audit hook arms. + result_fd = os.dup(1) + devnull_fd = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull_fd, 1) + # limits and the audit hook are armed BEFORE the untrusted module is loaded. + _apply_rlimits(mem_bytes, cpu_seconds) + _install_audit_hook() + strategy = load_from_path(path) + ordered = strategy.rank(query, candidates, limit=limit) + valid = {c.id for c in candidates} + ordered_ids = [str(x) for x in ordered if str(x) in valid] + os.write(result_fd, json.dumps({"ordered": ordered_ids}).encode("utf-8")) + return 0 + + +def main(argv: list[str] | None = None) -> int: + argv = sys.argv[1:] if argv is None else argv + if argv and argv[0] == "--child": + return _child_main() + print("usage: python -I -m vouch.strategy --child (reads a job on stdin)") + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_bench.py b/tests/test_bench.py index ca59e98b..25554512 100644 --- a/tests/test_bench.py +++ b/tests/test_bench.py @@ -3,12 +3,28 @@ from __future__ import annotations import json +from pathlib import Path +import pytest from click.testing import CliRunner -from vouch import bench +from vouch import bench, health, lifecycle from vouch.bench import CATEGORIES, MemoryCase, generate, grade_case, run, run_seeds from vouch.cli import cli +from vouch.context import build_context_pack +from vouch.extract import ingest_source +from vouch.storage import KBStore + + +def _mini_kb(tmp_path: Path, *sentences: str) -> KBStore: + store = KBStore.init(tmp_path / "kb") + store.config_path.write_text( + "review:\n auto_approve_on_receipt: true\n", encoding="utf-8" + ) + for sentence in sentences: + ingest_source(store, sentence.encode("utf-8"), proposed_by="bench-test") + health.rebuild_index(store) + return store def test_generate_is_deterministic() -> None: @@ -88,3 +104,130 @@ def test_cli_bench_gen_emits_dataset() -> None: assert payload["seed"] == 5 assert len(payload["cases"]) == len(CATEGORIES) assert payload["sessions"] + + +def test_categories_include_verifiability_axes() -> None: + assert "citation-correctness" in CATEGORIES + assert "receipt-coverage" in CATEGORIES + assert "supersede-hygiene" in CATEGORIES + + +def test_grade_citation_correctness_requires_verifying_receipt( + tmp_path: Path, +) -> None: + store = _mini_kb( + tmp_path, "for the record, my favorite editor is zorvex right now." + ) + case = MemoryCase("citation-correctness", "what is my favorite editor?", "zorvex") + pack = dict( + build_context_pack(store, query=case.question, limit=10, max_chars=2000) + ) + assert bench.grade_citation_correctness(store, case, pack)[0] == 1.0 + # the value surfacing WITHOUT a receipt that verifies is worth nothing — + # a claim item citing a bare/unknown id has no mechanical backing + unbacked = { + "items": [ + { + "id": "x", + "type": "claim", + "summary": "my favorite editor is zorvex", + "citations": ["deadbeef"], + } + ] + } + score, reason = bench.grade_citation_correctness(store, case, unbacked) + assert score == 0.0 + assert reason is not None + + +def test_grade_receipt_coverage_full_on_receipt_backed_pack( + tmp_path: Path, +) -> None: + store = _mini_kb(tmp_path, "the project codename is mulopi now.") + case = MemoryCase("receipt-coverage", "what is the project codename?", "mulopi") + pack = dict( + build_context_pack(store, query=case.question, limit=10, max_chars=2000) + ) + assert bench.grade_receipt_coverage(store, case, pack)[0] == 1.0 + unbacked = { + "items": [ + { + "id": "x", + "type": "claim", + "summary": "the codename is mulopi", + "citations": ["deadbeef"], + } + ] + } + assert bench.grade_receipt_coverage(store, case, unbacked)[0] == 0.0 + + +def test_grade_supersede_hygiene_rewards_lifecycle(tmp_path: Path) -> None: + store = _mini_kb( + tmp_path, + "for the record, my deploy day is monday right now.", + "heads up, the deploy day changed to friday this week.", + ) + case = MemoryCase( + "supersede-hygiene", "which day do we deploy?", "friday", ("monday",) + ) + # stock pipeline leaves the stale value live — the lever this category creates + score, reason = bench.grade_supersede_hygiene(store, case) + assert score == 0.0 + assert reason is not None + old = next(c for c in store.list_claims() if "monday" in c.text) + new = next(c for c in store.list_claims() if "friday" in c.text) + lifecycle.supersede(store, old_claim_id=old.id, new_claim_id=new.id, actor="t") + assert bench.grade_supersede_hygiene(store, case)[0] == 1.0 + + +def test_run_verifiability_stock_scores() -> None: + report = run(1, sessions=4) + # guard categories: the receipt path makes these perfect on the stock + # engine; a change that surfaces unbacked content loses points here + assert report["categories"]["receipt-coverage"]["mean"] == 1.0 + assert report["categories"]["citation-correctness"]["mean"] == 1.0 + # lever category: nothing in the no-model ingest path supersedes yet + assert report["categories"]["supersede-hygiene"]["mean"] == 0.0 + + +def test_paired_verdict_floor_gatekeeps_zero_se() -> None: + # identical per-seed diffs collapse the SE; the floor still gates + verdict = bench.paired_verdict([0.5, 0.5, 0.5], [0.505, 0.505, 0.505]) + assert verdict["se"] == 0.0 + assert verdict["band"] == bench.DETHRONE_FLOOR + assert not verdict["dethroned"] + clear = bench.paired_verdict([0.5, 0.5, 0.5], [0.51, 0.51, 0.51]) + assert clear["dethroned"] + + +def test_paired_verdict_requires_same_seed_count() -> None: + with pytest.raises(ValueError): + bench.paired_verdict([0.5, 0.5], [0.5]) + + +def test_run_seeds_threads_strategy( + monkeypatch: pytest.MonkeyPatch, +) -> None: + seen: list[object] = [] + + def fake_run(seed: int, **kwargs: object) -> dict: + seen.append(kwargs.get("strategy")) + return { + "seed": seed, + "composite": 0.5, + "categories": {name: {"mean": 0.5} for name in CATEGORIES}, + } + + monkeypatch.setattr(bench, "run", fake_run) + sentinel = object() + bench.run_seeds([1, 2], strategy=sentinel) + assert seen == [sentinel, sentinel] + + +def test_cli_bench_run_against_requires_strategy(tmp_path: Path) -> None: + champion = tmp_path / "champ.py" + champion.write_text("def rank(query, candidates, *, limit):\n return []\n") + result = CliRunner().invoke(cli, ["bench", "run", "--against", str(champion)]) + assert result.exit_code != 0 + assert "--against requires --strategy" in result.output diff --git a/tests/test_koth_kit.py b/tests/test_koth_kit.py new file mode 100644 index 00000000..e809df00 --- /dev/null +++ b/tests/test_koth_kit.py @@ -0,0 +1,66 @@ +"""The koth kit validator — the closed-world allowlist that lets a kit-only +PR auto-merge without a human. Every test here guards a way an untrusted kit +could smuggle something past the gate.""" + +import importlib.util +from pathlib import Path + +_SPEC = importlib.util.spec_from_file_location( + "validate_kit", + Path(__file__).resolve().parents[1] / ".github" / "scripts" / "validate_kit.py", +) +assert _SPEC and _SPEC.loader +validate_kit = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(validate_kit) +validate = validate_kit.validate + +CHAMPION = ( + Path(__file__).resolve().parents[1] + / "competition" + / "kits" + / "current" + / "kit.yaml" +).read_text(encoding="utf-8") + + +def test_champion_kit_validates() -> None: + assert validate(CHAMPION) == [] + + +def test_empty_kit_is_rejected() -> None: + # the >1MB contents-api bypass: an oversized file returns empty content, + # which must NOT validate as "champion defaults". + assert validate("") != [] + assert validate(" \n \n") != [] + + +def test_kit_without_retrieval_section_is_rejected() -> None: + assert validate("review:\n auto_approve_on_receipt: true\n") != [] + + +def test_command_key_is_rejected() -> None: + # config can name executables (compile.llm_cmd); a kit must never reach one. + errors = validate("retrieval:\n backend: auto\ncompile:\n llm_cmd: 'sh -c evil'\n") + assert any("not in allowlist" in e for e in errors) + + +def test_unknown_retrieval_key_is_rejected() -> None: + errors = validate("retrieval:\n backend: auto\n secret_knob: 1\n") + assert any("not in allowlist" in e for e in errors) + + +def test_out_of_bounds_values_are_rejected() -> None: + assert validate("retrieval:\n backend: nonsense\n") != [] + assert validate("retrieval:\n recency:\n half_life_days: 99999\n") != [] + assert validate("retrieval:\n rerank:\n top_k: 0\n") != [] + assert validate("retrieval:\n pages_first:\n boost: -1\n") != [] + + +def test_bool_is_not_accepted_as_a_number() -> None: + # yaml true is an int subclass in python; a numeric knob must refuse it. + assert validate("retrieval:\n recency:\n half_life_days: true\n") != [] + + +def test_oversized_kit_is_rejected() -> None: + big = "retrieval:\n backend: auto\n" + ("# pad\n" * 2000) + assert any("larger than" in e for e in validate(big)) diff --git a/tests/test_memory_contract.py b/tests/test_memory_contract.py new file mode 100644 index 00000000..7e8c8b3f --- /dev/null +++ b/tests/test_memory_contract.py @@ -0,0 +1,89 @@ +"""The five-tool memory contract: gate-respecting saves, subject-scoped search.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from vouch.memory_contract import CONTRACT_TOOLS, MemoryContract, MemoryHit +from vouch.storage import ArtifactNotFoundError, KBStore + + +def _kb(tmp_path: Path, *, receipt_gate: bool = True) -> KBStore: + # init's starter config already opts into the receipt gate (phase d); + # the gate-off arm must configure it off explicitly. + store = KBStore.init(tmp_path / "kb") + flag = "true" if receipt_gate else "false" + store.config_path.write_text( + f"review:\n auto_approve_on_receipt: {flag}\n", encoding="utf-8" + ) + return store + + +def test_contract_names_the_five_ditto_tools() -> None: + assert CONTRACT_TOOLS == ( + "save_memory", + "search_memories", + "search_subjects", + "fetch_by_id", + "search_in_subject", + ) + + +def test_save_then_search_roundtrip(tmp_path: Path) -> None: + contract = MemoryContract(_kb(tmp_path)) + saved = contract.save_memory( + "for the record, my favorite editor is zorvex right now." + ) + assert saved, "receipt-gated save produced no durable memory" + hits = contract.search_memories("favorite editor") + assert any("zorvex" in h.text for h in hits) + + +def test_save_without_receipt_gate_stays_pending(tmp_path: Path) -> None: + # The review gate is never silently bypassed: with auto-approve off the + # save files a proposal and nothing becomes durable or searchable. + contract = MemoryContract(_kb(tmp_path, receipt_gate=False)) + saved = contract.save_memory("the staging region is vora-3 as of today.") + assert saved == [] + assert contract.search_memories("staging region") == [] + + +def test_fetch_by_id_returns_saved_memory(tmp_path: Path) -> None: + contract = MemoryContract(_kb(tmp_path)) + saved = contract.save_memory("the project codename is mulopi now.") + hit = contract.fetch_by_id(saved[0]) + assert isinstance(hit, MemoryHit) + assert hit.id == saved[0] + assert "mulopi" in hit.text + assert hit.receipt_backed is True + + +def test_fetch_by_id_unknown_raises(tmp_path: Path) -> None: + contract = MemoryContract(_kb(tmp_path)) + with pytest.raises(ArtifactNotFoundError): + contract.fetch_by_id("no-such-memory") + + +def test_search_subjects_matches_saved_subjects(tmp_path: Path) -> None: + contract = MemoryContract(_kb(tmp_path)) + contract.save_memory("my usual coffee order is a flat white.", subject="preferences") + contract.save_memory("the deploy day moved to tuesday.", subject="ops-runbook") + assert contract.search_subjects("prefer") == ["preferences"] + assert contract.search_subjects("runbook") == ["ops-runbook"] + assert contract.search_subjects("r") == ["ops-runbook", "preferences"] + + +def test_search_in_subject_scopes_hits(tmp_path: Path) -> None: + contract = MemoryContract(_kb(tmp_path)) + contract.save_memory( + "the api rate limit is 640 requests per minute.", subject="ops-runbook" + ) + contract.save_memory( + "personal note: my api rate limit worry is overblown.", subject="journal" + ) + hits = contract.search_in_subject("ops-runbook", "api rate limit") + assert hits, "subject-scoped search found nothing" + assert all(h.subject == "ops-runbook" for h in hits) + assert any("640" in h.text for h in hits) diff --git a/tests/test_strategy.py b/tests/test_strategy.py new file mode 100644 index 00000000..fb907cc2 --- /dev/null +++ b/tests/test_strategy.py @@ -0,0 +1,239 @@ +"""The pluggable-strategy engine lane, with the sandbox as the load-bearing +part. Every escape test here guards the boundary that lets vouch score +untrusted ranking code automatically.""" + +from pathlib import Path + +import pytest + +from vouch.strategy import ( + Candidate, + SandboxProxy, + apply_ordering, + load_from_path, + run_sandboxed, +) + +REPO = Path(__file__).resolve().parents[1] +BASELINE = str(REPO / "contrib" / "strategies" / "baseline.py") +EXAMPLE = str(REPO / "contrib" / "strategies" / "example_lexical.py") + +CANDS = [ + Candidate("claim", "a", "alpha beta", 0.9), + Candidate("claim", "b", "gamma delta", 0.5), + Candidate("page", "c", "beta gamma query", 0.3), +] + + +def _write(tmp_path: Path, body: str) -> str: + p = tmp_path / "challenger.py" + p.write_text(body, encoding="utf-8") + return str(p) + + +# --- interface + ordering discipline -------------------------------------- + + +def test_baseline_is_identity() -> None: + strat = load_from_path(BASELINE) + assert strat.rank("beta", CANDS, limit=10) == ["a", "b", "c"] + + +def test_apply_ordering_drops_unknown_and_appends_missing() -> None: + hits = [(c.kind, c.id, c.summary, c.score) for c in CANDS] + out = apply_ordering(["c", "invented", "a"], hits) + # 'invented' dropped, 'b' (unmentioned) appended at the tail. + assert [h[1] for h in out] == ["c", "a", "b"] + + +def test_apply_ordering_cannot_grow_the_set() -> None: + hits = [(c.kind, c.id, c.summary, c.score) for c in CANDS] + out = apply_ordering(["a", "a", "b", "c", "c"], hits) + assert sorted(h[1] for h in out) == ["a", "b", "c"] + + +# --- sandbox happy path ---------------------------------------------------- + + +def test_sandbox_runs_and_matches_in_process() -> None: + assert run_sandboxed(BASELINE, "beta", CANDS, limit=10) == ["a", "b", "c"] + + +def test_sandbox_is_deterministic() -> None: + a = run_sandboxed(EXAMPLE, "beta gamma query", CANDS, limit=10) + b = run_sandboxed(EXAMPLE, "beta gamma query", CANDS, limit=10) + assert a == b and a is not None + + +def test_sandbox_only_returns_known_ids() -> None: + body = """ +from vouch.strategy import Candidate +def rank(query, candidates, *, limit): + return ["a", "ghost", "c"] # 'ghost' is not a candidate +""" + # the child filters to valid ids before returning. + import tempfile + + with tempfile.TemporaryDirectory() as d: + path = _write(Path(d), body) + assert run_sandboxed(path, "q", CANDS, limit=10) == ["a", "c"] + + +# --- sandbox blocks (the security contract) ------------------------------- + + +def test_sandbox_blocks_network(tmp_path: Path) -> None: + body = """ +from vouch.strategy import Candidate +def rank(query, candidates, *, limit): + import socket + socket.socket().connect(("1.1.1.1", 80)) + return [c.id for c in candidates] +""" + assert run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10) is None + + +def test_sandbox_blocks_file_write(tmp_path: Path) -> None: + target = tmp_path / "pwned" + body = f""" +from vouch.strategy import Candidate +def rank(query, candidates, *, limit): + open({str(target)!r}, "w").write("x") + return [c.id for c in candidates] +""" + assert run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10) is None + assert not target.exists() + + +def test_sandbox_blocks_subprocess(tmp_path: Path) -> None: + body = """ +from vouch.strategy import Candidate +def rank(query, candidates, *, limit): + import subprocess + subprocess.Popen(["/bin/echo", "hi"]) + return [c.id for c in candidates] +""" + assert run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10) is None + + +def test_sandbox_kills_infinite_loop(tmp_path: Path) -> None: + body = """ +from vouch.strategy import Candidate +def rank(query, candidates, *, limit): + while True: + pass +""" + assert ( + run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10, timeout_s=5) + is None + ) + + +def test_audit_hook_survives_module_global_disarm(tmp_path: Path) -> None: + # the child runs as __main__; reassigning the guard globals must NOT + # re-enable network/exec, because the hook reads them from closure cells + # holding the original frozensets. + body = """ +import sys +_m = sys.modules["__main__"] +_m._BLOCKED_EXACT = frozenset() +_m._BLOCKED_PREFIXES = () +def rank(query, candidates, *, limit): + import socket + socket.socket().connect(("1.1.1.1", 80)) + return [c.id for c in candidates] +""" + assert run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10) is None + + +def test_strategy_stdout_does_not_corrupt_result(tmp_path: Path) -> None: + # a strategy that prints debug output must still have its reordering + # respected - the result travels on a channel the strategy cannot dirty. + body = """ +import sys +def rank(query, candidates, *, limit): + print("noisy debug line") + sys.stdout.write("more noise\\n") + return list(reversed([c.id for c in candidates])) +""" + assert run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10) == [ + "c", + "b", + "a", + ] + + +def test_sandbox_survives_a_crashing_strategy(tmp_path: Path) -> None: + body = """ +def rank(query, candidates, *, limit): + raise RuntimeError("boom") +""" + assert run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10) is None + + +def test_proxy_counts_failures_and_returns_empty(tmp_path: Path) -> None: + body = """ +def rank(query, candidates, *, limit): + raise RuntimeError("boom") +""" + proxy = SandboxProxy(_write(tmp_path, body)) + assert proxy.rank("q", CANDS, limit=10) == [] + assert proxy.failures == 1 + + +# --- retrieval integration ------------------------------------------------- + + +def test_build_context_pack_strategy_none_is_default(tmp_path: Path) -> None: + # strategy=None must not change retrieval - exercised in bench parity, but + # assert the config hook returns None cleanly on a bare KB here. + from vouch.context import _configured_strategy + from vouch.storage import KBStore + + store = KBStore.init(tmp_path / "kb") + assert _configured_strategy(store) is None + + +@pytest.mark.parametrize("path", [BASELINE, EXAMPLE]) +def test_shipped_examples_load(path: str) -> None: + strat = load_from_path(path) + assert hasattr(strat, "rank") + result = strat.rank("beta gamma", CANDS, limit=10) + assert sorted(result) == ["a", "b", "c"] + + +def test_strategy_demotion_excludes_from_pack(tmp_path: Path) -> None: + # with a strategy active, retrieval over-fetches a pool and the top + # ``limit`` of the strategy's order survive - de-prioritising below the + # window must EXCLUDE the candidate, not just reorder it. + from vouch import health + from vouch.context import build_context_pack + from vouch.models import Claim + from vouch.storage import KBStore + + store = KBStore.init(tmp_path / "kb") + src = store.put_source(b"raw") + for i in range(4): + store.put_claim(Claim( + id=f"kite-{i}", text=f"kite fact number {i}", evidence=[src.id], + )) + store.put_claim(Claim( + id="kite-noisy", text="kite fact but noisy hearsay", evidence=[src.id], + )) + health.rebuild_index(store) + + class DemoteNoisy: + def rank(self, query, candidates, *, limit): + keep = [c.id for c in candidates if "noisy" not in c.summary] + tail = [c.id for c in candidates if "noisy" in c.summary] + return keep + tail + + pack = dict(build_context_pack( + store, query="kite fact", limit=4, strategy=DemoteNoisy(), + )) + ids = [item["id"] for item in pack["items"]] + assert len(ids) == 4 + assert "kite-noisy" not in ids + + baseline = dict(build_context_pack(store, query="kite fact", limit=4)) + assert len(baseline["items"]) == 4 diff --git a/tests/test_update_leaderboard.py b/tests/test_update_leaderboard.py new file mode 100644 index 00000000..a30685ad --- /dev/null +++ b/tests/test_update_leaderboard.py @@ -0,0 +1,62 @@ +"""The ledger appender — every dethrone that lands on the ladder branch +writes its own LEADERBOARD.md row. These tests guard the row math and the +idempotency that lets the ledger workflow re-trigger on its own push.""" + +import importlib.util +from pathlib import Path + +_SPEC = importlib.util.spec_from_file_location( + "update_leaderboard", + Path(__file__).resolve().parents[1] + / ".github" / "scripts" / "update_leaderboard.py", +) +assert _SPEC and _SPEC.loader +update_leaderboard = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(update_leaderboard) + +BASELINE_ROW = "| 0 | baseline kit (repo defaults) | - | 2026-07-28 | 0.52 | - |" + +LEDGER = f"""# koth ladder — throne history + +prose above the table. + +| # | champion | PR | dethroned on | scored mean | margin over prior | +|---|----------|----|--------------|-------------|-------------------| +{BASELINE_ROW} + +payout footer prose stays at the bottom. +""" + +REPORT = { + "date": "2026-07-29", + "lane": "engine", + "challenger": {"scores": [0.6, 0.62], "mean": 0.61}, + "mean_diff": 0.09, + "dethroned": True, +} + + +def test_next_row_number_continues_the_table() -> None: + assert update_leaderboard.next_row_number(LEDGER) == 1 + assert update_leaderboard.next_row_number("no table yet") == 0 + + +def test_append_row_lands_after_last_row(tmp_path: Path) -> None: + ledger = tmp_path / "LEADERBOARD.md" + ledger.write_text(LEDGER, encoding="utf-8") + changed = update_leaderboard.append_row(ledger, REPORT, 123, "alice-example") + assert changed + lines = ledger.read_text(encoding="utf-8").splitlines() + row = "| 1 | alice-example (engine) | #123 | 2026-07-29 | 0.6100 | +0.0900 |" + assert row in lines + # the new row sits directly under the baseline row, above the footer + assert lines.index(row) == lines.index(BASELINE_ROW) + 1 + + +def test_append_row_is_idempotent_per_pr(tmp_path: Path) -> None: + ledger = tmp_path / "LEADERBOARD.md" + ledger.write_text(LEDGER, encoding="utf-8") + assert update_leaderboard.append_row(ledger, REPORT, 123, "alice-example") + before = ledger.read_text(encoding="utf-8") + assert not update_leaderboard.append_row(ledger, REPORT, 123, "alice-example") + assert ledger.read_text(encoding="utf-8") == before