diff --git a/.github/scripts/score_strategy.py b/.github/scripts/score_strategy.py index 128993fb..f49e5570 100644 --- a/.github/scripts/score_strategy.py +++ b/.github/scripts/score_strategy.py @@ -7,8 +7,10 @@ dethroned iff mean(challenger - champion) >= max(0.007, 1.96 x SE) -Exit code 0 = dethroned, 3 = held, 1 = error. Unlike the kit lane, a winning -verdict here does NOT auto-merge: engine code ships only through human review. +Exit code 0 = dethroned, 3 = held, 1 = error. A dethrone on the ladder +branch arms auto-merge into the quarantined contrib lane (merged strategies +only ever execute inside the sandbox); promotion into src/vouch as trusted +default code still ships through a human-reviewed PR. """ from __future__ import annotations diff --git a/.github/workflows/koth-engine-gate.yml b/.github/workflows/koth-engine-gate.yml index db682017..394c075e 100644 --- a/.github/workflows/koth-engine-gate.yml +++ b/.github/workflows/koth-engine-gate.yml @@ -1,21 +1,31 @@ -# koth engine gate - scores strategy (ranking-code) submissions. +# koth engine gate - scores strategy (ranking-code) submissions and, on a +# dethrone against the ladder branch, arms auto-merge: the benchmark decides +# what enters the quarantined contrib lane, with no human in the loop. # -# the deliberate contrast with koth-gate.yml (the kit lane): a kit is data -# and auto-merges; a STRATEGY is code and NEVER auto-merges. this job has no -# write token, holds no secrets, and merges nothing. it scores the submission -# in a sandbox and posts a scorecard + leaderboard note; a human reviews the -# code and merges it as a new default if it wins. that human review is the -# review gate applied to engine code. +# what keeps that sane - the containment contract, all three parts: +# 1. merged strategies are QUARANTINED code: everything that executes a +# contrib strategy (both gates, the local bench loop) runs it through +# the sandbox child (vouch.strategy.run_sandboxed). nothing imports +# contrib code in-process, and the ledger/ratchet scripts treat the +# winner as text (promote_champion.py copies bytes, imports nothing). +# 2. the scoring job never holds a write token. it executes the untrusted +# challenger, so it gets contents: read + the comment scope only; the +# arm job below holds the write scopes, checks out nothing, and runs +# no challenger code - it turns the verdict into a merge, that is all. +# accepted residual: a sandbox escape during scoring can forge the +# verdict and land its own PR in contrib/ - inside the quarantine. +# 3. shipped defaults stay human. auto-merge fires only when the PR base +# is the ladder branch (KOTH_LADDER_BASE); promotion of a champion +# into src/vouch as trusted, importable default code remains a +# human-reviewed PR. the benchmark is never the sole gate to code +# users install. # -# security model: +# security model of the scoring job itself (unchanged): # - pull_request_target: the workflow, the grader (score_strategy.py), and # the champion strategy all come from the BASE branch. only the challenger # .py is read from the PR, and it is executed ONLY inside the sandbox # child (vouch.strategy.run_sandboxed: rlimits + an audit hook blocking # network/subprocess/writes). -# - permissions: contents: read only. even a full sandbox escape lands on -# an ephemeral runner with a read-only token and no secrets, and cannot -# merge itself. name: koth-engine-gate on: @@ -35,7 +45,10 @@ jobs: timeout-minutes: 30 permissions: contents: read - pull-requests: write # scorecard comment only - no merge + pull-requests: write # scorecard comment only - the merge lives in `arm` + outputs: + mode: ${{ steps.classify.outputs.mode }} + verdict: ${{ steps.score.outputs.verdict }} steps: - name: checkout base branch (trusted code only) uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 @@ -136,7 +149,14 @@ jobs: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} VERDICT: ${{ steps.score.outputs.verdict }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + LADDER_BASE: ${{ vars.KOTH_LADDER_BASE }} run: | + if [ "$BASE_REF" = "$LADDER_BASE" ] && [ -n "$LADDER_BASE" ]; then + gate_note="auto-merge on dethrone (quarantined contrib lane: merged strategies only ever run inside the sandbox; shipped defaults still require a human PR)" + else + gate_note="scored only - auto-merge is disabled for base '${BASE_REF}'; a maintainer reviews and merges winners here" + fi { echo "koth engine lane - ${VERDICT}" echo @@ -144,10 +164,10 @@ jobs: cat /tmp/engine-report.json echo '```' echo - echo "engine code is NOT auto-merged. a winning strategy earns the" - echo "leaderboard place; a maintainer reviews the code and merges it" - echo "as a new default. the daily result is provisional (public seeds)" - echo "- payout rank is settled by the monthly sealed run." + echo "${gate_note}." + echo + echo "the daily result is provisional (public seeds) - payout rank" + echo "is settled by the monthly sealed run." } > /tmp/comment.md gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \ --body-file /tmp/comment.md @@ -158,7 +178,36 @@ jobs: VERDICT: ${{ steps.score.outputs.verdict }} run: | echo "verdict: ${VERDICT}" - # a held challenger is a green, informational result - nothing merges - # here regardless, so the gate never blocks. a scoring error already - # failed the job above. + # a held challenger is a green, informational result, and the arm + # job merges nothing for it. a scoring error already failed the + # job above. exit 0 + + # the write-capable half, deliberately separated from the job that + # executes the untrusted challenger: this job checks out nothing, runs no + # challenger code, and touches no file from the PR - it reads the gate's + # verdict and arms native auto-merge, exactly like the kit lane. the + # merge itself is performed by github once required checks are green. + # auto-merge fires ONLY when the PR base is the dedicated ladder branch + # (repo variable KOTH_LADDER_BASE): the trunk keeps human review, and a + # champion reaches shipped defaults only through a human PR. + arm: + needs: gate + if: >- + needs.gate.outputs.mode == 'engine' && + needs.gate.outputs.verdict == 'dethroned' && + vars.KOTH_LADDER_BASE != '' && + github.event.pull_request.base.ref == vars.KOTH_LADDER_BASE + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: write # auto-merge (squash) needs it + pull-requests: write # enable auto-merge + steps: + - name: enable auto-merge on dethrone (ladder branch only) + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + gh pr merge "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --auto --squash diff --git a/docs/koth-strategy-lane.md b/docs/koth-strategy-lane.md index 19bb38ab..77db6f83 100644 --- a/docs/koth-strategy-lane.md +++ b/docs/koth-strategy-lane.md @@ -28,8 +28,8 @@ places: |---|---|---| | what you submit | `competition/kits/current/kit.yaml` (data) | `contrib/strategies/.py` (code) | | scored by | vouchbench, config arm | vouchbench, sandboxed strategy arm | -| on a win | **auto-merges** (data cannot execute) | leaderboard + payout; **never auto-merges** | -| how it ships | promoted to defaults by a human PR | reviewed and merged by a human | +| on a win | **auto-merges** (data cannot execute) | leaderboard + payout; **auto-merges into the quarantined contrib lane** (ladder branch only) | +| how it ships | promoted to defaults by a human PR | merged code stays sandbox-executed; promoted to trusted defaults by a human PR | ## the interface @@ -77,12 +77,23 @@ strategy simply fails to improve; it cannot take down scoring. ### the honest boundary an in-process python guard cannot stop a determined native-code escape. it is -defence in depth, not the trust root. the real boundary is the same one ditto -relies on: the scoring job runs on an ephemeral CI runner with a **read-only -token and no secrets**, and **engine code is never auto-merged**. the benchmark -decides the *rank*; a human reviewing the diff decides what *ships*. if you -ever score submissions off ephemeral CI, add OS-level isolation (a container, -nsjail, or gVisor) before trusting the audit hook alone. +defence in depth, not the trust root. the trust root has two parts: + +- **the scoring job runs on an ephemeral CI runner with a read-only token + and no secrets.** the write-capable arm job is a separate job that checks + out nothing and runs no challenger code — it only turns the verdict into + a native auto-merge. accepted residual: an escape during scoring can + forge the verdict and land its own PR — inside the quarantine below. +- **merged strategy code is quarantined, not trusted.** everything that + executes a contrib strategy (both gates, the local bench loop) runs it + through the sandbox child; the ledger and ratchet scripts treat it as + text and import nothing. the benchmark decides the *rank* and the merge; + a human reviewing the diff still decides what *ships* — promotion into + `src/vouch` as importable default code is a human PR, and auto-merge is + confined to the dedicated ladder branch (`KOTH_LADDER_BASE`). + +if you ever score submissions off ephemeral CI, add OS-level isolation (a +container, nsjail, or gVisor) before trusting the audit hook alone. ## reproduce locally diff --git a/src/vouch/strategy.py b/src/vouch/strategy.py index 35d0a090..667fbf19 100644 --- a/src/vouch/strategy.py +++ b/src/vouch/strategy.py @@ -21,13 +21,19 @@ in-process Python guard is a true boundary - a determined attacker who can introspect the interpreter (native ctypes/cffi, or pure-Python gc/frame walking to reach and neutralise the hook object) can defeat it. The *real* boundary is -the same one ditto leans on: the scoring job runs on an ephemeral CI runner -with a read-only token and no secrets, and **strategy code is never -auto-merged** - it earns a leaderboard place and is shipped only through human -review. So the worst a full escape achieves during scoring is misbehaviour on a -throwaway runner that can reach nothing and merge nothing. The sandbox is -defence in depth; the trust root is the runner's least privilege plus the human -merge gate. +the same one ditto leans on, in two parts. First, least privilege at scoring +time: the scoring job runs on an ephemeral CI runner with a read-only token +and no secrets, and the write-capable job that arms auto-merge on a dethrone +checks out nothing and executes no challenger code. Second, quarantine after +merge: a winning strategy auto-merges into ``contrib/`` (ladder branch only), +where it is executed exclusively through this sandbox by both gates and the +bench - never imported in-process - and the ledger/ratchet scripts treat it +as text. Trusted, importable shipping (promotion into ``vouch.strategies``) +still happens only through a human-reviewed PR. So a full escape during +scoring can at worst forge its own verdict and land code in the quarantine +it already runs in; it cannot reach a write token, secrets, or an import +path. The sandbox is defence in depth; the trust root is least privilege +plus the quarantine plus the human gate on shipped defaults. """ from __future__ import annotations diff --git a/tests/test_session_transcript.py b/tests/test_session_transcript.py index 19c0a6d9..5af1a244 100644 --- a/tests/test_session_transcript.py +++ b/tests/test_session_transcript.py @@ -196,9 +196,17 @@ def test_handler_bad_agent_is_invalid_request() -> None: assert resp["error"]["code"] == "invalid_request" -def test_handler_returns_degraded_when_absent() -> None: +def test_handler_returns_degraded_when_absent( + store: KBStore, monkeypatch: pytest.MonkeyPatch +) -> None: from vouch.jsonl_server import handle_request + # handle_request discovers the KB via cwd; point both agent search roots + # at empty dirs so the locator misses and returns the degraded payload. + monkeypatch.chdir(store.root) + monkeypatch.setenv("VOUCH_CLAUDE_PROJECTS_DIR", str(store.kb_dir / "no-claude")) + monkeypatch.setenv("CODEX_HOME", str(store.kb_dir / "no-codex")) + resp = handle_request({ "id": "3", "method": "kb.session_transcript", "params": {"session_id": "11111111-1111-1111-1111-111111111111"},