From c424d4cbb9c46a1c34f2299341aaf71525d449f3 Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Tue, 28 Jul 2026 20:05:19 +0900 Subject: [PATCH] feat(competition): engine-lane auto-merge on dethrone, quarantined a winning strategy submission now auto-merges on the ladder branch - the benchmark decides what enters contrib/, with no human in the loop. the arm job is deliberately separate from the scoring job: scoring executes the untrusted challenger and keeps a read-only token; arming holds the write scopes, checks out nothing, and runs no challenger code. auto-merge fires only when the pr base is KOTH_LADDER_BASE, exactly like the kit lane. what keeps this sane is the quarantine, now stated as the contract in the workflow header and the strategy module docstring: merged contrib strategies execute exclusively through the sandbox child (both gates and the local bench wrap them in SandboxProxy), the ledger and ratchet scripts treat the winner as text and import nothing, and promotion into src/vouch as trusted importable default code remains a human-reviewed pr. accepted residual, documented: a sandbox escape during scoring can forge its own verdict and land its pr - inside the quarantine it already runs in, with no write token, secrets, or import path reachable. docs and the scorecard comment updated to stop claiming engine code never auto-merges. --- .github/scripts/score_strategy.py | 6 +- .github/workflows/koth-engine-gate.yml | 87 ++++++++++++++++++++------ docs/koth-strategy-lane.md | 27 +++++--- src/vouch/strategy.py | 20 +++--- 4 files changed, 104 insertions(+), 36 deletions(-) diff --git a/.github/scripts/score_strategy.py b/.github/scripts/score_strategy.py index 128993fb..f49e5570 100644 --- a/.github/scripts/score_strategy.py +++ b/.github/scripts/score_strategy.py @@ -7,8 +7,10 @@ dethroned iff mean(challenger - champion) >= max(0.007, 1.96 x SE) -Exit code 0 = dethroned, 3 = held, 1 = error. Unlike the kit lane, a winning -verdict here does NOT auto-merge: engine code ships only through human review. +Exit code 0 = dethroned, 3 = held, 1 = error. A dethrone on the ladder +branch arms auto-merge into the quarantined contrib lane (merged strategies +only ever execute inside the sandbox); promotion into src/vouch as trusted +default code still ships through a human-reviewed PR. """ from __future__ import annotations diff --git a/.github/workflows/koth-engine-gate.yml b/.github/workflows/koth-engine-gate.yml index db682017..394c075e 100644 --- a/.github/workflows/koth-engine-gate.yml +++ b/.github/workflows/koth-engine-gate.yml @@ -1,21 +1,31 @@ -# koth engine gate - scores strategy (ranking-code) submissions. +# koth engine gate - scores strategy (ranking-code) submissions and, on a +# dethrone against the ladder branch, arms auto-merge: the benchmark decides +# what enters the quarantined contrib lane, with no human in the loop. # -# the deliberate contrast with koth-gate.yml (the kit lane): a kit is data -# and auto-merges; a STRATEGY is code and NEVER auto-merges. this job has no -# write token, holds no secrets, and merges nothing. it scores the submission -# in a sandbox and posts a scorecard + leaderboard note; a human reviews the -# code and merges it as a new default if it wins. that human review is the -# review gate applied to engine code. +# what keeps that sane - the containment contract, all three parts: +# 1. merged strategies are QUARANTINED code: everything that executes a +# contrib strategy (both gates, the local bench loop) runs it through +# the sandbox child (vouch.strategy.run_sandboxed). nothing imports +# contrib code in-process, and the ledger/ratchet scripts treat the +# winner as text (promote_champion.py copies bytes, imports nothing). +# 2. the scoring job never holds a write token. it executes the untrusted +# challenger, so it gets contents: read + the comment scope only; the +# arm job below holds the write scopes, checks out nothing, and runs +# no challenger code - it turns the verdict into a merge, that is all. +# accepted residual: a sandbox escape during scoring can forge the +# verdict and land its own PR in contrib/ - inside the quarantine. +# 3. shipped defaults stay human. auto-merge fires only when the PR base +# is the ladder branch (KOTH_LADDER_BASE); promotion of a champion +# into src/vouch as trusted, importable default code remains a +# human-reviewed PR. the benchmark is never the sole gate to code +# users install. # -# security model: +# security model of the scoring job itself (unchanged): # - pull_request_target: the workflow, the grader (score_strategy.py), and # the champion strategy all come from the BASE branch. only the challenger # .py is read from the PR, and it is executed ONLY inside the sandbox # child (vouch.strategy.run_sandboxed: rlimits + an audit hook blocking # network/subprocess/writes). -# - permissions: contents: read only. even a full sandbox escape lands on -# an ephemeral runner with a read-only token and no secrets, and cannot -# merge itself. name: koth-engine-gate on: @@ -35,7 +45,10 @@ jobs: timeout-minutes: 30 permissions: contents: read - pull-requests: write # scorecard comment only - no merge + pull-requests: write # scorecard comment only - the merge lives in `arm` + outputs: + mode: ${{ steps.classify.outputs.mode }} + verdict: ${{ steps.score.outputs.verdict }} steps: - name: checkout base branch (trusted code only) uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 @@ -136,7 +149,14 @@ jobs: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} VERDICT: ${{ steps.score.outputs.verdict }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + LADDER_BASE: ${{ vars.KOTH_LADDER_BASE }} run: | + if [ "$BASE_REF" = "$LADDER_BASE" ] && [ -n "$LADDER_BASE" ]; then + gate_note="auto-merge on dethrone (quarantined contrib lane: merged strategies only ever run inside the sandbox; shipped defaults still require a human PR)" + else + gate_note="scored only - auto-merge is disabled for base '${BASE_REF}'; a maintainer reviews and merges winners here" + fi { echo "koth engine lane - ${VERDICT}" echo @@ -144,10 +164,10 @@ jobs: cat /tmp/engine-report.json echo '```' echo - echo "engine code is NOT auto-merged. a winning strategy earns the" - echo "leaderboard place; a maintainer reviews the code and merges it" - echo "as a new default. the daily result is provisional (public seeds)" - echo "- payout rank is settled by the monthly sealed run." + echo "${gate_note}." + echo + echo "the daily result is provisional (public seeds) - payout rank" + echo "is settled by the monthly sealed run." } > /tmp/comment.md gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \ --body-file /tmp/comment.md @@ -158,7 +178,36 @@ jobs: VERDICT: ${{ steps.score.outputs.verdict }} run: | echo "verdict: ${VERDICT}" - # a held challenger is a green, informational result - nothing merges - # here regardless, so the gate never blocks. a scoring error already - # failed the job above. + # a held challenger is a green, informational result, and the arm + # job merges nothing for it. a scoring error already failed the + # job above. exit 0 + + # the write-capable half, deliberately separated from the job that + # executes the untrusted challenger: this job checks out nothing, runs no + # challenger code, and touches no file from the PR - it reads the gate's + # verdict and arms native auto-merge, exactly like the kit lane. the + # merge itself is performed by github once required checks are green. + # auto-merge fires ONLY when the PR base is the dedicated ladder branch + # (repo variable KOTH_LADDER_BASE): the trunk keeps human review, and a + # champion reaches shipped defaults only through a human PR. + arm: + needs: gate + if: >- + needs.gate.outputs.mode == 'engine' && + needs.gate.outputs.verdict == 'dethroned' && + vars.KOTH_LADDER_BASE != '' && + github.event.pull_request.base.ref == vars.KOTH_LADDER_BASE + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: write # auto-merge (squash) needs it + pull-requests: write # enable auto-merge + steps: + - name: enable auto-merge on dethrone (ladder branch only) + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + gh pr merge "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --auto --squash diff --git a/docs/koth-strategy-lane.md b/docs/koth-strategy-lane.md index 19bb38ab..77db6f83 100644 --- a/docs/koth-strategy-lane.md +++ b/docs/koth-strategy-lane.md @@ -28,8 +28,8 @@ places: |---|---|---| | what you submit | `competition/kits/current/kit.yaml` (data) | `contrib/strategies/.py` (code) | | scored by | vouchbench, config arm | vouchbench, sandboxed strategy arm | -| on a win | **auto-merges** (data cannot execute) | leaderboard + payout; **never auto-merges** | -| how it ships | promoted to defaults by a human PR | reviewed and merged by a human | +| on a win | **auto-merges** (data cannot execute) | leaderboard + payout; **auto-merges into the quarantined contrib lane** (ladder branch only) | +| how it ships | promoted to defaults by a human PR | merged code stays sandbox-executed; promoted to trusted defaults by a human PR | ## the interface @@ -77,12 +77,23 @@ strategy simply fails to improve; it cannot take down scoring. ### the honest boundary an in-process python guard cannot stop a determined native-code escape. it is -defence in depth, not the trust root. the real boundary is the same one ditto -relies on: the scoring job runs on an ephemeral CI runner with a **read-only -token and no secrets**, and **engine code is never auto-merged**. the benchmark -decides the *rank*; a human reviewing the diff decides what *ships*. if you -ever score submissions off ephemeral CI, add OS-level isolation (a container, -nsjail, or gVisor) before trusting the audit hook alone. +defence in depth, not the trust root. the trust root has two parts: + +- **the scoring job runs on an ephemeral CI runner with a read-only token + and no secrets.** the write-capable arm job is a separate job that checks + out nothing and runs no challenger code — it only turns the verdict into + a native auto-merge. accepted residual: an escape during scoring can + forge the verdict and land its own PR — inside the quarantine below. +- **merged strategy code is quarantined, not trusted.** everything that + executes a contrib strategy (both gates, the local bench loop) runs it + through the sandbox child; the ledger and ratchet scripts treat it as + text and import nothing. the benchmark decides the *rank* and the merge; + a human reviewing the diff still decides what *ships* — promotion into + `src/vouch` as importable default code is a human PR, and auto-merge is + confined to the dedicated ladder branch (`KOTH_LADDER_BASE`). + +if you ever score submissions off ephemeral CI, add OS-level isolation (a +container, nsjail, or gVisor) before trusting the audit hook alone. ## reproduce locally diff --git a/src/vouch/strategy.py b/src/vouch/strategy.py index 35d0a090..667fbf19 100644 --- a/src/vouch/strategy.py +++ b/src/vouch/strategy.py @@ -21,13 +21,19 @@ in-process Python guard is a true boundary - a determined attacker who can introspect the interpreter (native ctypes/cffi, or pure-Python gc/frame walking to reach and neutralise the hook object) can defeat it. The *real* boundary is -the same one ditto leans on: the scoring job runs on an ephemeral CI runner -with a read-only token and no secrets, and **strategy code is never -auto-merged** - it earns a leaderboard place and is shipped only through human -review. So the worst a full escape achieves during scoring is misbehaviour on a -throwaway runner that can reach nothing and merge nothing. The sandbox is -defence in depth; the trust root is the runner's least privilege plus the human -merge gate. +the same one ditto leans on, in two parts. First, least privilege at scoring +time: the scoring job runs on an ephemeral CI runner with a read-only token +and no secrets, and the write-capable job that arms auto-merge on a dethrone +checks out nothing and executes no challenger code. Second, quarantine after +merge: a winning strategy auto-merges into ``contrib/`` (ladder branch only), +where it is executed exclusively through this sandbox by both gates and the +bench - never imported in-process - and the ledger/ratchet scripts treat it +as text. Trusted, importable shipping (promotion into ``vouch.strategies``) +still happens only through a human-reviewed PR. So a full escape during +scoring can at worst forge its own verdict and land code in the quarantine +it already runs in; it cannot reach a write token, secrets, or an import +path. The sandbox is defence in depth; the trust root is least privilege +plus the quarantine plus the human gate on shipped defaults. """ from __future__ import annotations