diff --git a/.github/workflows/koth-engine-gate.yml b/.github/workflows/koth-engine-gate.yml new file mode 100644 index 00000000..6230bbf3 --- /dev/null +++ b/.github/workflows/koth-engine-gate.yml @@ -0,0 +1,157 @@ +# koth engine gate - scores strategy (ranking-code) submissions. +# +# the deliberate contrast with koth-gate.yml (the kit lane): a kit is data +# and auto-merges; a STRATEGY is code and NEVER auto-merges. this job has no +# write token, holds no secrets, and merges nothing. it scores the submission +# in a sandbox and posts a scorecard + leaderboard note; a human reviews the +# code and merges it as a new default if it wins. that human review is the +# review gate applied to engine code. +# +# security model: +# - pull_request_target: the workflow, the grader (score_strategy.py), and +# the champion strategy all come from the BASE branch. only the challenger +# .py is read from the PR, and it is executed ONLY inside the sandbox +# child (vouch.strategy.run_sandboxed: rlimits + an audit hook blocking +# network/subprocess/writes). +# - permissions: contents: read only. even a full sandbox escape lands on +# an ephemeral runner with a read-only token and no secrets, and cannot +# merge itself. +name: koth-engine-gate + +on: + pull_request_target: # zizmor: ignore[dangerous-triggers] workflow + grader + champion all come from the base branch; the PR's .py runs only inside the sandbox child, and the job token is read-only with no secrets + types: [opened, synchronize, reopened, ready_for_review] + +concurrency: + group: koth-engine-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + gate: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + pull-requests: write # scorecard comment only - no merge + steps: + - name: checkout base branch (trusted code only) + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: classify the PR + id: classify + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \ + --paginate --jq '.[].filename' > /tmp/changed.txt + # a delete-only pr matches the filename shape but has nothing to + # fetch at the head sha - classify it normal, not engine. + gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \ + --paginate --jq '.[] | select(.status != "removed") | .filename' \ + > /tmp/present.txt + count=$(wc -l < /tmp/changed.txt) + only=$(head -1 /tmp/present.txt) + case "$only" in + contrib/strategies/baseline.py|contrib/strategies/README.md) only="" ;; + esac + if [ "$count" = "1" ] && \ + printf '%s' "$only" | grep -qE '^contrib/strategies/[A-Za-z0-9_]+\.py$'; then + echo "mode=engine" >> "$GITHUB_OUTPUT" + echo "path=$only" >> "$GITHUB_OUTPUT" + else + echo "mode=normal" >> "$GITHUB_OUTPUT" + fi + + - name: pass through (not a strategy PR) + if: steps.classify.outputs.mode == 'normal' + run: echo "not a single-strategy PR - engine gate does not apply." + + - name: fetch challenger strategy from the PR (as data) + if: steps.classify.outputs.mode == 'engine' + env: + GH_TOKEN: ${{ github.token }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + KIT_PATH: ${{ steps.classify.outputs.path }} + run: | + gh api "repos/${HEAD_REPO}/contents/${KIT_PATH}?ref=${HEAD_SHA}" \ + > /tmp/strat-meta.json + encoding=$(jq -r '.encoding // ""' /tmp/strat-meta.json) + if [ "$encoding" != "base64" ]; then + echo "strategy not returned as an inlined blob (encoding=$encoding)" >&2 + exit 1 + fi + jq -r '.content' /tmp/strat-meta.json | base64 -d > /tmp/challenger.py + if [ ! -s /tmp/challenger.py ]; then + echo "fetched strategy is empty" >&2 + exit 1 + fi + + - name: set up python + if: steps.classify.outputs.mode == 'engine' + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.12' + + - name: install vouch (base branch code) + if: steps.classify.outputs.mode == 'engine' + run: python -m pip install -e . + + - name: paired scoring - challenger vs baseline champion (sandboxed) + if: steps.classify.outputs.mode == 'engine' + id: score + env: + BASE_SHA: ${{ github.sha }} + run: | + set +e + python .github/scripts/score_strategy.py \ + --champion contrib/strategies/baseline.py \ + --challenger /tmp/challenger.py \ + --base-sha "$BASE_SHA" \ + --out /tmp/engine-report.json + code=$? + set -e + if [ "$code" = "0" ]; then + echo "verdict=dethroned" >> "$GITHUB_OUTPUT" + elif [ "$code" = "3" ]; then + echo "verdict=held" >> "$GITHUB_OUTPUT" + else + exit "$code" + fi + + - name: post the scorecard + if: steps.classify.outputs.mode == 'engine' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + VERDICT: ${{ steps.score.outputs.verdict }} + run: | + { + echo "koth engine lane - ${VERDICT}" + echo + echo '```json' + cat /tmp/engine-report.json + echo '```' + echo + echo "engine code is NOT auto-merged. a winning strategy earns the" + echo "leaderboard place; a maintainer reviews the code and merges it" + echo "as a new default. the daily result is provisional (public seeds)" + echo "- payout rank is settled by the monthly sealed run." + } > /tmp/comment.md + gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --body-file /tmp/comment.md + + - name: report the verdict as the check result + if: steps.classify.outputs.mode == 'engine' + env: + VERDICT: ${{ steps.score.outputs.verdict }} + run: | + echo "verdict: ${VERDICT}" + # a held challenger is a green, informational result - nothing merges + # here regardless, so the gate never blocks. a scoring error already + # failed the job above. + exit 0 diff --git a/.github/workflows/koth-gate.yml b/.github/workflows/koth-gate.yml index c089c8f7..548d4605 100644 --- a/.github/workflows/koth-gate.yml +++ b/.github/workflows/koth-gate.yml @@ -17,7 +17,7 @@ name: koth-gate on: - pull_request_target: + pull_request_target: # zizmor: ignore[dangerous-triggers] no PR code ever executes: the kit is fetched as data, validated against a closed-world allowlist, and scored by base-branch code with a read-only checkout types: [opened, synchronize, reopened, ready_for_review] concurrency: @@ -143,6 +143,8 @@ jobs: else gate_note="scored only - auto-merge is disabled for base '${BASE_REF}'; a maintainer promotes proven champions to shipped defaults by hand" fi + # the posted markdown carries literal backticks + # shellcheck disable=SC2016 { echo "koth ladder - ${VERDICT}" echo