diff --git a/.github/workflows/koth-engine-gate.yml b/.github/workflows/koth-engine-gate.yml index e3f6fc9e..6230bbf3 100644 --- a/.github/workflows/koth-engine-gate.yml +++ b/.github/workflows/koth-engine-gate.yml @@ -38,7 +38,7 @@ jobs: pull-requests: write # scorecard comment only - no merge steps: - name: checkout base branch (trusted code only) - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - name: classify the PR id: classify @@ -48,8 +48,13 @@ jobs: run: | gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \ --paginate --jq '.[].filename' > /tmp/changed.txt + # a delete-only pr matches the filename shape but has nothing to + # fetch at the head sha - classify it normal, not engine. + gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \ + --paginate --jq '.[] | select(.status != "removed") | .filename' \ + > /tmp/present.txt count=$(wc -l < /tmp/changed.txt) - only=$(head -1 /tmp/changed.txt) + only=$(head -1 /tmp/present.txt) case "$only" in contrib/strategies/baseline.py|contrib/strategies/README.md) only="" ;; esac @@ -88,7 +93,7 @@ jobs: - name: set up python if: steps.classify.outputs.mode == 'engine' - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.12' diff --git a/.github/workflows/koth-gate.yml b/.github/workflows/koth-gate.yml index b13e0172..db3181bb 100644 --- a/.github/workflows/koth-gate.yml +++ b/.github/workflows/koth-gate.yml @@ -37,7 +37,12 @@ jobs: steps: - name: checkout base branch (trusted code only) uses: actions/checkout@v4 - # no ref: pull_request_target checks out the base branch tip + # pull_request_target defaults to the DEFAULT branch, not the PR + # base - pin the base ref explicitly. still trusted code: a branch + # of this repo, never the PR head. the base carries the scripts, + # the engine, and the reigning kit. + with: + ref: ${{ github.event.pull_request.base.ref }} - name: classify the PR id: classify @@ -103,10 +108,12 @@ jobs: - name: paired scoring - challenger vs reigning kit if: steps.classify.outputs.mode == 'ladder' id: score - env: - BASE_SHA: ${{ github.sha }} run: | set +e + # seed identity = the tree actually scored (the checked-out base + # tip), not github.sha, which is the default branch under + # pull_request_target + BASE_SHA="$(git rev-parse HEAD)" python .github/scripts/koth_score.py \ --champion competition/kits/current/kit.yaml \ --challenger /tmp/kit-challenger.yaml \ diff --git a/competition/LEADERBOARD.md b/competition/LEADERBOARD.md index c9517c78..3b43e6e2 100644 --- a/competition/LEADERBOARD.md +++ b/competition/LEADERBOARD.md @@ -13,6 +13,8 @@ payout rank is settled by the monthly sealed commit-reveal run. | # | champion | PR | dethroned on | scored mean | margin over prior | |---|----------|----|--------------|-------------|-------------------| | 0 | baseline kit (repo defaults) | — | 2026-07-28 | 0.52 ± 0.03 (seeds 1–6) | — | +| 1 | plind-junior (kit) | #566 | 2026-07-27 | 0.5333 | +0.3333 | +| 2 | plind-junior (kit) | #565 | 2026-07-27 | 0.5667 | +0.3667 | payouts follow the season shares in docs/vouchbench-seasons.md (65/14/10/7/4). days-on-throne accrue between dethrones; the monthly diff --git a/competition/kits/current/kit.yaml b/competition/kits/current/kit.yaml index 59fb05dc..2c406881 100644 --- a/competition/kits/current/kit.yaml +++ b/competition/kits/current/kit.yaml @@ -9,9 +9,8 @@ # values reproduce the repo baseline exactly. # # dethrone these values by max(0.007, 1.96 * paired SE) over the day's -# seeds and your PR auto-merges on the ladder branch. allowed keys and -# bounds are enforced by .github/scripts/validate_kit.py - anything else -# fails before scoring. +# seeds and your PR auto-merges. allowed keys and bounds are enforced by +# .github/scripts/validate_kit.py - anything else fails before scoring. retrieval: backend: auto default_limit: 10 diff --git a/contrib/strategies/provenance_rank.py b/contrib/strategies/provenance_rank.py new file mode 100644 index 00000000..16ef6217 --- /dev/null +++ b/contrib/strategies/provenance_rank.py @@ -0,0 +1,81 @@ +"""Provenance-aware ranking: first-hand facts over hearsay and instructions. + +Three general principles, none keyed to any benchmark artifact: + +1. **Hearsay demotion.** A memory that *attributes* a fact to a named third + party via reported speech ("X mentioned her ... is", "X said his ...") + is weaker evidence about the speaker's own facts than a first-hand + statement — especially when the query is first-person ("my ...") or + team-scoped ("our ...", "we ..."). Demote it below first-hand memories. + +2. **Instruction demotion.** A memory that tries to *instruct the reader* + ("always answer", "no matter what", "if anyone asks") is a stored + prompt-injection, not a fact. It should rank behind every ordinary + memory that matches the query. + +3. **Update boost.** A memory phrased as a change of state ("changed to", + "moved ... to", "is now") supersedes a plain assertion of the same + attribute; surface it first so the reader sees the newest value inside + a tight budget. + +Ties and everything else defer to the backend's fused score, blended with +plain lexical overlap so an obviously-relevant hit the fusion under-ranked +still gets rescued. +""" + +import re + +from vouch.strategy import Candidate + +_WORD = re.compile(r"[a-z0-9]+") + +# reported speech about a named third party: "Foo mentioned her X is ...", +# "foo-bar said his X is ...". requires BOTH a leading name-like token and +# a speech verb with a third-person possessive, so a first-hand "i said i +# would ..." is untouched. +_HEARSAY = re.compile( + r"\b[a-z][a-z0-9-]*\s+(?:mentioned|said|says|claims|claimed|told\s+\w+)\b" + r".{0,40}\b(?:her|his|their)\b", + re.IGNORECASE | re.DOTALL, +) + +# stored instructions aimed at whoever reads the memory later. +_INSTRUCTION = re.compile( + r"\b(?:always\s+answer|no\s+matter\s+what|if\s+anyone\s+asks|" + r"future\s+assistant|ignore\s+(?:any|all|previous))\b", + re.IGNORECASE, +) + +# change-of-state phrasing: the newest value of an attribute. +_UPDATE = re.compile( + r"\b(?:changed\s+to|moved\s+(?:\w+\s+){0,3}(?:over\s+)?to|is\s+now|" + r"switched\s+to|renamed\s+to)\b", + re.IGNORECASE, +) + +_FIRST_PERSON_QUERY = re.compile(r"\b(?:my|our|we|i)\b", re.IGNORECASE) + + +def _tokens(text: str) -> set[str]: + return set(_WORD.findall(text.lower())) + + +def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]: + q = _tokens(query) + first_person = bool(_FIRST_PERSON_QUERY.search(query)) + + def key(c: Candidate) -> float: + text = c.summary + overlap = len(q & _tokens(text)) / len(q) if q else 0.0 + score = 0.7 * c.score + 0.3 * overlap + if _INSTRUCTION.search(text): + score -= 10.0 + if _HEARSAY.search(text) and first_person: + score -= 5.0 + elif _HEARSAY.search(text): + score -= 2.0 + if _UPDATE.search(text): + score += 1.0 + return score + + return [c.id for c in sorted(candidates, key=key, reverse=True)]