From eaed5bc44e12438ce5ca70cd283fa137fc65b33e Mon Sep 17 00:00:00 2001
From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com>
Date: Sat, 17 Oct 2026 10:53:03 +0900
Subject: [PATCH 1/8] ci(verify): fail pull requests that make export, seek or
 playback slower

A new Perf budget workflow builds the engine suite's export, seek and
playback binaries for a pull request and for its merge base on the same
Blacksmith macOS 26 machine, then runs four workloads in interleaved
pairs. It fails when the median test-process CPU time (rusage user +
system) is more than 5% above the base's in every pair. Wall time is
reported and warns at 8%.

It runs for ready pull requests that change an engine crate or a
high-risk region, like the Engine verify FULL tier. Every other pull
request gets a green "Perf budget" verdict without a macOS job.

Calibrated locally with the same binaries on both sides: every workload
within 0.6% CPU and 0.4% wall.
---
 .github/workflows/perf-budget.yml | 139 ++++++++++++++++++++++++++
 scripts/verify/perf_ab.py         | 161 ++++++++++++++++++++++++++++++
 scripts/verify/perf_build.py      |  65 ++++++++++++
 3 files changed, 365 insertions(+)
 create mode 100644 .github/workflows/perf-budget.yml
 create mode 100644 scripts/verify/perf_ab.py
 create mode 100644 scripts/verify/perf_build.py

diff --git a/.github/workflows/perf-budget.yml b/.github/workflows/perf-budget.yml
new file mode 100644
index 0000000000..63abf699d2
--- /dev/null
+++ b/.github/workflows/perf-budget.yml
@@ -0,0 +1,139 @@
+name: Perf budget
+
+# Builds the engine suite's export, seek and playback binaries for a pull
+# request and for its merge base on the same Blacksmith macOS machine, runs
+# each workload in interleaved pairs and fails when the pull request is
+# measurably slower (scripts/verify/perf_ab.py). Runs for ready (non-draft)
+# pull requests that change an engine crate or a high-risk region, like the
+# Engine verify FULL tier; every other pull request gets a green verdict
+# without running. The "Perf budget" job is the one to require.
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_dispatch:
+    inputs:
+      base:
+        description: "Ref to compare against (default: the merge base with main)"
+        default: ""
+
+concurrency:
+  group: perf-budget-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  scope:
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    outputs:
+      run: ${{ steps.scope.outputs.run }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Scope
+        id: scope
+        env:
+          GH_TOKEN: ${{ github.token }}
+          EVENT: ${{ github.event_name }}
+          PR: ${{ github.event.pull_request.number }}
+          DRAFT: ${{ github.event.pull_request.draft }}
+          RUSTUP_TOOLCHAIN: stable
+        run: |
+          set -euo pipefail
+          if [[ "$EVENT" == workflow_dispatch ]]; then
+            echo "run=true" >> "$GITHUB_OUTPUT"; exit 0
+          fi
+          if [[ "$DRAFT" == true ]]; then
+            echo "Draft: the perf budget runs once the pull request is ready." | tee -a "$GITHUB_STEP_SUMMARY"
+            echo "run=false" >> "$GITHUB_OUTPUT"; exit 0
+          fi
+          gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$RUNNER_TEMP/changed.txt"
+          python3 scripts/verify/plan.py risk --files "$RUNNER_TEMP/changed.txt" --output "$RUNNER_TEMP/risk.txt"
+          if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt" \
+            || grep -qE '^(\.github/workflows/perf-budget\.yml|scripts/verify/perf_(ab|build)\.py)$' "$RUNNER_TEMP/changed.txt"; then
+            echo "run=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "No engine or high-risk code changed: nothing to time." | tee -a "$GITHUB_STEP_SUMMARY"
+            echo "run=false" >> "$GITHUB_OUTPUT"
+          fi
+
+  measure:
+    needs: scope
+    if: needs.scope.outputs.run == 'true'
+    runs-on: blacksmith-6vcpu-macos-26
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Setup
+        uses: ./.github/actions/engine-verify-setup
+        with:
+          target: aarch64-apple-darwin
+          tier: fast
+
+      - name: Perf build cache
+        uses: actions/cache@v4
+        with:
+          path: target/perf-verify/
+          key: perf-verify-v1-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }}-${{ github.run_id }}
+          restore-keys: |
+            perf-verify-v1-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }}-
+            perf-verify-v1-
+
+      - name: Base worktree
+        env:
+          BASE_INPUT: ${{ inputs.base }}
+          PR_BASE: ${{ github.event.pull_request.base.sha }}
+        run: |
+          set -euo pipefail
+          base="${BASE_INPUT:-}"
+          if [[ -z "$base" ]]; then
+            base="$(git merge-base HEAD "${PR_BASE:-origin/main}")"
+          fi
+          echo "Comparing $(git rev-parse --short HEAD) with base $(git rev-parse --short "$base")" | tee -a "$GITHUB_STEP_SUMMARY"
+          git worktree add --detach "$RUNNER_TEMP/base" "$base"
+          # The base builds with the head's native dependencies.
+          ln -s "$GITHUB_WORKSPACE/target" "$RUNNER_TEMP/base/target"
+
+      - name: Build head
+        run: python3 scripts/verify/perf_build.py . "$RUNNER_TEMP/perf/head"
+
+      - name: Build base
+        run: python3 scripts/verify/perf_build.py "$RUNNER_TEMP/base" "$RUNNER_TEMP/perf/base"
+
+      - name: Measure
+        run: |
+          python3 scripts/verify/perf_ab.py \
+            --base "$RUNNER_TEMP/perf/base/binaries.json" \
+            --head "$RUNNER_TEMP/perf/head/binaries.json" \
+            --fixtures "$RUNNER_TEMP/perf/fixtures" \
+            --out "$RUNNER_TEMP/perf/runs" \
+            --summary "$GITHUB_STEP_SUMMARY"
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: perf-budget
+          path: ${{ runner.temp }}/perf/runs/**/perf-budget.md
+          if-no-files-found: ignore
+
+  verdict:
+    name: Perf budget
+    needs: [scope, measure]
+    if: always()
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    steps:
+      - env:
+          RUN: ${{ needs.scope.outputs.run }}
+          SCOPE: ${{ needs.scope.result }}
+          MEASURE: ${{ needs.measure.result }}
+        run: |
+          if [[ "$SCOPE" != success ]]; then echo "scope failed"; exit 1; fi
+          if [[ "$RUN" != true ]]; then echo "not measured"; exit 0; fi
+          [[ "$MEASURE" == success ]]
diff --git a/scripts/verify/perf_ab.py b/scripts/verify/perf_ab.py
new file mode 100644
index 0000000000..0aa1aeb5b5
--- /dev/null
+++ b/scripts/verify/perf_ab.py
@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""Paired performance comparison of two builds of the engine suite binaries.
+
+Runs each workload alternately with the base and head binaries on the same
+machine and compares the medians of the test process's CPU time (user +
+system, from the kernel's rusage) and wall time. CPU time ignores scheduling
+noise better than wall time; pairing cancels most of the machine's state.
+
+    perf_ab.py --base base.json --head head.json --fixtures DIR --out DIR
+
+base.json / head.json map a test target ("export", "seek") to its binary and
+the crate directory to run it from:
+    {"worktree": "...", "binaries": {"export": "...", "seek": "..."}}
+"""
+import argparse
+import json
+import os
+import statistics
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+WORKLOADS = [
+    # name, test target, case glob, tier
+    ("export-basic", "export", "export/basic/*", "fast"),
+    ("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast"),
+    ("seek-storm", "seek", "seek/*/storm30/*", "fast"),
+    ("playback-basic", "playback", "playback/basic/*", "fast"),
+]
+
+# A head build is slower only when its median CPU time exceeds the base's by
+# more than this, and in most pairs.
+CPU_LIMIT = 0.05
+WALL_WARN = 0.08
+
+
+def run_once(side, target, case, tier, fixtures, report):
+    binary = side["binaries"][target]
+    env = dict(
+        os.environ,
+        CAP_VERIFY_FIXTURE_DIR=str(fixtures),
+        CAP_VERIFY_REPORT_DIR=str(report),
+        CAP_VERIFY_CASE=case,
+        CAP_VERIFY_TIER=tier,
+    )
+    report.mkdir(parents=True, exist_ok=True)
+    started = time.monotonic()
+    with (report / "test.log").open("w") as log:
+        process = subprocess.Popen(
+            [binary, "--test-threads", "1", "--nocapture"],
+            cwd=Path(side["worktree"]) / "crates/verify",
+            env=env,
+            stdout=log,
+            stderr=subprocess.STDOUT,
+        )
+        _, status, usage = os.wait4(process.pid, 0)
+    wall = time.monotonic() - started
+    outcomes = {}
+    for path in report.glob("cap-verify-*.jsonl"):
+        for line in path.read_text().splitlines():
+            try:
+                row = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if row.get("case"):
+                outcomes[row["case"]] = row.get("outcome")
+    return {
+        "cpu": usage.ru_utime + usage.ru_stime,
+        "wall": wall,
+        "exit": os.waitstatus_to_exitcode(status),
+        "outcomes": outcomes,
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base", required=True)
+    parser.add_argument("--head", required=True)
+    parser.add_argument("--fixtures", required=True)
+    parser.add_argument("--out", required=True)
+    parser.add_argument("--pairs", type=int, default=3)
+    parser.add_argument("--summary")
+    args = parser.parse_args()
+
+    sides = {
+        "base": json.loads(Path(args.base).read_text()),
+        "head": json.loads(Path(args.head).read_text()),
+    }
+    fixtures = Path(args.fixtures)
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+
+    rows = []
+    failed = []
+    for name, target, case, tier in WORKLOADS:
+        # Untimed warm-up per side: generates fixtures and warms caches.
+        for label, side in sides.items():
+            run_once(side, target, case, tier, fixtures, out / name / f"warm-{label}")
+        samples = {"base": [], "head": []}
+        for pair in range(args.pairs):
+            order = ("base", "head") if pair % 2 == 0 else ("head", "base")
+            for label in order:
+                result = run_once(
+                    sides[label], target, case, tier, fixtures, out / name / f"{pair}-{label}"
+                )
+                samples[label].append(result)
+                print(
+                    f"{name} pair {pair} {label}: cpu {result['cpu']:.2f}s wall {result['wall']:.2f}s exit {result['exit']}",
+                    flush=True,
+                )
+        statuses = {label: [s["outcomes"] for s in runs] for label, runs in samples.items()}
+        if statuses["base"][0] != statuses["head"][0]:
+            # Correctness is the suite's job; a case that now passes or fails
+            # can change how much work runs, so its time is not comparable.
+            rows.append((name, None, None, None, None, "case outcomes differ: timing not compared"))
+            continue
+        cpu = {label: statistics.median(s["cpu"] for s in runs) for label, runs in samples.items()}
+        wall = {label: statistics.median(s["wall"] for s in runs) for label, runs in samples.items()}
+        cpu_delta = cpu["head"] / cpu["base"] - 1
+        wall_delta = wall["head"] / wall["base"] - 1
+        slower_pairs = sum(
+            h["cpu"] > b["cpu"] for b, h in zip(samples["base"], samples["head"])
+        )
+        verdict = "ok"
+        if cpu_delta > CPU_LIMIT and slower_pairs * 4 >= args.pairs * 3:
+            verdict = f"SLOWER: CPU +{cpu_delta:.1%}, slower in {slower_pairs} of {args.pairs} pairs"
+            failed.append(name)
+        elif wall_delta > WALL_WARN:
+            verdict = f"wall +{wall_delta:.1%} (warning only)"
+        rows.append((name, cpu["base"], cpu["head"], wall["base"], wall["head"], verdict))
+
+    lines = [
+        "## Perf budget",
+        "",
+        f"Median of {args.pairs} interleaved pairs, test process CPU time (user + system) and wall time.",
+        "",
+        "| Workload | CPU base | CPU head | Change | Wall base | Wall head | Change | Verdict |",
+        "| --- | --- | --- | --- | --- | --- | --- | --- |",
+    ]
+    for name, cb, ch, wb, wh, verdict in rows:
+        if cb is None:
+            lines.append(f"| {name} | | | | | | | {verdict} |")
+        else:
+            lines.append(
+                f"| {name} | {cb:.2f} s | {ch:.2f} s | {ch / cb - 1:+.1%} | {wb:.2f} s | {wh:.2f} s | {wh / wb - 1:+.1%} | {verdict} |"
+            )
+    text = "\n".join(lines) + "\n"
+    print(text)
+    (out / "perf-budget.md").write_text(text)
+    if args.summary:
+        with open(args.summary, "a") as summary:
+            summary.write(text)
+    if failed:
+        print(f"::error::slower than the base build: {', '.join(failed)}")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/scripts/verify/perf_build.py b/scripts/verify/perf_build.py
new file mode 100644
index 0000000000..13b2617387
--- /dev/null
+++ b/scripts/verify/perf_build.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Builds the engine suite's export, seek and playback test binaries of one
+worktree with the perf-verify profile and copies them out, so a second build
+in the same target directory cannot replace them.
+
+    perf_build.py WORKTREE OUT_DIR   -> OUT_DIR/binaries.json
+"""
+import json
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+TARGETS = ["export", "seek", "playback"]
+# Release optimisation without link-time optimisation or single-unit codegen,
+# so two builds per pull request stay affordable. Passed as --config so a base
+# commit without this profile builds it too.
+PROFILE = [
+    "--config", 'profile.perf-verify.inherits="release"',
+    "--config", "profile.perf-verify.lto=false",
+    "--config", "profile.perf-verify.codegen-units=16",
+    "--config", "profile.perf-verify.incremental=false",
+    "--config", "profile.perf-verify.debug=0",
+]
+
+
+def main():
+    worktree, out = Path(sys.argv[1]).resolve(), Path(sys.argv[2]).resolve()
+    out.mkdir(parents=True, exist_ok=True)
+    command = ["cargo", *PROFILE, "test", "--locked", "-p", "cap-verify", "--profile", "perf-verify", "--no-run", "--message-format", "json"]
+    for target in TARGETS:
+        command += ["--test", target]
+    result = subprocess.run(command, cwd=worktree, stdout=subprocess.PIPE, text=True, check=True)
+    binaries = {}
+    for line in result.stdout.splitlines():
+        try:
+            row = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if row.get("reason") == "compiler-artifact" and row.get("executable") and row["target"]["name"] in TARGETS:
+            source = Path(row["executable"])
+            deps = out / "deps"
+            deps.mkdir(exist_ok=True)
+            dest = deps / source.name
+            shutil.copy2(source, dest)
+            binaries[row["target"]["name"]] = str(dest)
+            # macOS binaries load FFmpeg from @executable_path/../Frameworks,
+            # which scripts/setup.js fills for the debug and release profiles.
+            target_dir = source.parent.parent.parent
+            link = out / "Frameworks"
+            for profile in (source.parent.parent.name, "release", "debug"):
+                frameworks = target_dir / profile / "Frameworks"
+                if frameworks.exists() and not link.exists():
+                    link.symlink_to(frameworks, target_is_directory=True)
+                    break
+    missing = [t for t in TARGETS if t not in binaries]
+    if missing:
+        sys.exit(f"missing test binaries: {missing}")
+    (out / "binaries.json").write_text(json.dumps({"worktree": str(worktree), "binaries": binaries}, indent=2))
+    print(json.dumps(binaries, indent=2))
+
+
+if __name__ == "__main__":
+    main()

From b1928a797d5450832b060bdd80595d2357cfc466 Mon Sep 17 00:00:00 2001
From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com>
Date: Sat, 17 Oct 2026 11:28:57 +0900
Subject: [PATCH 2/8] ci(verify): fail the perf budget when a binary does not
 run, and find target/Frameworks

---
 .../__pycache__/perf_ab.cpython-314.pyc       | Bin 0 -> 11135 bytes
 .../__pycache__/perf_build.cpython-314.pyc    | Bin 0 -> 4426 bytes
 scripts/verify/perf_ab.py                     |  35 +++++++++++++++---
 scripts/verify/perf_build.py                  |  12 ++++--
 4 files changed, 37 insertions(+), 10 deletions(-)
 create mode 100644 scripts/verify/__pycache__/perf_ab.cpython-314.pyc
 create mode 100644 scripts/verify/__pycache__/perf_build.cpython-314.pyc

diff --git a/scripts/verify/__pycache__/perf_ab.cpython-314.pyc b/scripts/verify/__pycache__/perf_ab.cpython-314.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4615bd1af2b591218ef845abff623995a57d900d
GIT binary patch
literal 11135
zcmb_CZBQG@l`U!X{eeIN@nsmk5k_FMcx_|H4l!U8z?KIXY)3{G(tu?PNuC)QTWcp|
zD^)9Rk}{jxEo-w$R$Hl5wE4JYtCBnKWmD(mF11eOK7@fqo+XuB-H)sM3SMuW&wD>|
zuSXh5uz1(Gx&d|fbiaQ6`gQkvuiu-UCZi6)<NN)+3#~g4`VagfJw&$f@JCRXM-<}G
zB%)-4GF~R;a$YXx3SJ@R1W$;$l2?kkidTubnpZ<EAJiPv^4eO|fhf%h8Ly)hyq+R>
z1EsVf-bkq+HJvS@)cZ~WJ(UPmqq8a}DG`&;>(E(K9qP_jbjvb@TA-{!{MoghD{_RK
z9zV<Y$S}iBhuB$9(94kC&}`Vl`ngb$3{8{#T!@^C_yaypd^^vOOmM~@WJoUJ=NWR!
z9|Ty2>(J?_NRT5LkM}$|7h*34LLMLK3GfUX^zcmJDmmxp&yzg#pYm`F=?VJC^9+DU
z2uK(Xw{o6YMmGys@E8~>Lk`1<t+R~J?}1@3VF1iC98ZSXkeA`O7LGjP94C4IEJL<N
zIEE#6>bR>M&&;-y(`;x~>~ImtVFCckMmWz5V;4u5^8^B7zm7Zx|4cB%!l0b@JmZT5
z{J|MrFysfqrx+gQ1jsx=GS~3{83sNA5b4D%3~=OZ2q?=^nVk)c%JIMpU=;}uX1Uul
z)e*i*IvgTTaME!B^WEVPd4m(FV%mR&k6?=W1}SI{MR;7&>GGXAMTTbJXFXxklVK0<
zVP_bgY^`UmghMP}-%i$Z40EyGPVynr&vTg-xQaQU!#wq}KqKjcZQ<oZ>{SfKMuGq-
za<NCG{keL)#5~I|^*v;LM@I+7Dy<|`Jtqy8ZxzSPzFxnsJ8qW=gww;HpZWHy_<8g2
z%yZViIr{I>pB?t9^1B$}bEkmcLo=MSc|^ncYKL7fJ+!RwEv#~7_c=l?*xZ@NDV<QM
z0&<sHdZ7vFY!uWfr8-(pD=67sWd=H|rj<^u)H9z`a`;p1&5l@$T%xqV?M_1;N-E_~
zWmrQiDWY93wE%okzATlf95n}IZdpVgQMhyGI;?e>B{*k^RLbXdh*nW5N=<1x2v@08
z4_wq1@=cm8r7NU`R?~&p%cS1<oYLnwqbuNdc^*P40UbJL!MaKHlc+n(wK-U2_T8|N
zGT^^%yyoycJuhiZdyeY}g-x9(Tm-8H(t7Jd8KwD}(EtaqX4>qm-3o<rJa*PSxeYKJ
z2Monk$W<>vIUA)?K8N`iOQVKM#!8&_e4A9FOcF#(wrU2rP>qJ3LFfrRiliQ6MPntD
zsc=o30mqONAuXywLy=5QnNJy=ZPIx8oHjb!^KVkAu!<_CO;kz28h1!g`MeIDGD!4L
z7P_dDXh0;|iE^`nm99p9c`e!j>x%z||9n8P1NmiSA@?xhTOJki!3nH}9y`#`-y`Hl
znSOSwppm4Goe;{7^g7*iKQ%aT+C4CMYHXb9cY}-+N(<_#e&-2lOsp;{s6Ns=+AkDV
zj1BfvV{gg@HK;-+$a8{*x#H(|cj)5B8mx6X0-+f}HWxKH9Qf72pJzc>bAoJlRPS&E
zLtal9i)%q63CgHZ6s%n5Wrp=nUj=d%-Z`J3V1kzg703HPpo=-fvd5z((A|*%bhL>Q
zJs=QXP!EC{;6NiX&^sIv2&{nw0!ZTpO}2%zmkWf?@8tz~h!fO+-_M4Ef`a2&K{<O7
z1iL_BB`IiUL%|Rq3i`bQ5ehRwLB~a=vdUCYI&l??52v6S9qT(WJ|-yVJbu1gP=x(H
zftU$|rUWgXw3}zH@X$FN@bjQ38BQRu5)+iz1mJ|CFOHr#-p6=DKBk{#L#&_xbtGum
zh&v*ZflHiND#Q4(ii~h>KIF!;!oBPkf#Cc;Mo?u`8EeHv!5kP?&`4@gP_YbFZ-PqP
z5Lek^45S8ZEYAR&@p7r9$;sT9)p#61DcS+74WQWy5pNijl0e{z2nsed$627ZS$we8
z&OJf}=Vkq2URu!3ygtX13%WyK2(Z1lGyJ&kgJPOTX=Cw1+p2L#e0tT`G(WJeGp%cM
zH+463>$UZXeXm_#uc=EkzjiTQUJd^r=~a5w{E-b4QtDqA{_1etnk43jR|v<tspMwx
z#o)53X8veeXI|)8)$Lf*HY{ly5=Yjwtt;Br<dOA~>a~*kdnNU0qa{_^l;DyhDMQb?
z*}5<mZ(A*GS~fQ)Ygf(t(q>EAQnqf{v1VypvNXb&7F|sJkqK4TyuR<-`w~ZgIP}KQ
zV)1g-GfSHCm~3Isqhh49e^i1D=9`CJJhZH@iqEX-o96q{MaA>S(&n;T`j_;n>Rs=f
zcgKjdwfxrcmyR!68`rFzOV-Z!t-E7=8*+uQV!f>L)`f3fNLZK4nimusDrC0YoVzi%
zaQWs}zVVfG>&~~-Z>kry%dI^Nnz$uSf46Gc-1N|hO1mB?P;u8r3DTNkk(*z>@#Tc_
zho(17t3@4YjcI{^<YoDq*?!M#Pc|=`cc(Qa3&%FJN`q<xDV3^6EkHYP=zl({LDup$
zOUsg_<s&3BRtV<u4F%*Mb65}jTH9+sTA}=nsqAPw@m_`gs9pPBd+9Ny`Ys`Z)C(ZF
z2k%9!s~-M0X!?2N$~8E%dM#H0g#yhFmW(rNRTM%{V4G9&ye07jD6mPM0*ZJlDCJW@
zsa!d$Y*Im~_vZBT6DZV3nxGcXbQ_yVx|xJ!l=idiC0EWWqGisUy_4sgG?o*e)h&-S
zjs!#L{;JXN`fhn7Oiwdfr#vj*MEy2al!Py=*aW$a6_tY|Xu?^wg)Df5m6U-^mfd;y
z4jER$PX#|UWw4>J##Jr#GNXRQKp3Gpu#VYpv+q)}u#uL3njHkT4p<^{fSvvG#9*PN
zNEvC90d>g>>%!)VbRC-bo-1cPO?*7@F0G)=-PxT4B^Vn&?WRnQ)39-9PsUoIjXBF<
zC)WO8%{AoVq!O*AOxeGpXH*i0VArX)ht5L`Jr5ky!aZBpfnc4AT}?ThGUql{h}0AK
z@H3#Zgf5wsx93g;f?L0MCx??!ji{ax8Fe6Rp)COYDnOeybBr!AAj&LqO|hA41^ntL
zwC5&%Jn@ft-i58S26LCT=DC>T7Tg1XTc0_98f?145w_)*Rhl(e;yIg@)&lj#;v7ow
z9KhPMQr1FTRH+eBWl(GTL~S|LRutC4&e6lZt<3M+q1Uo@>)&SbiCg<&X?(Eb|Jq&9
zFT=JkUHs|$dfYT#I$jnbI#EPEKxX!M*c2`uGts4H1TFFo+yW9lTt=5sRfW_amk+`2
z#a4Di9xj`b57lSh!nSZJ;3%VQo9|kaoA0kSxN{ZUi#E6Jn|tb6R~PPmnDxff%{rnG
z=iHc|Gi`F^ZV21m3hEHG!-g<lwn|o*JDeKKiEugQ-qUdX|1^HhXO3SXj$gi+w{-dD
zl_)%Sld?^>$4a_lOW%s8?px*Rl;j9iYeTNx(i>g883(*?T6^gx{7KG+t0^+PgRXRS
zNx1gC3_D{`JP*~AM0<fH0UB}}*)^#sgxU8l^oCUiIaNnjQuT0F8|Z2{$Bo<fhL)x-
zHBg}1Rz_&^GGe5Ju5w8m1Bp~>#BrMGDsh~aE(PFuf+XL%Lh!zcL$E5F{1%^Br5&42
zfntcu>_@7VvQuqOoZHSW1+=$Qjt*70hE{^a=HWbmWDje@wP7+`7p@1IYo{~vr)E1I
zP#vO7?4)-9uXc6iJT|pxs1;7)r}f!Q*NW1zizcO<u6^>}q@ID*+D+GGVe6%wuKUbB
z8QOAw2%zn9mOtIjz0%Ig=X8y<u2grSFCg4dpa~iz7|^%`wWp8<Cl=aU9=Z@-=Iq(h
z46l+F^Fi-w<ocWh0jswSH1(`>YPRu>HPVgF!&3izUPwd6a>#p|XmxJ<UE=sTok#7Z
z_HD-{7rx?&kL!q3qSX>tw&vJMlInnCs@!NbTSnWz{b+Q~!r6i;NB0ca+Mp%#m_>VJ
z61@JCr~!0Tt*EE`nGz8$!&)l)ruby}GyFI4tRLoJRF&~zkAYiP!!nm5II4gvu?st@
zb2y}XdQ>U;uA^#)1G~&QyBu!vSHs}g@G;Y#NPv%O9S$xsJL_Su3Iuj;3k1f&!nBY#
zYcGm?HbRQ+{fCp{)tZ)X_6$=d(PqflLUL-8pE$?$jIAbbK;Y%wY=>bB8}M#@B+-_`
z0GboHO*jG5)`ub}<=!lg?e2OKE~<!9$<T-lTkClrG^M5y$Px`LgH*e9c2K8tVU~yw
zv~HT8w8H>O-&G);3$PoU1zzVIo}Ap0FhIP>*nsS27T-oh2gw)<W)OH*z}*t?co|r4
z6}YTIEO$Jr?dcJVJw1YqjH-J`?7<;Fmba79A_$Cd5WL7ZX}}>P(+$|=f5HR*iGjrb
z;4gpqi(Lh{u$PTvL1-e@4vC-)LtKRi2M#!>hzlWq5TJ-41c|`E<e6du*zv=`zG30k
zfY^Pw3&DH*0S;T{*lk0S?B{W<5+XU?i-Kj=;}3Qm41k9&z#ZxkE2p5H1Dg}gBXC6+
zN*0>Gc_@BrMc<TcUaVYeJGjzz@V>>CHdz;<tERe`BHe%Ny)(Z$^PcP1t{8Xo`i<*}
z!PxcXqK>tq{Yyprmx~VEK61Nnsi-$KaXLn<Ym5upcw@XQekoC!P$%Y+mlvmh`sGx|
zq1#jcWVu7%{o-o>#B%Sc)aOs9PM=Si{41IZ&?n}*>A&HR6JHOlXlm{kSswy<C<-n(
zcGht`y7LRviJ|`EJ!Ce>lW}@M)W%1GPcIv=;WA7YWR9px3{a8m5JsZvA;Eb_viL3<
zRdjSsN7Y;=mL>YZ@SC`D;-bR>E2jey?!37AQ9E8oQgSuIfYZ|q+k>NLIQoUh&c^F}
z-ooR9j|W^H(L-#()Po~;9*v;^IKr2THy_(8XkH152`>EDki@~tJqo|<O}R+y@i!5;
zWaS;Z1R2M{Jrp%(*H(LQh6zG^i#;URHm`54;Tr(XSt0oojGbyZ{7&QI$z;=8njdL?
z(fIfw0S(WW+GPKqMBC*eJCCz&K+c|nq>yITi)($5d=t0iC~NI~r*H96^4Yg4epK;`
zzQ-&+9h<1aq>8K@hC+^&aEB0Y<|SJt1ufgm;C)|1D9dE<1r#!wodkp;v33RPfwCZ*
z5@hE&ymrzl%2T}nwOj)vn4cR+UZuO=e)5gJL~pD&#(eYG!liiWD_7n()GoK5e9U2*
zpRaF}flK;g)HphPVxphw5reHeVFT`NiW=HUE)bf7z^Xq;MhP4_&F)z)suIJjJ3)rD
z!tM{^eHID^uG*ufh6d6JQB;Ue`>;<LMRg-$d|2#$3{XL&Ho$l;;{Z2SHtm}N#`$1x
zHe5_*!o^#I!cqDfIUz-~uZg|HaBZe225O-Q@!%PT1V|iA&bEu?tpGH`V(&b8ExP*}
z1ackF=c&`E{2G>}*Ean{<$y~8RZ%6$;f&Y`EM-T4=TQ^(N;@+A>6itv>xtTOQW~Gu
zS;oUboWj>bvUop5n{ut$^$-Kyxv;(XB|r}eTYCVY*`ts^8E0aJy<X5jxEW$5ptnuE
z>`Y`9f+tS=$_fUL&*z3#u})B6sjd?taV5v%TPF*S9wY`GGWbO`AY#YLQc!^0VZl;D
zf_CBrH8gypw{KKX!e<Dxoakv6RGu(IEqsEK#j63j1Ft%J68F$yhd<~Rzn2qKna>Cy
zCK8zy6j6Uz(88w!?%~0a!7)JHJ3Q>3=%tRcqhiN77DmVfIn-IpgssCZO0fk(2@rR{
zF$Rv|?M;G8oD&3D@T>$Sj$CjoCZA<-Cj!S51l2UCDCnb_VnY`p&T<O3kKqIhNHM~}
z;zKT|AU-h_VAye7jjxiTo`b-KXEq$bp%y$vK^bCwpn7TCgMi=z$6}QxXfi<#7$1vz
zQ5p$a5dCf+6W~394j-&+*^pUox5zO;4Y4ONgrv*_bmZPCLGC>-$mbxRgYXCr`LJIB
zq#Smd<$@xgR>){!0*hz%Rp^Ab8&?NfdLFGCOV^Bz_l%9p#^(8fG@)A~Y)gbKu1R*M
z2-^y=>r<-?NnV8N&O@n_jbc@2zDoNO#o|e^a&Nv;zd|&o^~G!Y>Lq>kvc6`%5A?sb
z=w;%T?pwO=bSH=(YTwYlc6hC-d#S2>x$4<v^IjZc_>|@irNXG1@Bc`HAk@-#YviSo
z<)ZreV-V?y@i(vBxbpRm`6FpU{lef^2N!&avH8IjqV11*Bg9XPMKE=od`M`OrH`zr
zsUtDCq#<L(g71&Tr7=~yihTXRs|UXKxf?^V{&Z>iD`yh@$)=<&(Yac>J2sF8yM>AO
z{iJ-c@eXnKSc-Nn+s~%l%yQZEs$u%Yfd@)dZGQxjma;U&Zia3Q#jSBZF_!FJw5Im@
zQsvBwVH$AU9Jw(PZ(K3d-Zxm&mWp^+{BnX#m9)jw>(+{R<Lhm&w!Lp{iuI*i+mn}m
zs#)y)iE+u=m^uJ7eNa)epiEa(f44GeO{$WfMP+g>)x3ALV()@#y`nDB^n><A)otJ1
z>E(*?1=W39b$kpAtmZY_{(H9lX=^3sR-$pm+LAWg;>uS{Yv!gUb5mk^#q3x&m!@r1
z3)As|M0cV&DZ}$#R4rb<&HViOozc{@gLiGIlM|^^ldC7)sdL_x)0gsHOa<5!cXc%o
zO<jC0WxbZJta+qV6qi68sj?QhQQEvw4q>cs9C|W81mMB&=)aBpcI2N=tXf>ZJ@L+$
zZjayTTj+lqcEi56yOW-HZ~VgRBk_^9VTbIyXWjSP6EV#_iz}_SEDWqdaIAM#Uz2D}
zKD%1qwYYn=e$R@24{R}G@zw>HN|k$83_d*I=$W<Ab4#P=Rz{~*Ox~r@sb3A<>A%|@
zi@bbct*qr<S<4c(UY1)9FP9y@S9BNvVu~e`H$CoJ8=qPlpIRCBtrRg!<Gx>Ayu;ib
zTTs3nU9+{`v$ZZ^+hV!3ciGl^&)f?DF=DBRNn34k)hk1ZhQ#y_cCK0-i?US+(`nPC
z74ha*T#3G9*$;+SOLqbFrQI>zx~=j7R`%=dor~(*s-G9#vE42EP3=ni$%hK4eu(7d
zx|ji`XtBlAX}$I3t1Egk;aconZQ66&`VW=wRNnFYvSy{}Xu7;9;a^lF+>66^s_&TY
z`ck74YolkEM$e|G$<(ATRmQ}Q=Jw;Np*C%_eaF6HY)F^a#G}cER82=Rlqx#}KpMkM
z{S7??HlK-K{_cUKB02C={o>f|=AXOnbl+9|=3wgNsg(8fn&!-dFUjCkZJd)KZADt6
zOY0ySdh5(fXBLK2<oT4wkLxt&(wYMsMm0Xg56!5pV@cB#BVtnv^$Sz+`uI#*Z+x*Q
zZ79Aud}A06hc8Yi4lnLqwBPPZRUCTXaQKgzrgRD5UKofs#HSLL_^HHrvMcFLwk|f^
zmfxm+slHS9OZ}a`)ZW3nmb+BSISS`>>~!jkD|Pl<>a-{2nNH34Q-R>>OfYqcU7dk_
zJeQijlDhKT213_meR9aI$@+2DFF%U2;$!mvQpk;_2U;K*LdI~!_utsRFpdMXzEyqw
zeBU~ujWvF4D6Q0_^i`|Msx_iziD*d@5KvC5^)Fm|;oAJQb%;+l-fDlTJ)r=Bykcxj
zs?!EbP9!{1AXC$Ss*P|xNtvPRTiQ3ZKWl#1a-04|)pA$=a_c~<<!H)uY(+Ephb+{d
zMc?1~-}c{8EkAoSwdYvMG`OPq;vdXquQ$hbr-+t>2a?Aho|U2M{f}2HuE!4zsK_UK
z{6P`(=nFC!4aoSI+XbrPcck%jneKOvlGDW{zdNjk^1oEJo!+7OS4bN)YwD8IcIEwI
zCDh-y>re033L3W?>l(M)PS}a4A@lVc{?HD-cYtdq{#YPt%H*A5ZD*T6OnW#Upv$w+
z7yS5RKZh5to2<vty^CY<Eiqb|?e>Z0&NlWt000<t_<J`82Vc<5!q*E>j+$}W+14r6
zJ+mthYJ!XO)hUnnVwO0K^d2=!C5gZc_F_TpcKbqJx0{`X(K413I42Q%1t$=I08<HU
zSr!Mg@fR%o`G9{aV^(n(;u=n_<D?fSSQ}$@;afG}6tDW?o#hN}{k+(Ea5e;=K{AKf
z@53v;*>U)W06}w^>;Ykr5f3Yo%=`h;e}J?fpknb~2d}?J<sT9%qkLYwVFjr-uLW5<
vul-QfAX9xfEITigwQkS~L>OYx6-5Qsi^rZnc4O@8^!tSM0n}Hrn1ufYp>1w(

literal 0
HcmV?d00001

diff --git a/scripts/verify/__pycache__/perf_build.cpython-314.pyc b/scripts/verify/__pycache__/perf_build.cpython-314.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1d6c453fb7f966087b148fc1c49f797c253c9770
GIT binary patch
literal 4426
zcmb7HU2Gf25#Bo<e?(CtB~t&Ab+YwG$)w`Mi5xq29mkR_*_P-FO~<V)=5#zsXOVa3
zyJKuIHlVyDj8Q;Jfgm!F!fH_<N`DX+NKpq35CeTN0<<6_(o|0h0<?W8`XbvY&=!5@
zEXgw|$7$Rpc4u#QW_EUGcJ^a`b(I}KiG2CR1+xR8zu*s}kj2ge4LS>mK^mGw3^77z
zM4plwnWrYrl&6$N<*8XS10_c+$E}*R9r+Ps9w#&#Pz%rs#?*i``}s<S+V={q4C}X|
z^Y#w3x42w8tg$O!y;jyrAao=yL?bG#&GIxa&Ilq;t8qc&dsLdgEK7>En^t*#j%LLO
zEl1h;DVCd~HD1-|DM4fvfmdm1nwCV~eo0d1G==BsOM*6=*C+GJwC^=u5vJ#9S&^oN
zD4*BENivp$C1bP{$16xQ3(Ig4EHZ`du?u2egUZHuT4R+NUZW#|!gHFW%+nkzijqbv
zJQ&2mJm~S;?KBj;VmPnCFVE9w$C(pBW@v~WKNAce9btgMcgXk{FzokVP$khm<|TA0
zz-qIc;UG+HP)$%D9I;|2O~82-%7o!Q6rhZ;5}hP=H<!Nl7Nr89<I!Tc<%u<u|F34n
z6(Rx^kFJX#1mH&w7WV;RVe%6b`EbR&;{-#E5sVqC1*#RQ4Qd5cJJd?3jxj<eCm^f~
z#f65e5t2<IlBo)r{N(N`V+Kaa6E1`*&<->KJcSZ~kw;4hI!l*v)IK_2w7PoGT$qum
z0VJD4RKW91&OpOA$Bff<p&?@U=U|tm1x+*>e1S5b8DkwNWHJ0FTZ7g>yD?LujMLy?
zT=>$>K*Y476VD*@E&MeGe@{_KQQ{<)U~0=f%(%<HN~iriNEW4LP+44pd`-E8v65^H
z5{zeeWyxx}B3SV)aZxu|jM#Q`ZaF}gWQ^W6J@$#6#(1F9=#^*(I(KdF!zi0Dwv2*y
z!?ts0|JR<oK~>Ne=q+h9dQUngSHj-wx7qACck4-|zia>7H3uuVfBQ+fDp<wPE_9|k
zPWn-t9EPaA`1fFSxjoxu#>f*8&-k{)Nx6F4Jpx^D&yGXnI)lzYzi|>JYDBnk%*7aK
zfirYG34f#%yN-abq}S*LoscyeN?D^FZqkX>g0;~5Rq>XWWrZ`Y;=XZqaD`kU^Yaj0
zMZ176<bwScvx8g%95sgR5x9@7s4Ua^4J%5t12K)E8V7osG@_}!T;iz`r_p1Y@(&{%
zR4WQS5*JT$LNkz8p1`W+l0C3WOS!iLkQp8=Wm(8@VHpeTvbA(`Ldej~w1vzA<Rr=L
zFg#++z$pwG3cgEs$ozN>O#3cGm*>Y;Lt}|rP{e}IqaC%OArkvvKnF~1NFd^5dp>>z
zX!AXnfV>@l!<i-MC3UOJD!izvx{bdqs9IQ>)6E<_^pZ|tbdj+6d{K#;<0A>uCnxM2
zD;JSOqQd7Zf{8?B8N$F4qU2NJV#4L~#duX^XL#SVq{LWFr|=t}u=spfx0ljQ+@*+a
zQMCxb54_-I4OXA!xjEgU@GR!7;iQ-hfJO1KiY836oTl6P%RCp?*r_P5TMD2=r$j&~
zIu+q%)k`RF6iBxZD=>!95De+=0%KTE!w&&Wq9G=zsvyp62U7cK-4bDyOM;j<AxiY{
za7^ZB=#sWF1SEpRv?k>-QVAaIr(yTB$|-`ZsRJsn#bp3DiK=Z#N4FV3O}7~nV5db8
zBjArQ>>7oS!XfawIl@oHXS}TnPT#soomX`VT=W{|P^IW=z!95+UFp^UGd?_Wa!9uZ
zUu2FA1z*<fYJ3X3#DN=h6Icb<Y9uhE;P(gS<fx!U0cxrW1}3^WDzOn&caFY1K6VsP
zV1yr16iLxd0Q;00%wbVy<C+i!CrR@B(_o6G=$1kR>6Ux|si1Z~21@6`8pIHcX}S&Y
zp@t(sw_j2K(}oL<j>KcKs+(m+5H)YDQlH-(4g@RsJO)5l!O#Rpf*Ox%3dRvSh0TSX
zNSDACDXwt5g3$@~yb6&~bTU2*%sHn+z={ze!a}@3s28xWZl8V_89D*W6{A4KnlO6Z
zUi4POlLZJpunpYmRE!lw<p_x4YpI$caW9}vPviUU8;#c+S2}*&vgYZ^d3<S)Z_P9C
zvFBs%x3j;Vy>;<7b8CB!<o1lF_l&OXIho|vDeGG!Z;UKOR)Py78LDsHS-*vlbBuhy
zp+Hx<3v{)wKyOA0^w!A&z0+Tysk6n2)U;d}fmS7k$cG=t$Wz6Sg=1gUHYF|VHs{ju
z<>{2WD{JezX1!-Z?&iN%H7rhEn_NEjY2Rl9pA4h|!PLa7nJRW+c->jEaD3fW|6bKQ
zRXJC0+SQwL^(Cn-(o|8mSzG^J-;es1gKM=rlP_&qki)q+`1at^zO~BM6#dNI%4cqw
zKC^yez3s^DJC)gYD#e^hg(kC1IQ8mumYK=SDR&uVv#x1**K#!5+`m@0J2|}Wu3xTy
z_l4xqb%$$l^zG3VDmj{QbgjD^ZuDR8&$_#kM>id{izlz0T&~YJx;OoMZg&1cNFHBq
zeE0mybJ@DSw4;B^hMMUcZP(j!O}=!KFW0m;-LyA3idDDfHLN#xesJ|CS93iF(me-q
zJ%`ghhqKLxZ@-Xj9#5VAL2B|s_Vk5R(;SF6>eFLWYt0<Aw+C0P*_MHuyK^mvQZ0vI
zB;Cwq9b9sBV@L0jecj!bb9biQojG@R#@)TDef0W=uYYv)msi(4jqi=WGrlr<`{>em
z#xt_vY5LlV+P(LBP)+|o{HSH;TJ_-do~6?(r&njwUC%9EOkTX+vv4d`J@}Qidec_9
zXuD=hx%*ZJSG}p4y&2oHn^yZ<SKhd?bU9;fUv;lu$$AgudS6QSzLe`7`l5Hp5UH8W
z*uv{(Tk?BZb90XBPE*}K(>_T2G;wqPr|Ru}zq@+pLhAcdsi(PAG?wWRGnBMRRs67X
zvF}>n(km-$#@>6E^8WMdXOYX3bM8tzcYTeBin<MlXUhciKlgW_mI(2$`*!5qr(z8H
z`~Lb<`^h^UjbjbA&)X`-TB*<bEztg9f8ChJ{6{wdt!@j4@f`|>y(XMQ@Sk0s!hd%2
zi6j3{*K7YbJS*_ObXu56bQkZjzwFiV3*s#N!PV4*JBwL!r*aGoNVGkUIjTtqr`f2=
z7kD}!$Ai<~3@`fPBG$U+xdLm~<5&feQ}`HUZ8lo2w&iiW`BZ-JDX&AfhQkqw3x^B2
zmy}cmV{4@YH-Hlnewww}s4$gJ`wET6@xPvYuEB?fpS*DRL-YU!k@6xmy6uIS6p2Uq
zL&_L@;x}GB3JrLNAns9Sf_mUUgku9$Y@n(QRI`EXU!ul8+j|$R_idI+5=d<q(YF;K
Y5amcFGNz_KQMNaSAs<l9$|+FvUtZeS6951J

literal 0
HcmV?d00001

diff --git a/scripts/verify/perf_ab.py b/scripts/verify/perf_ab.py
index 0aa1aeb5b5..2458d25180 100644
--- a/scripts/verify/perf_ab.py
+++ b/scripts/verify/perf_ab.py
@@ -93,22 +93,44 @@ def main():
 
     rows = []
     failed = []
+    broken = []
+
+    def check(name, label, result, report):
+        # A test binary exits 0, or 101 when a case fails. Anything else (a
+        # signal, a missing library) means nothing was measured.
+        if result["exit"] in (0, 101) and result["outcomes"]:
+            return True
+        log = report / "test.log"
+        tail = log.read_text(errors="replace").splitlines()[-15:] if log.exists() else []
+        print(f"::error::{name} {label}: exit {result['exit']}, {len(result['outcomes'])} cases reported")
+        print("\n".join(tail))
+        return False
     for name, target, case, tier in WORKLOADS:
         # Untimed warm-up per side: generates fixtures and warms caches.
+        usable = True
         for label, side in sides.items():
-            run_once(side, target, case, tier, fixtures, out / name / f"warm-{label}")
+            report = out / name / f"warm-{label}"
+            usable &= check(name, label, run_once(side, target, case, tier, fixtures, report), report)
+        if not usable:
+            rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases"))
+            broken.append(name)
+            continue
         samples = {"base": [], "head": []}
         for pair in range(args.pairs):
             order = ("base", "head") if pair % 2 == 0 else ("head", "base")
             for label in order:
-                result = run_once(
-                    sides[label], target, case, tier, fixtures, out / name / f"{pair}-{label}"
-                )
+                report = out / name / f"{pair}-{label}"
+                result = run_once(sides[label], target, case, tier, fixtures, report)
+                if not check(name, label, result, report):
+                    broken.append(name)
                 samples[label].append(result)
                 print(
                     f"{name} pair {pair} {label}: cpu {result['cpu']:.2f}s wall {result['wall']:.2f}s exit {result['exit']}",
                     flush=True,
                 )
+        if name in broken:
+            rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases"))
+            continue
         statuses = {label: [s["outcomes"] for s in runs] for label, runs in samples.items()}
         if statuses["base"][0] != statuses["head"][0]:
             # Correctness is the suite's job; a case that now passes or fails
@@ -151,10 +173,11 @@ def main():
     if args.summary:
         with open(args.summary, "a") as summary:
             summary.write(text)
+    if broken:
+        print(f"::error::not measured: {', '.join(sorted(set(broken)))}")
     if failed:
         print(f"::error::slower than the base build: {', '.join(failed)}")
-        return 1
-    return 0
+    return 1 if broken or failed else 0
 
 
 if __name__ == "__main__":
diff --git a/scripts/verify/perf_build.py b/scripts/verify/perf_build.py
index 13b2617387..72702cb5a0 100644
--- a/scripts/verify/perf_build.py
+++ b/scripts/verify/perf_build.py
@@ -45,18 +45,22 @@ def main():
             dest = deps / source.name
             shutil.copy2(source, dest)
             binaries[row["target"]["name"]] = str(dest)
-            # macOS binaries load FFmpeg from @executable_path/../Frameworks,
-            # which scripts/setup.js fills for the debug and release profiles.
+            # macOS binaries load FFmpeg from @executable_path/../Frameworks.
+            # scripts/setup.js puts the framework in target/Frameworks; local
+            # builds may also have one per profile.
             target_dir = source.parent.parent.parent
             link = out / "Frameworks"
-            for profile in (source.parent.parent.name, "release", "debug"):
-                frameworks = target_dir / profile / "Frameworks"
+            candidates = [target_dir / source.parent.parent.name / "Frameworks", target_dir / "Frameworks"]
+            candidates += [target_dir / profile / "Frameworks" for profile in ("release", "debug")]
+            for frameworks in candidates:
                 if frameworks.exists() and not link.exists():
                     link.symlink_to(frameworks, target_is_directory=True)
                     break
     missing = [t for t in TARGETS if t not in binaries]
     if missing:
         sys.exit(f"missing test binaries: {missing}")
+    if sys.platform == "darwin" and not (out / "Frameworks").exists():
+        sys.exit("no FFmpeg Frameworks directory next to the target dir: run scripts/setup.js")
     (out / "binaries.json").write_text(json.dumps({"worktree": str(worktree), "binaries": binaries}, indent=2))
     print(json.dumps(binaries, indent=2))
 

From b5ba81549ae9f0c63fcc904c2f91c81f75e8798c Mon Sep 17 00:00:00 2001
From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com>
Date: Sat, 17 Oct 2026 11:54:14 +0900
Subject: [PATCH 3/8] ci(verify): per-workload perf limits from the Blacksmith
 noise floor, playback report-only

---
 scripts/verify/perf_ab.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/scripts/verify/perf_ab.py b/scripts/verify/perf_ab.py
index 2458d25180..452b541776 100644
--- a/scripts/verify/perf_ab.py
+++ b/scripts/verify/perf_ab.py
@@ -21,17 +21,19 @@
 import time
 from pathlib import Path
 
+# A head build is slower when its median CPU time exceeds the base's by more
+# than the workload's limit, and in most pairs. Limits sit above the noise of
+# identical code on blacksmith-6vcpu-macos-26 (run 38995088761): export-basic
+# +0.8%, seek-storm +0.9%, export-jitter-camera-40s -4.4%. Real-time playback
+# CPU varied by 15% there, so playback is reported without a limit.
 WORKLOADS = [
-    # name, test target, case glob, tier
-    ("export-basic", "export", "export/basic/*", "fast"),
-    ("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast"),
-    ("seek-storm", "seek", "seek/*/storm30/*", "fast"),
-    ("playback-basic", "playback", "playback/basic/*", "fast"),
+    # name, test target, case glob, tier, CPU limit (None: report only)
+    ("export-basic", "export", "export/basic/*", "fast", 0.05),
+    ("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast", 0.08),
+    ("seek-storm", "seek", "seek/*/storm30/*", "fast", 0.05),
+    ("playback-basic", "playback", "playback/basic/*", "fast", None),
 ]
 
-# A head build is slower only when its median CPU time exceeds the base's by
-# more than this, and in most pairs.
-CPU_LIMIT = 0.05
 WALL_WARN = 0.08
 
 
@@ -105,7 +107,7 @@ def check(name, label, result, report):
         print(f"::error::{name} {label}: exit {result['exit']}, {len(result['outcomes'])} cases reported")
         print("\n".join(tail))
         return False
-    for name, target, case, tier in WORKLOADS:
+    for name, target, case, tier, cpu_limit in WORKLOADS:
         # Untimed warm-up per side: generates fixtures and warms caches.
         usable = True
         for label, side in sides.items():
@@ -144,8 +146,8 @@ def check(name, label, result, report):
         slower_pairs = sum(
             h["cpu"] > b["cpu"] for b, h in zip(samples["base"], samples["head"])
         )
-        verdict = "ok"
-        if cpu_delta > CPU_LIMIT and slower_pairs * 4 >= args.pairs * 3:
+        verdict = "ok" if cpu_limit is not None else "reported only"
+        if cpu_limit is not None and cpu_delta > cpu_limit and slower_pairs * 4 >= args.pairs * 3:
             verdict = f"SLOWER: CPU +{cpu_delta:.1%}, slower in {slower_pairs} of {args.pairs} pairs"
             failed.append(name)
         elif wall_delta > WALL_WARN:

From 1a1845067bb65a4b7f5703af4d60d415b929fbb6 Mon Sep 17 00:00:00 2001
From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com>
Date: Sat, 17 Oct 2026 12:21:41 +0900
Subject: [PATCH 4/8] ci(verify): base builds with the head's cargo config,
 compare every run's outcomes, kebab-case scripts

---
 .github/workflows/perf-budget.yml             |  24 +++++++++---------
 .../__pycache__/perf-ab.cpython-314.pyc       | Bin 0 -> 11381 bytes
 .../__pycache__/perf-build.cpython-314.pyc    | Bin 0 -> 4426 bytes
 .../__pycache__/perf_ab.cpython-314.pyc       | Bin 11135 -> 11185 bytes
 scripts/verify/{perf_ab.py => perf-ab.py}     |  11 ++++----
 .../verify/{perf_build.py => perf-build.py}   |   2 +-
 6 files changed, 18 insertions(+), 19 deletions(-)
 create mode 100644 scripts/verify/__pycache__/perf-ab.cpython-314.pyc
 create mode 100644 scripts/verify/__pycache__/perf-build.cpython-314.pyc
 rename scripts/verify/{perf_ab.py => perf-ab.py} (94%)
 rename scripts/verify/{perf_build.py => perf-build.py} (97%)

diff --git a/.github/workflows/perf-budget.yml b/.github/workflows/perf-budget.yml
index 63abf699d2..57c4595fb3 100644
--- a/.github/workflows/perf-budget.yml
+++ b/.github/workflows/perf-budget.yml
@@ -1,12 +1,8 @@
 name: Perf budget
 
-# Builds the engine suite's export, seek and playback binaries for a pull
-# request and for its merge base on the same Blacksmith macOS machine, runs
-# each workload in interleaved pairs and fails when the pull request is
-# measurably slower (scripts/verify/perf_ab.py). Runs for ready (non-draft)
-# pull requests that change an engine crate or a high-risk region, like the
-# Engine verify FULL tier; every other pull request gets a green verdict
-# without running. The "Perf budget" job is the one to require.
+# Head and base are built and timed on one machine, interleaved, because
+# Blacksmith VMs have no hardware instruction counters and machine state
+# drifts between jobs. "Perf budget" is the job to require.
 
 on:
   pull_request:
@@ -53,7 +49,7 @@ jobs:
           gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$RUNNER_TEMP/changed.txt"
           python3 scripts/verify/plan.py risk --files "$RUNNER_TEMP/changed.txt" --output "$RUNNER_TEMP/risk.txt"
           if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt" \
-            || grep -qE '^(\.github/workflows/perf-budget\.yml|scripts/verify/perf_(ab|build)\.py)$' "$RUNNER_TEMP/changed.txt"; then
+            || grep -qE '^(\.github/workflows/perf-budget\.yml|scripts/verify/perf-(ab|build)\.py)$' "$RUNNER_TEMP/changed.txt"; then
             echo "run=true" >> "$GITHUB_OUTPUT"
           else
             echo "No engine or high-risk code changed: nothing to time." | tee -a "$GITHUB_STEP_SUMMARY"
@@ -98,18 +94,22 @@ jobs:
           fi
           echo "Comparing $(git rev-parse --short HEAD) with base $(git rev-parse --short "$base")" | tee -a "$GITHUB_STEP_SUMMARY"
           git worktree add --detach "$RUNNER_TEMP/base" "$base"
-          # The base builds with the head's native dependencies.
+          # The base builds against the head's native dependencies: the same
+          # target dir and the FFMPEG_DIR that scripts/setup.js writes to the
+          # (git-ignored) .cargo/config.toml.
           ln -s "$GITHUB_WORKSPACE/target" "$RUNNER_TEMP/base/target"
+          mkdir -p "$RUNNER_TEMP/base/.cargo"
+          cp .cargo/config.toml "$RUNNER_TEMP/base/.cargo/config.toml"
 
       - name: Build head
-        run: python3 scripts/verify/perf_build.py . "$RUNNER_TEMP/perf/head"
+        run: python3 scripts/verify/perf-build.py . "$RUNNER_TEMP/perf/head"
 
       - name: Build base
-        run: python3 scripts/verify/perf_build.py "$RUNNER_TEMP/base" "$RUNNER_TEMP/perf/base"
+        run: python3 scripts/verify/perf-build.py "$RUNNER_TEMP/base" "$RUNNER_TEMP/perf/base"
 
       - name: Measure
         run: |
-          python3 scripts/verify/perf_ab.py \
+          python3 scripts/verify/perf-ab.py \
             --base "$RUNNER_TEMP/perf/base/binaries.json" \
             --head "$RUNNER_TEMP/perf/head/binaries.json" \
             --fixtures "$RUNNER_TEMP/perf/fixtures" \
diff --git a/scripts/verify/__pycache__/perf-ab.cpython-314.pyc b/scripts/verify/__pycache__/perf-ab.cpython-314.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6d55493591943fff162bfc41a93f303862adb659
GIT binary patch
literal 11381
zcmcIKZEzdMb$h_!`;#O{@Jr<Jo1pk1QM4sdvJ_e*MVcRij^t5NAVDB-Bq4zSbO(qc
zEG2Z38AEnbQ5ky#$8il$Ck>R3nw?2nrAcO#v>B_DX$KTZ6F#$(bowLpuS%Auj@=*W
z+XD^&Y06e++9R>MxBK?(+qb*#y?yU?8x2|n&z%qMU-)trLjQqZq=(ED9{n{G77>j?
z=p3SnK_Wzmxhy0Tb9qQE=46Nzb45rY=E{&#%vB*3<g!8afF`7=M(v1JoghM5S{~BT
zq!oqqv;tDYStG4Ha0($$IYO1_tkMqjY=3IdSwju#%2am|=|VM7RVV&zYS$GRLUxyz
zV?9)m<)#DNoXhWKDR*El=;FM5z)uCHsnC3YnhJY;9$tK#WhvG_<Mp!?ANGb=YRc;e
zSe9?sYU!|_r&yPJmYNT67kvSjhjRHsEa!KHSl<FQ?+wjTA?QEl;#tb&_fWGefJg`^
z7!J4ct~pja2UzeJ7%EK;%ZsgZtjFtuVK89;9Afzp732bLmgk#z>X?0k3VG*PsyWQF
z9JO1^FYqCDu9ceR0&`-Ai$D(R15hr^yJlFMIKsTk=M($2=P-C@`~eOI<=wNaC+ze3
zXSDu+7YLtXLog>m=JHe7j)$lq@DYG0H)f%ar{)4cS%%8YY+zJA1iS!NQSe}vx4EX;
zg9}t!o5&NKv|qq{Z)+2IgA=J@+IuAw#uW7q($F3Vhj2-&&2{P!8JdQla|J0^nmr*G
zH^YXg=34ejFu;XsTd7)}WiQs+s8E3NhWPXfEMQJ(F;Cqb&`5b;Tew33ZUIAaVLw2M
zT<jKUf4&wkafoBt+HR`0y}cb{l~xj}o|gv8wTfeAUawu#j@k%;w7WvHe|X;VH^=`y
z@{^-(W!@IXM@<gDM`k!$i->_E)e4(kdKe=AEvR&4wl_i!*wg99E*(m#0&<5+dSOWE
zNaWQiq&h~%$Z2A~A`Kl>F$%jz>Y2-F8T`rjXGW|>4pBhhcDp_YC6#iAF{ow~G})??
zS^(aZD@!FhOU)s|NrYu#xifp5gBpifg0mM&rCeTv7$vQwRkXUDbQDYVz(q|y-=x{n
z+I(6V6_by>MCzT(X<e2x+B|-j<{+dJ(4n>Gt(#PT4s~U?HV>=J{UT_f_4uy~uQ@zV
z&Pzt!n&mn|L1PCBnqakdrSasav_YD$8TIo3Yi7*$>U@ZFSrVD{n#Z>RhEc#!P=Oq^
z5|q7OD&=yRe}Ob=uyDN4Zp*bvCE6%KG-ax0fD4sqXdgn4>0y$3jGM*_X=DDHHUf?z
zJ3<;%g@(fEoHm~_*juFWayetLx8~lYQhpU(z!>Smyfto@pmKQ)I;EHBp^F$(2U&+G
zv>Rn-11nvLys~Pv3)U6?4gc36`7Y!osC@2W!nZw2<be}d2|ae9p?^Tgi_-mU7C|iu
z8yhK<9P6<=nLc{3|FpAz@YMJO-RA_+C=}<_(|z_6^tf1U%Bw!sGu9{MSBww#(c`ye
zf(kSs>ksjQn!V!XL(ag(Pt{o2wEF@x0x=&kwzc8cw$LmGGMX2Ny%Akon?K-o1+kPC
z)RKIR7({u>cU)#U@ALwYD|gR(1Uc)!EGYSq2jsYzvm7@XDTMBAX~4D?QIPut(hXWc
zPyrk$LKb=l!vcvFk3a%xAwivK;ccY?>G8Tlf-JxbD!}jM0)9cxhd4nocM;^dKw{k}
zXyyX`Kq%n%x&<;2Wc`Ac4^L%usGzXpDh?k`K{+<wdtzc-P|Ul$p)Nrl^m+tx#uu0p
zG<ed^5PKyAor6Aa2=pV%3nbQEf&y!PUNAj3c4D-bbq73bAIAkaK@M6-P;+5tSR@0N
zc(GK1@ng*x=AEH{6VD3wvK0v=@Aa^PGOfWlEXVL1SWZw&+E7q(EY@m*Qrr*=92SOx
z3b1h@7TAoJOC?Q?aAH>DaRh~^{y8kNI4qw8G6=dzAn`;5ITx7ci=ehqe6X_4K0+Dq
z=Dfj>w4fb1HI66ecLqTJbI(F&y#M*X0>!k5Qig)1mUY9f*z~%gVX=QxYur?8Z)mS;
zH>+#o2VTFnSydBneEnjov=aV5(kXSy#bXbQNTGXi_=Vw^B|$C@uaRw=#=;x^m;9^7
zs>S0et$C??UAt>TQ@5h2iyzz2G_Prz6UR0SD>n*j?-$ml3`NP}hB%)XN$R^d&6cI{
zSj&2G!>YM4QN3<HkTMsgib^(%c5M{ZuN2k8m_^#C>LVknta|gncMimle}Cw$q2+?r
zihV2U(kQX?)W-!#Yx}4W>CHEeymVw$R}q_8*EKBmrc4Ek11WRKE!|CBvU1P6=Dkre
zWhuQidUJHuQomv8Sg~}xYuOv^eISz?$~H^NZ(aEIg}7z4q;X09K#9ynH|DR;FI~Rz
z)o*?^)x7&{)os<XX0^F{NgXSSF>h3?nj0P&P;uu&IV$LUP>3|fX!yogu74%2_`dNi
z<GQImr8X{+kh~(>Fx&2%ZHdNJ^WK!YaB1LyMxj?eKnjKO<0haTIP^asRUu31Mp4sB
zQPW3=Fq8@A(g$+LKjpC=__e0TcDzjS8)M1wR`Q)P-Eo`doz~(3h3XzjK<WmN>|=ov
z)>V%v146EbMdZjf+414o_T=e)ux0ESyCNTm2BVyo<!p(^K!H{A1W@D?K`EXHO6kbj
zW#{CyYJXNkKZZiJqzkI?On0!FB>fsBXw4T{OpdHw#1QtZ#gpTkG?pEo)@_e8js!z%
z|E$sQ3U7NPOiwaehb$=DLj4YQl!PxR-vYUV9hHS78PZ;{jVySDl_bC_%j`USaSSTp
zr-Yx1)>~0f?WmM`nNgp-|2jhRunW2Gh_)9IG%~W!vxUIk0d~#<{-7yno*c}#7ij}y
z(xXmUeqFF&@;__P<eQGH4aJxze>nM%U=3w<6O=Ocd8<*P4BKx*Pp54b#+bDqc4Hd=
zY`VG}oK#{ov@!E%+NYGb1UpZ)GjtAO=o`Q_4P3Cb?FiPY(9w{^N!WL=MWmj<iJt(S
zMNH8-S!?!WAh_;}7jifqm56?3grJWEEsO=AU&B1x$}y%$k7%>VHThPq<@hz(k#E&)
z<z29tQDg2h#W^l!xdjRkxb=neX9^ip{s=qrt5}*f*yCBtmC*q81>zh42b7DLV%n0A
zi!L@Gx&&&ipQ$Z{+Oqsw*f~1byXCpPJM?<Sdi}d}K6&dPER6?t{hzxF`iP-<gqT7*
z?4a$BBpu9XqGZAvmbas@tRF0lU<p$Ks|NSCpp~)G75TJ|%7<XDik7uCormQ?Yp^8H
z;K&-yhlrEpbI1ew1lz4l!#;wpv?8B+tihbUy<zOx^%8f?5Znc;;1rKoFm`&^h#77U
zJNA`UMC$*9e5uMm9euJP7!~;PIkz9+i)o@HEbqX4ftyX{?3XYmNA?b2-?N>o2-P6E
z+KMm_B@RmzPsxVhT;cfzOF@d1{J)J)ec||J;`pU-@6EfLGNoH@$N4Ao9I@q=S<aMg
z>s$83eJdPruS-94jTJeblHQoYtvKL)3w_d!`Lmo2Rx$=QEqQ7|^3(-)!8_am;k=YH
zh6As_=@}H?D$kLvdA0}W)Ey+<Ie9+JfnPvxxI2PWt7ppT1~}P`OeLK9rk#63OLM0R
zC{XSwQ8dzSs751&OoaoaefklpREy)-m<n;6mQFd~d5qlOzFP3Ug<G&fTYMd#S+QMP
zPK12uo9Vqt@1|Snw#Uw`y;BbD9rT`dWw44-fTRe)*#JEd)C8-8RInyk3p7_xr=?ZZ
zPCTIZiW0Pw*#*4X*O~S5RHLD0IF+B*r;Di;W$RN6CFM-@<M%23G^|z^Q<H(Mm2#%$
z3;U#L%la;Ww#ym*?6+fayzODCq;;kDXGI~v1nctjMx6u$a*?DD<kMh3v<)G=GWqa?
z{qScYvN8}Tszi>X5(KQ?4$$;L=>+ZIPpfC@?LAWeT%J!u+Rn&%s2Ejte2{}MepW-$
z-SnZIxa2q{VRrOMB}OH2WqXcoDya@Qrp%58&i*Z<9o~5~CTkCE!<40aMuRcsVHRb`
zIS3G(Lv<kct5I6Nd8Kkm(6qJu1%94yXZ$rsBg(XkdmOyJYL2}W#z6;Mi9OsAZCjgk
zpN}X+*LFnJ)`mUmyiEo-{)HeoKs@ZUE9?tJG;M8scy7+cEeIs`eG4SU!C@z$J7ZCb
zyoGxi!ncA)=fo>GBirghrj4Rik+zm(^cH_|meXmODyu`_4c=<0VLKe~Zhs`5m&E{@
z6F5xT0n_$}CTYd~433>{gc2^=L@Nn2LSUOd=c}ew3<CL~W(Y_%+h+%LS_iCd`hnIh
z^OJTJK<PU3Byt{hy*<x6oyC)tRT2gWCmHIH&CHbnjOa)i=P=_16}V-5E;kG7tpwLr
zfa6Can(l6~*xfA<R7BNHVNVbBsjQWXm>_P#Lu@1M%mK#@q3yTHKEMNiz(V4E_@_Vp
z$)*Hc*c-=l_^5H%qbVqY5OxWHqX!&W<b{CO4^X5ZVo2bJa!s*5>>%P{-*8ydaCpsO
zL%F;@9^3BNy+ctPKKOzH0zK}FLeZSd>u*2o15cihKhiE%x}lv1s}wCFa9QXJmm0ry
zBz9^|*N|vjF5hT5yw-B~y&`MMXjzJ^8*8HSRNufmXMT0&9mlU7QU1oY>(}Cg(QB)w
z_6^g)71P00)1f=Z?)0vhdXkf;qvWRAu%wCA$4X+C;>B@Qd_Hk`dHTm+Nwy!kGxbkJ
zcbR+7t@ll?_MA$7>2&h+Y|`jmQ(u5SQO^zUb#IJ(Ik2X#de3Be1mvLzxazpMw$aG$
zXXz6|eWTq}CI*ytjzLhzLxJxv6VKt(Oc#i@h*FGSQQR;_qU@%?$w+Zv<D!VXy>mLE
z;?ton(O-t&#FY~l9Tr$I?F;j>;_AEe&jdb=*w%T}4as4m7VP0gG>-b=9)GD|Om=Qd
zMCZT^LRZkD0yGVrNzn=hIrNm=OW0)G2|zG<8`<Q7tbLC_1jMm;e5M+y#q{!N4$rWD
zh;VU71nbr%%r-;;p2wH#KYjp&B#oW7OXEFD1F@5@4!vut{$c4SJf`^Tr4>Yf<Y~Mg
zDao}8jGLZ@pdZBk=2_N{`NwzTh#`*nVTNX6l%6f~$MlKw?}S2j_HaBd6>OD7nR$b+
zBY)nvW#S5qa!%+DFXtf>2%g86{KoXA<))y_;qCb?`0*A<#HlqM{aO9;$wb53>K~|o
zUjNA>Jg=`6TZ#Wm@?cpoFVp04(|G(HDf<iDlBKM<=V!gkml99EUG{^rpZ9*k%|h!I
zsxYY{)9_ZzG7Zdhd;#GKw+lLk*viTS-^gwwaCqdilf!aE<fToX;pLP-%w~4cc7EOj
zV7?9#%+Ci%R-t{b_2gT<@t$Z;l>OGg(xq7Ot5@FDSFg67OmCa7)z%Z>dA}GjNHPop
z+P;N|VQl!sWFOrvMwWNOaoXDu(YI2(FE9@q%Il{hBn}g2P9PsqiZSKgAQ7A4Ec@{W
z!+2~FV_h9(hmbJ@<2~4ujw0F-F&Hj(9{^MkmiDo(%Q%{kRcGs#=y)!&or#T8>Dc)8
z2zZ3KN=-`P`>SFvF*cnpicx7OLhyTrr2rB~&NJ;|c{>2ZayTYIU5)I$3Q_Ae=<~#B
zM0OQxhO1ltA~L`whpLEz;&Ddq29|Q8uv8JF=!;MDr+p5Th$~{v>KuGb=U5jH(HT!S
z#f{(&4cXSrdWcc|>^BPHXAj*J?DXRR%?&^TWt@o>wmLx#v2}<PffZux;by{f5Hhmk
zS5DBoJRT>sigkh<>nN=Vi7R;ydn7n)DT|?&G=5R3iP-U-P>_T4;qVPz&`h47hlWq|
z^o|J%_~v4c7vl?p(iMbQjYm*$cr`#1;#KEp+(UWU<qJbZM(*-2K!^rn9-ub8ZeCEP
zKcIlHP<T#|N4!Bm-ZMPxoa~`TxpCZ8JI}!w>EMnkV`*`<xJ4nhKnMlmEjYHqvmslf
zpcH2X!6iH|L4hMSJclR1g>fem$5I64G^kwYqnzRb7a_iM8n;j21fO*=vcq9}N6>)g
zar!{8LPQ5bG*dp7V=x33p`r?h*p6#1=)=(_JYPW(;5=YeOybTYmdhL{5+q7aQ9h+H
zdRSk`1yh6V>P(mQ=@oa1OcPWPbQ5D|igYAP=AIH{?pZ-L5BWUAg>bBk`#Q#qJqA3!
zHfKXZpI7)&T04<gC2?;+FTC&gHZVT^xQI3l#T$nD`-b{eL*rt9iqvk9))mqkQzyET
zq;-wl^SM>}L`a0{%0a0T^<q^=u1fPG`SMAza(}K;w?@{dbOjr_$`xJZs;+9W7wj61
z=@s&p_S@R;cE!o>Yu?hlesrUvYo(%Vwc_bj^L`v+`JCnl3b{eK*!PhdL8z(s*2v9~
zRa5Qa07SZ?p&M7OUwOHG@mPvfy*T*7;F2dkzBsr>w)|dafcTNY1XIV!honYP{ILb?
zIha^jQSXkD0BHMtL2*=>s-WIH^xC1neCGO4v~Q^^Rb2Y&nRs8KAz_VotQYT%_NTxm
zV`IHPDqXI>qrB@$GLBW-*`$+QEty`|PruavP=P9KA49aMBn9!Sq3c62ODq&0PjoF?
zlKVZ$Qg%&04LEL$Tpx+muj#Ab(_2zSWwFlK<v5otY>BEiEoHI#H(Oq7dDqeq?M*ee
zCNBM0z1;I7!-}Ooc?hU_zpQFWkt(ZvqdZ|rC=;$_MPfeLxPQHD|B`aEtR~*@z1C&r
z9nZb#)v}2t<$Ko3*f<!EjT_d3_pJw0mU7Ioc>S8CDP^|C6t5aL%nd8%hWPZFxoy*2
zoU&FdO~?A<UGc^Qf#<!fT)upV{pq#4W67rn?^%;4CzGent)Fxz&%2X$PttQS>En|8
z!n!Y#y!d?5ay3<6^|3-;PzbTK@@n8lapQwh2<3hA$m97T01w)qdfR{7zt(<at!QMW
z{Yc#NTk`JSrJkh=w?>vm{+YaqEAJPLMAg638DHMFhApa9W9N;1H}<`hecRA=-q&@)
z{xTG7=X@Yp8eG#~!Xu5H*%&*&GIoA#Y--KuUKyME)zIC(dtK4+D;G9On(mi0tzesG
zwdv?;$<h0!qW}<<uNd8_3CG67)XK!v+JtA##I8(ues%FKdvAP6@k(UF+I-*Iyn-!`
z)#jd6YtMai4**2T6%(7XSYygphvIed>F@1cx3n!2>ky;Vq>9U8jjuZ5y@`_V4X+pP
zS>C%|+!fVsTFW0|6JWEoV_9`a`BT$f>%EfSRIjz3d?bhJM@Uwxjp|{F74`SaJ8sr5
zJEN+U&T?Nz#ZNDntu=Jtxq8pFe)L3gY;t4l?8?~L<ayWnm}~8*D^=PM_b$uh&gJ2|
zm3NKzJUI~hT=JYJS;9t-XLshhzB*;Fe%H2Us7sYr#UhEiWL0}2kSsX@Kx+LB-E|#=
zdH2OGzi}ubPxSv-w>*BQ@u!ZvUH6o~Ih;IsDrq^rp+58Q%LJUU2j>Z-DNCufDJ{f?
zZ=Jb$W@$J{%_h}eT&F&tQXhI?P~kKE$c$RrSJVwrGCH+XyEGN6jm@NVhL^fi`hpw7
z*N5Slcw$Wa=<@z$+nvs2*^zhkM}MDgN)-a`rT$o5Y${$9I~AWubSB)1=H-SvvODxI
zRd;KCsk_^o+&_4)=pLQ4kKG$cj-O7RaU{>4Po8!qUDL@KZ_?*qpYbOzaqBa%qvw;;
zSCUtre}K?6qE`mlRiY1PeX`>?D;SXdw_Ij0KGXoo5I~2+e(?Iir3oCa_N?n_7kf8J
zO|<?SLn(zisjFC5RBVt<D`ZoGgz$PwrF-$}i&qz~ZbG!a{#NVF*0>zR^O~VPp-SnC
zvclnGIWji<L1lnTSkeq#-`3pL{G{<0MR%B=SFCpStv2^3n~o=q18eHR-({elTK4?C
z_iw#-m8(x5Pd+t}G!Cw*pZlG;<juzD-Xz%+cR}*Wqq79Ej1r$b(j$|H_~d;P`|+~`
z3<WfN%Ht0_e_LB|dbj#t+FMTVRsSob`_&sYg{PlUy!VXmj6ox)oldM}oK72QQ$+OX
zul?|ch47&V+?(-N5)orM?+|M{S_E?1#fSdzo!9Vl`zUlP&2+oq#a}tLx#1$sx!St+
z@Z6g~L8Ls>|1-@UEnFM`IDF>;G+O+%UmFkK56;0y98iv!aoW++A=d5NBf-CwhBtz<
zU}4JTzL=p`Exkv~Qc0p&9B0&K7F15BC*XEExgZ``G^oJjN8B}>VACBelm(una0x!=
z3C;SvQ)z1pT$zY_0VmJm1gmAtJ?u3AbfPN*-?_Li<3w~0s1MHt;44t}2$z65e97e<
zkif?S1o4p66Xc_EM3~=4y7!UheN-U+YvJ|ZQ0a$8g+aEcf!$~*Sk!!|tRs{k4imEk
i(fojsBT^rYtjWu;Y8?35!1eK$nRiLcL#QtoNB>`cHk^|H

literal 0
HcmV?d00001

diff --git a/scripts/verify/__pycache__/perf-build.cpython-314.pyc b/scripts/verify/__pycache__/perf-build.cpython-314.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8f5e413151fa7a7918730d5674a16cee16cc8e07
GIT binary patch
literal 4426
zcmb7HYit|G5#Bo<AEGFc5~;Ujoou}*nN;jJk>gh#$C54Cmgoyj$E_mfbUaCCk$302
zqir!Zp!`V~qkxnGL1Z9>)uKR@J`fj3Q3nkW1N~zJXhB9~sGd{=X!|GVA0>8yw&;(}
zl01`goW@;ZclLH?W@mS1XFv8=SJ@GiJ70cw(d<CzulU0#WU+H!gU$kCkcK7^LyQm_
zk*B0a=BY_D<te36d1}_oK*<ryNvmdUM}EYZ#|g~_)B?1EF*P8~{&FQl?SBcO2&~_V
zUbc6jeZ}S4VU1n+>b0^~0-<AZAsSI>ZI-8baYhh%T8#@D-=os}RasKB-L%T{b2KYP
zXgSKxPqEw_t?{ZxPYEKc2)s&5)3hY=_A8Pyrzt#7UlFv~ygr#%rhS)rMVOwaWks46
zqI_NtC&^e2mW<I-9IqhJEG)xGu*ej)$1aF@4JsSsX^mB8c#Vz-3eRbhGEZ}?C`uZw
z@L&)J^PtCXx6@GaE9N!$<$3!2ICCn<3=Pra=YrwmBMdP3ju;;UhW-AFswCRSyo62#
zSZ$Uw9E7P2stM}-V^-{>2{^ApnK0ak0+caUqLak#=F-=`qErBMJX#F5Jh5i-|JAIx
zLPVhA(RC4o0Q|_o;yxfOOnzb_AFh~poM5Oif-yt2K(#`(L9Kvlhgu2MF-FMb1cY^=
zxX^GlLb54DGF2gypWIz#%)lsl%7st`+JPp3r%(bg@@VNm=jk$z+DGS$R#y+23o|k`
zfMj!s3V6QB8EE+Cm~q-JG(-&l9PF~RpovC<FHq()W2^&(EQbGNYtR~KH)cwdaT*+q
z3tze!h?rJ%>S=_&g}=t&?<p!NN}R?LOl`S`8F%?t>9n5%$)eN@DvL{yuPK)>R+4Q&
zg7NIGELkmA1S`HJF6su05!;R~EC=Y4jM3Ys$3C&s7!PzBy%Oy}7jEo(7-bX2mQm1d
z*mmLk|Jrjms0!Kwy(NuC?@7nxO4xh-Hk%z6?mVgVckO?>=3wRaZ$Bwl1*;g^h0ayS
zNk58{!w}V%{vNC@w`aS|7<mff8Q+#TDOYd1N1zMt*-408XV4kwH%_8NjR-f6xfmlY
zaE6X2;g6JJ*AeiQ^cuaO6S77_DQmRDO**w&uoim1D&F$4tZ>d%+&9h+u8=Eaeh#9m
zXczE>T(I9_c93g;qsFj30{5{Mm1SDLVMU2{Af_=?<3M{!BbwUFC7vpA8a<{d|1iQq
zwW81?aq%Q4Gy{3%39M=^*#oPzlzTe>nc>k=mW2!#ma)JtTT3@5gbdwGTgW^>PLj+H
z!z0EFoWh`?;Jb8(%#YW=wC_T6d46m)G?u6ZMJ)I{+EE)CBC-DkbkNj>1R_qh=i^s^
zHs5mz$lLKH&MZMMsas`M;YCf=ZTwY1)xy%8Zsy>jmvjoFi-gVRi%Q%aA4!ltIbr8m
zxrihZ6+T}POe8AH5C)bIC7%)(6E2@G#;Ynj!~3QsCB|wxh2Qvu#plDiy_9a^u0(W;
zszm^P;03R0u=*^|&FL0}XEARLC&gp{EQ*g+G+~<MG~Lc$<+-@VPDOd$QUE17B?3ax
zsR%EtUP6JRK)QWcfiaARU`Tft7{h`Zeh6R^4KYDg1#xCOkUBu?mI$j{5yZqPQKE;3
zV=_NOm$a23AQ3F4H7SpgO7Q3a4ZEjRP7!2H9Z-2KE(5qpRBb~#y3GJ;y3LpXJ1v43
z0e_5P*C>1x4uRLr5q>H@<84)N`qoYAysA^+qRW^=m7<q{BQ^)S(yakze0b#akZuh=
z&zu+vzNp*P_!M}F12^a<unMr%NMK08?+?t$Q9+9W)KnD=OmuTpVk4^V9DQ+o>^Pvn
z2tTAKlA@ac_9-=(!=lc{H6aR4lH~clV2Y;bmO=#SmV5xIpmsh6O6S5F#1M>Wx()E5
zh9f|?Ur_+lh6|34#AC9mn`K21HE*p_pWhq~1S|MF20&NA&;&<<8jorU#t}M&&4ryv
zSHKo2u5i49(Fyjv3XxHCGCm8;Ij2LwiV-2gLcBq!7qG8xpMDq_IswZSqd>))FnZly
z^j5=@1qeN`4czKfj1@%X7>MF)shS~iFQ83N<Gb!#jW-)tI)2=;=IP3Ld})tw%`@<!
z=R@wdv%j9bbLls8YkQ96_Kc?YjIQlDo#fUj>l-7ljVwi0f(s)Vs&Cy{zlD%<jC{AD
zKv%j8bhWQQZ$}FB&glZZ*I%Hi^TmnOv|Jd0RwahWhaV@%GsTaE6JOLeB`xbV=hDgL
z>6E)GYwNmU{mO*g&3~(ESe(2uxqRZ|zE1`|8b}3#sfkxIRqVp>y0d2C<hrZ=ovOF0
za<1OAt2gKBOHx~;siJPPw*H;IAN4H<*J^hrU)ZuBhjVf8&B3MpYn81j`svRrpT1-I
z#QKr-t|PbqOlJR?6mu>Wn#?lc)GO0jW+pSIe9kDFbxq5=mZRC`{<XT@$>DW({c`==
zhmyzF9j?XEH%C{f<Y>mxweD`X)qk@;>+VV(-*nV2p1yH<xjy6Q-t_Oe-T4b4d2+e&
z?Uz@c&DQm$9sOH2)J)%MyV;g&@}--6xu$*TrhUm#thzO?VZFKYz3V@@p6fZ7?m3w2
zIhyV{nr%LM_fWQZJazU5smY7kvlmlMb0FfVPmfKlHFMD39bC0$TLy0L&b1s#wH$$w
zbTgNAaLLh)9lcBTb$46N-I;cG=G@&GclWCH!K?4T`oZ;IUSIb#zBB&T_{!+r<4fZi
z&&Y<S>46osd%x;IHU0naqn4d()q^*Cmd>u6U7byLJ-c`*dFf`)!iiM%;1|~FO<U!n
z?S?Jo?pqyP^`>g}Wo*xETJ3LKd+plN)r_@$)xCNx>pht3eIecZLauk{v)&;?q-HW>
z3$L4P$?s*&%{i(&O?CfFdoS_R#O(tgt9SSR?)trpsqasv_HwCcEYl-qC~1?b_+jT_
z-;KVdmsZ$}z4vp<`_Bi@AeSfS+?96jdVq+Ex($bC%LMd4U++LI5#nE8+mUm>iZST#
z`|HmfAn$cFjy2dmZL1h-r9SPqK>LRSbz>g$AKe7Bx-A^WcPJe8ns5@qe|B{W|Jlta
zj{HAeul?Wftib=$X<;VOUA)KsvRB71h_mnqS5pt~EN0D}$_X$a(e^mzs3skrW}_-!
z;OTrE4^DqGyy%OISnHl=3#?s_V--YB;bV}s*=V`imdEktQ~BYiybj$O4o4&|94_Qu
zQc@L+t(6Yk08T{sY1U?=!c;!(D>NR*|9bMd1|J%J^1|Ww(R~<1%Jb0ZwnH%~5|8pn
zlri|kZ@hXO8t@K5d_|cF>b?UJjtx|?fvPr8%?7gn88!aJ-n(G^+GeRFfz);peOmzn
YQI2FHV`};nWqW-X@&VPXoB>7u1;_^24*&oF

literal 0
HcmV?d00001

diff --git a/scripts/verify/__pycache__/perf_ab.cpython-314.pyc b/scripts/verify/__pycache__/perf_ab.cpython-314.pyc
index 4615bd1af2b591218ef845abff623995a57d900d..672b6f1fd8ff4a52cc0b2df5a68c83c29e5a0993 100644
GIT binary patch
delta 2675
zcmahKOKcm*b#_TEKN3Hp#GfdV%b%##hmvU7(I<*vqjnm|R@k*Xj2y`{Wl}a%N?Od4
zmE_h{4sD=ZTszYOv5`2ww6NPA9IziBdPrLoFwiz|j5u-j)S_seUJ9e99tyN?mZT-Q
zO*_E8dH;F8nf(`jKj#>5*lh^bkv~0n?T6v34nOyUtEp_t{4!Uy$%bm1Z1kaOyKDkz
zUvkL8_$vq%yAkR^OF}kGM@(J(lKtx45NCP+lQ($oaec)g37m&su_ib-{i$_G@Pl{K
zO+UBFrXzs%)n{yf=6H*}-rHX1J(~z^m_Xq=`GsPaImLnbY~(E=v^@p_9wEi4xN<KI
zwy_u+OSWTYJIe5Tc}hp_^C-%_w-{|{VOQ=)xnJXsF`>Z$%-FyN9W5R3?wCD<kQIdy
zcBGo@^8+k9`iAo>&Tk<kThmCkrO`02xanV<!`q#>^9(=O^vX=mAiX9AW+hLZ;N<6(
zF5Klp$F8yH_s5Pq+mY<j<2US&e>d{n@F3K;wf1MBcx>3h62%_eL2tMw3~ubCnkxzw
z|EDWB!Q2A}U?=v-U3(qL9y^k~!1etH*N;6k+p%iiKnS~8^?TB%oTt2}d^O`Rs`1Cf
zr>*eLJF(Y=K*Fb)q#aW9VIOt4gC5^9KO1WlQ}HWaXq*rG=?m@<-SRl;akqnRx~I&4
ze|m<Wol!IBCnR=HfPDVuPNv`#m;il^ag81P_cr|pz`M5@>lgc3>r5v9x1Vm^Pk>(Z
zgy|=p!OkG=mTi0MgS+XyPKoQG>8?PBTej|#x#=rik`%)BQlt4IG(W0@fzGmFz(H&u
z|1MmYmvkybZ*)yNV>l?sp%V!lf-WWL6Q|b(NN>grh626Z;iaO-9`9MqG_yy#3yVHo
zDcqwQ_hk%9m}Wfz`ld&m?tUh$XHy6t5LzXSpreM_uQ!S)OLD(FzyjEY00uJ#Acy31
zT2La`1mjzUp1_zZRz*~zN=%7EfZ|F}&n1$Lx5C@q<zZ|EPa`-Ca5Q7uJ#@AYy8gvB
z+epq}Q6JC)INI8=xc7#f;Rf}0s&Sk=1|`hkSW`FN+Hvgv+BAae*c05>wwnbxm_4|g
zO$)=3ma2Sku*HO3Ni0e&5+_<Z5rmEMp}j$5pKEFD?)&t)EH_S1`=ZgQmICIo<1Ku5
zm!G5W_=15XPG+Zfxh)JQ=zZV8__#dr-;!9IY3VH<rhWcd(<IZQoBoBAuM8JCf@(&^
z$1ZJ?GCYgZfA}SCn#KaloUQ)tz&E+}0JtRVrJ5&p>I|L}SUE^P-3$)(z@v)D1OUxg
zEtD%NdoFuni6S{ne;JH$Mf%6!U#E`)`=T%@o>OWf@iD@X&a7NCtKvqXQV|LJbug;5
zS+0tN9Vl^{X{AzAFTfwg9J&->Msv5(ZC>NkBO0fw@XJ8G?38yyDef};O{m(-yv&ia
z07(JBQx(Q2GEf5W9hwh^xD|RS{8~ECm}db<2C@vB#w}~y1r;tixCY;_?;5^EJ&{Q}
zDJkouS}Ka`Yn83~iO2<xo1;GyN4OL8u{h*=k(qoMfW+$?VGJos2cnY_{B96olSNqU
zx`gPZWcq|nq-I#JRY`<ij(U5JFj^=mN@=aAnMh&nd`UAYg)&izOh1Umw{wS&AJ69w
z%X4I&soFM3xmwCsOPf{Atkzan3uKGLn8c(@G(pt^*9>Z@D)nlDQXp!HXvP9LuWBZ>
zTr8=25JY80MzX%4YQhRMwN%uEWwQR&(i&L+5jn-c0X-29L)r7KS)sN0N_n+h)r6W_
zSgw=^W<vJ5kwf%ytXs@6JtL{DspK3(Eo!w;t*LM<YcBEAd$D;VJp718;|YJW{+e;M
zP+lX~fjtMGdZ_+tJjfjxnYgv}k!4`Vc%^t#`a_3jM|czz-<!HNbz}O?*`1?T#vXb6
z-#dNt=&j@}-_7BNp3$9SkJ{Z=N^gJZ$NoEsyTS*>`cD$C^P|DUgYMz0i97in^P{$|
z2W{fbg*$<dlaqJ9aj)=j=H>e*bDy4E`sn1+{j-IKCkr3X6zF_vlOLI=f0+6?HyyJt
zgl)eab}t0nzvIk+|GuYxA!+##;DF^*r+Z=4^jW|J{Le<)PIFexlFzej@_ET9ndq;i
z!M4ZOud#J|2IwbJkb8%=_ls@svU;#71a783*`RMU=%xOlZ6=?ST$(wbFRq`<=gAr~
z(C4d%VOtEahZ~x`r4o^0W~ETQP$@4rh7aC+L@qJF=1eayi*Bv3S^{_M0ygf^s|;iJ
sK(jo*x?ZeRN-vP_TY+Z3YITu%2m0C^f}MZe`ox9o9dvGB$jEH|4Ryc}e*gdg

delta 2462
zcmZ`)U2Ggz6}~e&vwz<I{~xb+cGnww{IgCrc77bYsns;WvK^bLa~Gpcm$ALEUDoz+
zXIDx8hy@Zd5}=mb2U1XL#7h%|(rO=|qNqi@M2ZSD6p;>!!~=aOFO(NV;Q`K_&EO3Z
zSDJJ0J?Gr-eD~aY&%9OnH}(ztylw<-{lUFw1~V`F65Oqq2bPANi(J#K*qS|x(DW!`
z95ua)9i;EFUvZrJ0{Ev8>P46BOA`HCC?Z|<<<L08fi}){`Yckog89j=x9)v|cW=CU
zgSDGyH`*&U+0I4i*IoTwkiOv>1#E;!lK#=9h%+!e(k{C{=J<bj+DT8F_iG4U6~N(D
z@{Yo3Ui!e7TRaXr9&~@L%#QSQA~LCXvCo6be78;uRPN_c<+W<g@~ix_@(#A)z_`7m
z!2!DHkM?-yCJ=I=K7_q=*I)C04e(qgq_|6HjMswnfq%>v!XfGpq_0mdaLO4ijKcwR
z{suyt?@=B&<UvZn<YIf2OBajM2&B<!l^<7rb;wAIU?<~@BcOx5vo_G_&_)O$dNWY8
z1+kBQ5Xha49Ok?#KXgLa1q^~Fj}YUbgmFZPbVpR8UZliugdPh<;WRD;x5XAhI7n}Y
zve$i=qL*SVVGOnS^HL|*noo-^`*1XXz`~c9#REH!;h2){#=XeT`8s)Q@mg$^pUZb<
zT3m|)MhwRlSGWJusCx!h_ehFXLQjdGT5RILre!m^^dEA8-G3Y^a#7k7&Nvc`U5ESs
z^{$6I-q|SgO|Uptwjff1z8jXg6#YqfYN$G98I)`sRgYOS9DI~I1YC^BnO;TGGB{N&
zTYjfrfj2VC-U#lcKa5QKb2z2sA>9Ss3rEpUZ%1No7!=D6SZJrgSY#Z#4v%pmgws{)
zMIjT-$<UM|IBnt#lx?t3WD6tM1$%iUqp+9T;Xe9&G?>X8w;#ejX`9nKXQ2!#vNFVI
zl!3-@*#^@QWwd11`mhLBx(R89$Dz5jtR`tWEf3MMSJLK=`XFjFTQV=$7|y`@qh;|h
z$lM?#)AUJqe2(F)dDY5TvU;4Q?}t+<<v66|7|uOZ=dB*+nA!8tY(}8JiA8PWODE|s
z?ctd|E1EJsVx=5<t<ZJ-3zkn7VL_SbeuPV>8H&H#gIk)Um*Y9_v;~Kmh7-cv3Ho|G
zE%xL7rL$}mSD?R(PZUlnr<P7yzK4C4RWGwupP<8uIq@{}W9#U566XhETkvSIs3mAI
z_7on@Z2vo^|4PW*6fGuKIahl>`Bl!708Ekuu}nNPlwPw#juDaR^Xbtks9A`d0$~VE
zb*rJXnsx-5B$3CNR$SYxuYK7FZ>w87rK!g5n%dAGE1Akw`sef~({teUoPAPS(^?XV
zv5BozzVe(ym#(UfhD2C#X0!UWt)@g66`3={D~*=E2{qfDc`?ZrUA&2I@&;czYH+#^
zl^Gp@`1Bw%z<rC}$u!R~l#8SWl6(ooe_fbjjfr&--)5Ewh)>WS`t9J%+vnc=${Qzc
zCvPQxe(p0;r+a<Rl%8hJ3`e#R6*_}kHMmXP219Glhk!59M0T>RX0LGE0=*+0<>u*U
z(rElV!??gOc(Wr+A?N9IZc-j%wqYjNE=&tC)6M(|dz6N4x78$l^dJ}OWe0B9RZXk!
ztQ#UxcQ)#VsHs~-C!eSHa)sCvOP6Z%vkSA!hO06&KVPfNC>P07-_0jau-}m(UL{-2
zx?$IMNwdCg*z|f+E*f@CC3>A0f=V`YL)5p{>$(}55W1L8jleG$k!2=W$ssIb1pbJ?
zq@}B?jXJr+G^}iq33`xENthW0(%R9<DjPcWrrK=jfXdE}#Ho>A=8ANzFc|CH8i8FG
z0)I9%51*cE?-bHprdT?d{lF>h3ootzDHz?i-%m@gpSf}7wdrrp?LSFx^=E|lggc*W
z-|gSyrEq-Tala?>%CjH#NC(w7XMSIt_*vxL)Z3}|)L-;{SiC@A7`Vn4OYOf8{FIxP
zyel#H-O<oWFm(5<1IF*ChE_7pUxO?-|KJF%$l^yq5%eF)Ju9bOhO<^<1Zy=}kOle|
zd4z{9H<aeSPp5|@`wxIKC^m<R*6@2nqt{Js&~Vgh>$_{U8rf!BH;)?%8zR@3fPWR7
zul}q~vTWqg)aGVmYqfJ1pm-wk0uwJX!Oq@H5u>xCZr1^mJ&(>U{~{Yb4#IFwZSStP
p8s7S2<VP^);L}yQFgzF#>|TD)^=SlogL|&~o&en&9u?Ta{{Sce_OAc{

diff --git a/scripts/verify/perf_ab.py b/scripts/verify/perf-ab.py
similarity index 94%
rename from scripts/verify/perf_ab.py
rename to scripts/verify/perf-ab.py
index 452b541776..2d492060ed 100644
--- a/scripts/verify/perf_ab.py
+++ b/scripts/verify/perf-ab.py
@@ -6,7 +6,7 @@
 system, from the kernel's rusage) and wall time. CPU time ignores scheduling
 noise better than wall time; pairing cancels most of the machine's state.
 
-    perf_ab.py --base base.json --head head.json --fixtures DIR --out DIR
+    perf-ab.py --base base.json --head head.json --fixtures DIR --out DIR
 
 base.json / head.json map a test target ("export", "seek") to its binary and
 the crate directory to run it from:
@@ -27,7 +27,6 @@
 # +0.8%, seek-storm +0.9%, export-jitter-camera-40s -4.4%. Real-time playback
 # CPU varied by 15% there, so playback is reported without a limit.
 WORKLOADS = [
-    # name, test target, case glob, tier, CPU limit (None: report only)
     ("export-basic", "export", "export/basic/*", "fast", 0.05),
     ("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast", 0.08),
     ("seek-storm", "seek", "seek/*/storm30/*", "fast", 0.05),
@@ -108,7 +107,7 @@ def check(name, label, result, report):
         print("\n".join(tail))
         return False
     for name, target, case, tier, cpu_limit in WORKLOADS:
-        # Untimed warm-up per side: generates fixtures and warms caches.
+        # The first run of a side generates fixtures and fills caches.
         usable = True
         for label, side in sides.items():
             report = out / name / f"warm-{label}"
@@ -133,11 +132,11 @@ def check(name, label, result, report):
         if name in broken:
             rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases"))
             continue
-        statuses = {label: [s["outcomes"] for s in runs] for label, runs in samples.items()}
-        if statuses["base"][0] != statuses["head"][0]:
+        outcomes = [s["outcomes"] for runs in samples.values() for s in runs]
+        if any(o != outcomes[0] for o in outcomes):
             # Correctness is the suite's job; a case that now passes or fails
             # can change how much work runs, so its time is not comparable.
-            rows.append((name, None, None, None, None, "case outcomes differ: timing not compared"))
+            rows.append((name, None, None, None, None, "case outcomes differ between runs: timing not compared"))
             continue
         cpu = {label: statistics.median(s["cpu"] for s in runs) for label, runs in samples.items()}
         wall = {label: statistics.median(s["wall"] for s in runs) for label, runs in samples.items()}
diff --git a/scripts/verify/perf_build.py b/scripts/verify/perf-build.py
similarity index 97%
rename from scripts/verify/perf_build.py
rename to scripts/verify/perf-build.py
index 72702cb5a0..e970415eb7 100644
--- a/scripts/verify/perf_build.py
+++ b/scripts/verify/perf-build.py
@@ -3,7 +3,7 @@
 worktree with the perf-verify profile and copies them out, so a second build
 in the same target directory cannot replace them.
 
-    perf_build.py WORKTREE OUT_DIR   -> OUT_DIR/binaries.json
+    perf-build.py WORKTREE OUT_DIR   -> OUT_DIR/binaries.json
 """
 import json
 import os

From dfad1c164c1bfa65e6244cf59fae23aa04af13dd Mon Sep 17 00:00:00 2001
From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com>
Date: Sat, 17 Oct 2026 12:45:08 +0900
Subject: [PATCH 5/8] ci(verify): judge the perf budget on per-pair CPU ratios,
 10% limits where Blacksmith noise reaches 5%

---
 scripts/verify/perf-ab.py | 37 +++++++++++++++++++++----------------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/scripts/verify/perf-ab.py b/scripts/verify/perf-ab.py
index 2d492060ed..6b5b1bfb97 100644
--- a/scripts/verify/perf-ab.py
+++ b/scripts/verify/perf-ab.py
@@ -21,15 +21,18 @@
 import time
 from pathlib import Path
 
-# A head build is slower when its median CPU time exceeds the base's by more
-# than the workload's limit, and in most pairs. Limits sit above the noise of
-# identical code on blacksmith-6vcpu-macos-26 (run 38995088761): export-basic
-# +0.8%, seek-storm +0.9%, export-jitter-camera-40s -4.4%. Real-time playback
-# CPU varied by 15% there, so playback is reported without a limit.
+# A head build is slower when the median of its per-pair CPU ratios to the
+# base exceeds the workload's limit and it is slower in most pairs. Adjacent
+# runs of a pair share the machine's state; across a job a Blacksmith VM's
+# speed drifts (one seek-storm job ran 10.1 to 13.2 s per run for the same
+# code). Limits sit above the median-ratio noise of identical code on
+# blacksmith-6vcpu-macos-26 (runs 38995088761, 38998485953): export-basic
+# 1.1%, export-jitter-camera-40s 4.4%, seek-storm 5.0%. Real-time playback's
+# CPU varied by 15%, so it is reported without a limit.
 WORKLOADS = [
     ("export-basic", "export", "export/basic/*", "fast", 0.05),
-    ("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast", 0.08),
-    ("seek-storm", "seek", "seek/*/storm30/*", "fast", 0.05),
+    ("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast", 0.10),
+    ("seek-storm", "seek", "seek/*/storm30/*", "fast", 0.10),
     ("playback-basic", "playback", "playback/basic/*", "fast", None),
 ]
 
@@ -113,7 +116,7 @@ def check(name, label, result, report):
             report = out / name / f"warm-{label}"
             usable &= check(name, label, run_once(side, target, case, tier, fixtures, report), report)
         if not usable:
-            rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases"))
+            rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases", None))
             broken.append(name)
             continue
         samples = {"base": [], "head": []}
@@ -130,17 +133,19 @@ def check(name, label, result, report):
                     flush=True,
                 )
         if name in broken:
-            rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases"))
+            rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases", None))
             continue
         outcomes = [s["outcomes"] for runs in samples.values() for s in runs]
         if any(o != outcomes[0] for o in outcomes):
             # Correctness is the suite's job; a case that now passes or fails
             # can change how much work runs, so its time is not comparable.
-            rows.append((name, None, None, None, None, "case outcomes differ between runs: timing not compared"))
+            rows.append((name, None, None, None, None, "case outcomes differ between runs: timing not compared", None))
             continue
         cpu = {label: statistics.median(s["cpu"] for s in runs) for label, runs in samples.items()}
         wall = {label: statistics.median(s["wall"] for s in runs) for label, runs in samples.items()}
-        cpu_delta = cpu["head"] / cpu["base"] - 1
+        cpu_delta = statistics.median(
+            h["cpu"] / b["cpu"] for b, h in zip(samples["base"], samples["head"])
+        ) - 1
         wall_delta = wall["head"] / wall["base"] - 1
         slower_pairs = sum(
             h["cpu"] > b["cpu"] for b, h in zip(samples["base"], samples["head"])
@@ -151,22 +156,22 @@ def check(name, label, result, report):
             failed.append(name)
         elif wall_delta > WALL_WARN:
             verdict = f"wall +{wall_delta:.1%} (warning only)"
-        rows.append((name, cpu["base"], cpu["head"], wall["base"], wall["head"], verdict))
+        rows.append((name, cpu["base"], cpu["head"], wall["base"], wall["head"], verdict, cpu_delta))
 
     lines = [
         "## Perf budget",
         "",
-        f"Median of {args.pairs} interleaved pairs, test process CPU time (user + system) and wall time.",
+        f"{args.pairs} interleaved pairs: median test process CPU time (user + system) and wall time per side, and the median of the per-pair CPU ratios.",
         "",
-        "| Workload | CPU base | CPU head | Change | Wall base | Wall head | Change | Verdict |",
+        "| Workload | CPU base | CPU head | CPU change (median of pairs) | Wall base | Wall head | Wall change | Verdict |",
         "| --- | --- | --- | --- | --- | --- | --- | --- |",
     ]
-    for name, cb, ch, wb, wh, verdict in rows:
+    for name, cb, ch, wb, wh, verdict, cpu_delta in rows:
         if cb is None:
             lines.append(f"| {name} | | | | | | | {verdict} |")
         else:
             lines.append(
-                f"| {name} | {cb:.2f} s | {ch:.2f} s | {ch / cb - 1:+.1%} | {wb:.2f} s | {wh:.2f} s | {wh / wb - 1:+.1%} | {verdict} |"
+                f"| {name} | {cb:.2f} s | {ch:.2f} s | {cpu_delta:+.1%} | {wb:.2f} s | {wh:.2f} s | {wh / wb - 1:+.1%} | {verdict} |"
             )
     text = "\n".join(lines) + "\n"
     print(text)

From aa2ea182fc4c6f8fa2c8c15984b2d949fd4adf5d Mon Sep 17 00:00:00 2001
From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com>
Date: Sat, 17 Oct 2026 12:51:12 +0900
Subject: [PATCH 6/8] ci(verify): measure perf budget changes only from this
 repository's branches

---
 .github/workflows/perf-budget.yml | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/perf-budget.yml b/.github/workflows/perf-budget.yml
index 57c4595fb3..999f0b4633 100644
--- a/.github/workflows/perf-budget.yml
+++ b/.github/workflows/perf-budget.yml
@@ -36,6 +36,7 @@ jobs:
           EVENT: ${{ github.event_name }}
           PR: ${{ github.event.pull_request.number }}
           DRAFT: ${{ github.event.pull_request.draft }}
+          SAME_REPO: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
           RUSTUP_TOOLCHAIN: stable
         run: |
           set -euo pipefail
@@ -48,8 +49,14 @@ jobs:
           fi
           gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$RUNNER_TEMP/changed.txt"
           python3 scripts/verify/plan.py risk --files "$RUNNER_TEMP/changed.txt" --output "$RUNNER_TEMP/risk.txt"
-          if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt" \
-            || grep -qE '^(\.github/workflows/perf-budget\.yml|scripts/verify/perf-(ab|build)\.py)$' "$RUNNER_TEMP/changed.txt"; then
+          # A change to the budget itself is measured only from this
+          # repository's branches, so a fork cannot queue the macOS job by
+          # editing these files.
+          own_files=false
+          if [[ "$SAME_REPO" == true ]] && grep -qE '^(\.github/workflows/perf-budget\.yml|scripts/verify/perf-(ab|build)\.py)$' "$RUNNER_TEMP/changed.txt"; then
+            own_files=true
+          fi
+          if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt" || [[ "$own_files" == true ]]; then
             echo "run=true" >> "$GITHUB_OUTPUT"
           else
             echo "No engine or high-risk code changed: nothing to time." | tee -a "$GITHUB_STEP_SUMMARY"

From 9b078cd56ec7ff17bf459e7c2bf324376a19cf06 Mon Sep 17 00:00:00 2001
From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com>
Date: Sat, 17 Oct 2026 13:20:53 +0900
Subject: [PATCH 7/8] ci(verify): perf budget fails past +25%, warns past +8%,
 after a false +10.8% on identical code

---
 scripts/verify/perf-ab.py | 39 ++++++++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/scripts/verify/perf-ab.py b/scripts/verify/perf-ab.py
index 6b5b1bfb97..422f3b0697 100644
--- a/scripts/verify/perf-ab.py
+++ b/scripts/verify/perf-ab.py
@@ -21,22 +21,24 @@
 import time
 from pathlib import Path
 
-# A head build is slower when the median of its per-pair CPU ratios to the
-# base exceeds the workload's limit and it is slower in most pairs. Adjacent
-# runs of a pair share the machine's state; across a job a Blacksmith VM's
-# speed drifts (one seek-storm job ran 10.1 to 13.2 s per run for the same
-# code). Limits sit above the median-ratio noise of identical code on
-# blacksmith-6vcpu-macos-26 (runs 38995088761, 38998485953): export-basic
-# 1.1%, export-jitter-camera-40s 4.4%, seek-storm 5.0%. Real-time playback's
-# CPU varied by 15%, so it is reported without a limit.
+# A head build fails the budget when the median of its per-pair CPU ratios to
+# the base is more than FAIL_LIMIT above 1 and it is slower in most pairs;
+# above WARN_LIMIT it is reported as a warning. Adjacent runs of a pair share
+# the machine's state, but a Blacksmith VM's speed drifts within a job and
+# between builds: identical code measured up to +10.8% (export-jitter-camera-
+# 40s, run 39000364369, pairs +13%, +11%, +0.3%) and seek-storm runs ranged
+# 10.1 to 13.2 s (run 38998485953). The fail limit sits well above that, so
+# the check stops large slowdowns without failing on the machine. Real-time
+# playback's CPU varied by 15% and is only reported.
 WORKLOADS = [
-    ("export-basic", "export", "export/basic/*", "fast", 0.05),
-    ("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast", 0.10),
-    ("seek-storm", "seek", "seek/*/storm30/*", "fast", 0.10),
-    ("playback-basic", "playback", "playback/basic/*", "fast", None),
+    ("export-basic", "export", "export/basic/*", "fast", True),
+    ("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast", True),
+    ("seek-storm", "seek", "seek/*/storm30/*", "fast", True),
+    ("playback-basic", "playback", "playback/basic/*", "fast", False),
 ]
 
-WALL_WARN = 0.08
+FAIL_LIMIT = 0.25
+WARN_LIMIT = 0.08
 
 
 def run_once(side, target, case, tier, fixtures, report):
@@ -109,7 +111,7 @@ def check(name, label, result, report):
         print(f"::error::{name} {label}: exit {result['exit']}, {len(result['outcomes'])} cases reported")
         print("\n".join(tail))
         return False
-    for name, target, case, tier, cpu_limit in WORKLOADS:
+    for name, target, case, tier, gated in WORKLOADS:
         # The first run of a side generates fixtures and fills caches.
         usable = True
         for label, side in sides.items():
@@ -150,11 +152,14 @@ def check(name, label, result, report):
         slower_pairs = sum(
             h["cpu"] > b["cpu"] for b, h in zip(samples["base"], samples["head"])
         )
-        verdict = "ok" if cpu_limit is not None else "reported only"
-        if cpu_limit is not None and cpu_delta > cpu_limit and slower_pairs * 4 >= args.pairs * 3:
+        mostly_slower = slower_pairs * 4 >= args.pairs * 3
+        verdict = "ok" if gated else "reported only"
+        if gated and cpu_delta > FAIL_LIMIT and mostly_slower:
             verdict = f"SLOWER: CPU +{cpu_delta:.1%}, slower in {slower_pairs} of {args.pairs} pairs"
             failed.append(name)
-        elif wall_delta > WALL_WARN:
+        elif cpu_delta > WARN_LIMIT and mostly_slower:
+            verdict = f"CPU +{cpu_delta:.1%} (warning; fails past +{FAIL_LIMIT:.0%})"
+        elif wall_delta > WARN_LIMIT:
             verdict = f"wall +{wall_delta:.1%} (warning only)"
         rows.append((name, cpu["base"], cpu["head"], wall["base"], wall["head"], verdict, cpu_delta))
 

From 6a73825ede19e2c92ae6fcc7ba9f5f13f8c29dda Mon Sep 17 00:00:00 2001
From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com>
Date: Sat, 17 Oct 2026 14:27:25 +0900
Subject: [PATCH 8/8] ci(verify): perf budget compares runs whose cases passed
 or failed as expected

---
 scripts/verify/perf-ab.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/verify/perf-ab.py b/scripts/verify/perf-ab.py
index 422f3b0697..2d48a702ca 100644
--- a/scripts/verify/perf-ab.py
+++ b/scripts/verify/perf-ab.py
@@ -70,7 +70,10 @@ def run_once(side, target, case, tier, fixtures, report):
             except json.JSONDecodeError:
                 continue
             if row.get("case"):
-                outcomes[row["case"]] = row.get("outcome")
+                # A case that passed, failed as expected or passed against
+                # an expectation ran to the end: the same work either way.
+                outcome = row.get("outcome")
+                outcomes[row["case"]] = "ran" if outcome in ("pass", "xfail", "xpass") else outcome
     return {
         "cpu": usage.ru_utime + usage.ru_stime,
         "wall": wall,