ci(verify): perf budget for export, seek and playback

.github/workflows/perf-budget.yml

@@ -0,0 +1,146 @@
+name: Perf budget
+
+# Head and base are built and timed on one machine, interleaved, because
+# Blacksmith VMs have no hardware instruction counters and machine state
+# drifts between jobs. "Perf budget" is the job to require.
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+  workflow_dispatch:
+    inputs:
+      base:
+        description: "Ref to compare against (default: the merge base with main)"
+        default: ""
+
+concurrency:
+  group: perf-budget-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  scope:
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    outputs:
+      run: ${{ steps.scope.outputs.run }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Scope
+        id: scope
+        env:
+          GH_TOKEN: ${{ github.token }}
+          EVENT: ${{ github.event_name }}
+          PR: ${{ github.event.pull_request.number }}
+          DRAFT: ${{ github.event.pull_request.draft }}
+          SAME_REPO: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
+          RUSTUP_TOOLCHAIN: stable
+        run: |
+          set -euo pipefail
+          if [[ "$EVENT" == workflow_dispatch ]]; then
+            echo "run=true" >> "$GITHUB_OUTPUT"; exit 0
+          fi
+          if [[ "$DRAFT" == true ]]; then
+            echo "Draft: the perf budget runs once the pull request is ready." | tee -a "$GITHUB_STEP_SUMMARY"
+            echo "run=false" >> "$GITHUB_OUTPUT"; exit 0
+          fi
+          gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$RUNNER_TEMP/changed.txt"
+          python3 scripts/verify/plan.py risk --files "$RUNNER_TEMP/changed.txt" --output "$RUNNER_TEMP/risk.txt"
+          # A change to the budget itself is measured only from this
+          # repository's branches, so a fork cannot queue the macOS job by
+          # editing these files.
+          own_files=false
+          if [[ "$SAME_REPO" == true ]] && grep -qE '^(\.github/workflows/perf-budget\.yml|scripts/verify/perf-(ab|build)\.py)$' "$RUNNER_TEMP/changed.txt"; then
+            own_files=true
+          fi
+          if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt" || [[ "$own_files" == true ]]; then
+            echo "run=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "No engine or high-risk code changed: nothing to time." | tee -a "$GITHUB_STEP_SUMMARY"
+            echo "run=false" >> "$GITHUB_OUTPUT"
+          fi
+
+  measure:
+    needs: scope
+    if: needs.scope.outputs.run == 'true'
+    runs-on: blacksmith-6vcpu-macos-26
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Setup
+        uses: ./.github/actions/engine-verify-setup
+        with:
+          target: aarch64-apple-darwin
+          tier: fast
+
+      - name: Perf build cache
+        uses: actions/cache@v4
+        with:
+          path: target/perf-verify/
+          key: perf-verify-v1-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }}-${{ github.run_id }}
+          restore-keys: |
+            perf-verify-v1-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }}-
+            perf-verify-v1-
+
+      - name: Base worktree
+        env:
+          BASE_INPUT: ${{ inputs.base }}
+          PR_BASE: ${{ github.event.pull_request.base.sha }}
+        run: |
+          set -euo pipefail
+          base="${BASE_INPUT:-}"
+          if [[ -z "$base" ]]; then
+            base="$(git merge-base HEAD "${PR_BASE:-origin/main}")"
+          fi
+          echo "Comparing $(git rev-parse --short HEAD) with base $(git rev-parse --short "$base")" | tee -a "$GITHUB_STEP_SUMMARY"
+          git worktree add --detach "$RUNNER_TEMP/base" "$base"
+          # The base builds against the head's native dependencies: the same
+          # target dir and the FFMPEG_DIR that scripts/setup.js writes to the
+          # (git-ignored) .cargo/config.toml.
+          ln -s "$GITHUB_WORKSPACE/target" "$RUNNER_TEMP/base/target"
+          mkdir -p "$RUNNER_TEMP/base/.cargo"
+          cp .cargo/config.toml "$RUNNER_TEMP/base/.cargo/config.toml"
+
+      - name: Build head
+        run: python3 scripts/verify/perf-build.py . "$RUNNER_TEMP/perf/head"
+
+      - name: Build base
+        run: python3 scripts/verify/perf-build.py "$RUNNER_TEMP/base" "$RUNNER_TEMP/perf/base"
+
+      - name: Measure
+        run: |
+          python3 scripts/verify/perf-ab.py \
+            --base "$RUNNER_TEMP/perf/base/binaries.json" \
+            --head "$RUNNER_TEMP/perf/head/binaries.json" \
+            --fixtures "$RUNNER_TEMP/perf/fixtures" \
+            --out "$RUNNER_TEMP/perf/runs" \
+            --summary "$GITHUB_STEP_SUMMARY"
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: perf-budget
+          path: ${{ runner.temp }}/perf/runs/**/perf-budget.md
+          if-no-files-found: ignore
+
+  verdict:
+    name: Perf budget
+    needs: [scope, measure]
+    if: always()
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    steps:
+      - env:
+          RUN: ${{ needs.scope.outputs.run }}
+          SCOPE: ${{ needs.scope.result }}
+          MEASURE: ${{ needs.measure.result }}
+        run: |
+          if [[ "$SCOPE" != success ]]; then echo "scope failed"; exit 1; fi
+          if [[ "$RUN" != true ]]; then echo "not measured"; exit 0; fi
+          [[ "$MEASURE" == success ]]

scripts/verify/perf-ab.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+"""Paired performance comparison of two builds of the engine suite binaries.
+
+Runs each workload alternately with the base and head binaries on the same
+machine and compares the medians of the test process's CPU time (user +
+system, from the kernel's rusage) and wall time. CPU time ignores scheduling
+noise better than wall time; pairing cancels most of the machine's state.
+
+    perf-ab.py --base base.json --head head.json --fixtures DIR --out DIR
+
+base.json / head.json map a test target ("export", "seek") to its binary and
+the crate directory to run it from:
+    {"worktree": "...", "binaries": {"export": "...", "seek": "..."}}
+"""
+import argparse
+import json
+import os
+import statistics
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+# A head build fails the budget when the median of its per-pair CPU ratios to
+# the base is more than FAIL_LIMIT above 1 and it is slower in most pairs;
+# above WARN_LIMIT it is reported as a warning. Adjacent runs of a pair share
+# the machine's state, but a Blacksmith VM's speed drifts within a job and
+# between builds: identical code measured up to +10.8% (export-jitter-camera-
+# 40s, run 39000364369, pairs +13%, +11%, +0.3%) and seek-storm runs ranged
+# 10.1 to 13.2 s (run 38998485953). The fail limit sits well above that, so
+# the check stops large slowdowns without failing on the machine. Real-time
+# playback's CPU varied by 15% and is only reported.
+WORKLOADS = [
+    ("export-basic", "export", "export/basic/*", "fast", True),
+    ("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast", True),
+    ("seek-storm", "seek", "seek/*/storm30/*", "fast", True),
+    ("playback-basic", "playback", "playback/basic/*", "fast", False),
+]
+
+FAIL_LIMIT = 0.25
+WARN_LIMIT = 0.08
+
+
+def run_once(side, target, case, tier, fixtures, report):
+    binary = side["binaries"][target]
+    env = dict(
+        os.environ,
+        CAP_VERIFY_FIXTURE_DIR=str(fixtures),
+        CAP_VERIFY_REPORT_DIR=str(report),
+        CAP_VERIFY_CASE=case,
+        CAP_VERIFY_TIER=tier,
+    )
+    report.mkdir(parents=True, exist_ok=True)
+    started = time.monotonic()
+    with (report / "test.log").open("w") as log:
+        process = subprocess.Popen(
+            [binary, "--test-threads", "1", "--nocapture"],
+            cwd=Path(side["worktree"]) / "crates/verify",
+            env=env,
+            stdout=log,
+            stderr=subprocess.STDOUT,
+        )
+        _, status, usage = os.wait4(process.pid, 0)
+    wall = time.monotonic() - started
+    outcomes = {}
+    for path in report.glob("cap-verify-*.jsonl"):
+        for line in path.read_text().splitlines():
+            try:
+                row = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if row.get("case"):
+                # A case that passed, failed as expected or passed against
+                # an expectation ran to the end: the same work either way.
+                outcome = row.get("outcome")
+                outcomes[row["case"]] = "ran" if outcome in ("pass", "xfail", "xpass") else outcome
+    return {
+        "cpu": usage.ru_utime + usage.ru_stime,
+        "wall": wall,
+        "exit": os.waitstatus_to_exitcode(status),
+        "outcomes": outcomes,
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base", required=True)
+    parser.add_argument("--head", required=True)
+    parser.add_argument("--fixtures", required=True)
+    parser.add_argument("--out", required=True)
+    parser.add_argument("--pairs", type=int, default=3)
+    parser.add_argument("--summary")
+    args = parser.parse_args()
+
+    sides = {
+        "base": json.loads(Path(args.base).read_text()),
+        "head": json.loads(Path(args.head).read_text()),
+    }
+    fixtures = Path(args.fixtures)
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+
+    rows = []
+    failed = []
+    broken = []
+
+    def check(name, label, result, report):
+        # A test binary exits 0, or 101 when a case fails. Anything else (a
+        # signal, a missing library) means nothing was measured.
+        if result["exit"] in (0, 101) and result["outcomes"]:
+            return True
+        log = report / "test.log"
+        tail = log.read_text(errors="replace").splitlines()[-15:] if log.exists() else []
+        print(f"::error::{name} {label}: exit {result['exit']}, {len(result['outcomes'])} cases reported")
+        print("\n".join(tail))
+        return False
+    for name, target, case, tier, gated in WORKLOADS:
+        # The first run of a side generates fixtures and fills caches.
+        usable = True
+        for label, side in sides.items():
+            report = out / name / f"warm-{label}"
+            usable &= check(name, label, run_once(side, target, case, tier, fixtures, report), report)
+        if not usable:
+            rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases", None))
+            broken.append(name)
+            continue
+        samples = {"base": [], "head": []}
+        for pair in range(args.pairs):
+            order = ("base", "head") if pair % 2 == 0 else ("head", "base")
+            for label in order:
+                report = out / name / f"{pair}-{label}"
+                result = run_once(sides[label], target, case, tier, fixtures, report)
+                if not check(name, label, result, report):
+                    broken.append(name)
+                samples[label].append(result)
+                print(
+                    f"{name} pair {pair} {label}: cpu {result['cpu']:.2f}s wall {result['wall']:.2f}s exit {result['exit']}",
+                    flush=True,
+                )
+        if name in broken:
+            rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases", None))
+            continue
+        outcomes = [s["outcomes"] for runs in samples.values() for s in runs]
+        if any(o != outcomes[0] for o in outcomes):
+            # Correctness is the suite's job; a case that now passes or fails
+            # can change how much work runs, so its time is not comparable.
+            rows.append((name, None, None, None, None, "case outcomes differ between runs: timing not compared", None))
+            continue
+        cpu = {label: statistics.median(s["cpu"] for s in runs) for label, runs in samples.items()}
+        wall = {label: statistics.median(s["wall"] for s in runs) for label, runs in samples.items()}
+        cpu_delta = statistics.median(
+            h["cpu"] / b["cpu"] for b, h in zip(samples["base"], samples["head"])
+        ) - 1
+        wall_delta = wall["head"] / wall["base"] - 1
+        slower_pairs = sum(
+            h["cpu"] > b["cpu"] for b, h in zip(samples["base"], samples["head"])
+        )
+        mostly_slower = slower_pairs * 4 >= args.pairs * 3
+        verdict = "ok" if gated else "reported only"
+        if gated and cpu_delta > FAIL_LIMIT and mostly_slower:
+            verdict = f"SLOWER: CPU +{cpu_delta:.1%}, slower in {slower_pairs} of {args.pairs} pairs"
+            failed.append(name)
+        elif cpu_delta > WARN_LIMIT and mostly_slower:
+            verdict = f"CPU +{cpu_delta:.1%} (warning; fails past +{FAIL_LIMIT:.0%})"
+        elif wall_delta > WARN_LIMIT:
+            verdict = f"wall +{wall_delta:.1%} (warning only)"
+        rows.append((name, cpu["base"], cpu["head"], wall["base"], wall["head"], verdict, cpu_delta))
+
+    lines = [
+        "## Perf budget",
+        "",
+        f"{args.pairs} interleaved pairs: median test process CPU time (user + system) and wall time per side, and the median of the per-pair CPU ratios.",
+        "",
+        "| Workload | CPU base | CPU head | CPU change (median of pairs) | Wall base | Wall head | Wall change | Verdict |",
+        "| --- | --- | --- | --- | --- | --- | --- | --- |",
+    ]
+    for name, cb, ch, wb, wh, verdict, cpu_delta in rows:
+        if cb is None:
+            lines.append(f"| {name} | | | | | | | {verdict} |")
+        else:
+            lines.append(
+                f"| {name} | {cb:.2f} s | {ch:.2f} s | {cpu_delta:+.1%} | {wb:.2f} s | {wh:.2f} s | {wh / wb - 1:+.1%} | {verdict} |"
+            )
+    text = "\n".join(lines) + "\n"
+    print(text)
+    (out / "perf-budget.md").write_text(text)
+    if args.summary:
+        with open(args.summary, "a") as summary:
+            summary.write(text)
+    if broken:
+        print(f"::error::not measured: {', '.join(sorted(set(broken)))}")
+    if failed:
+        print(f"::error::slower than the base build: {', '.join(failed)}")
+    return 1 if broken or failed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())