diff --git a/.github/workflows/perf-budget.yml b/.github/workflows/perf-budget.yml new file mode 100644 index 0000000000..999f0b4633 --- /dev/null +++ b/.github/workflows/perf-budget.yml @@ -0,0 +1,146 @@ +name: Perf budget + +# Head and base are built and timed on one machine, interleaved, because +# Blacksmith VMs have no hardware instruction counters and machine state +# drifts between jobs. "Perf budget" is the job to require. + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + workflow_dispatch: + inputs: + base: + description: "Ref to compare against (default: the merge base with main)" + default: "" + +concurrency: + group: perf-budget-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + scope: + runs-on: blacksmith-2vcpu-ubuntu-2404 + outputs: + run: ${{ steps.scope.outputs.run }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Scope + id: scope + env: + GH_TOKEN: ${{ github.token }} + EVENT: ${{ github.event_name }} + PR: ${{ github.event.pull_request.number }} + DRAFT: ${{ github.event.pull_request.draft }} + SAME_REPO: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + RUSTUP_TOOLCHAIN: stable + run: | + set -euo pipefail + if [[ "$EVENT" == workflow_dispatch ]]; then + echo "run=true" >> "$GITHUB_OUTPUT"; exit 0 + fi + if [[ "$DRAFT" == true ]]; then + echo "Draft: the perf budget runs once the pull request is ready." | tee -a "$GITHUB_STEP_SUMMARY" + echo "run=false" >> "$GITHUB_OUTPUT"; exit 0 + fi + gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$RUNNER_TEMP/changed.txt" + python3 scripts/verify/plan.py risk --files "$RUNNER_TEMP/changed.txt" --output "$RUNNER_TEMP/risk.txt" + # A change to the budget itself is measured only from this + # repository's branches, so a fork cannot queue the macOS job by + # editing these files. + own_files=false + if [[ "$SAME_REPO" == true ]] && grep -qE '^(\.github/workflows/perf-budget\.yml|scripts/verify/perf-(ab|build)\.py)$' "$RUNNER_TEMP/changed.txt"; then + own_files=true + fi + if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt" || [[ "$own_files" == true ]]; then + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "No engine or high-risk code changed: nothing to time." | tee -a "$GITHUB_STEP_SUMMARY" + echo "run=false" >> "$GITHUB_OUTPUT" + fi + + measure: + needs: scope + if: needs.scope.outputs.run == 'true' + runs-on: blacksmith-6vcpu-macos-26 + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Setup + uses: ./.github/actions/engine-verify-setup + with: + target: aarch64-apple-darwin + tier: fast + + - name: Perf build cache + uses: actions/cache@v4 + with: + path: target/perf-verify/ + key: perf-verify-v1-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }}-${{ github.run_id }} + restore-keys: | + perf-verify-v1-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }}- + perf-verify-v1- + + - name: Base worktree + env: + BASE_INPUT: ${{ inputs.base }} + PR_BASE: ${{ github.event.pull_request.base.sha }} + run: | + set -euo pipefail + base="${BASE_INPUT:-}" + if [[ -z "$base" ]]; then + base="$(git merge-base HEAD "${PR_BASE:-origin/main}")" + fi + echo "Comparing $(git rev-parse --short HEAD) with base $(git rev-parse --short "$base")" | tee -a "$GITHUB_STEP_SUMMARY" + git worktree add --detach "$RUNNER_TEMP/base" "$base" + # The base builds against the head's native dependencies: the same + # target dir and the FFMPEG_DIR that scripts/setup.js writes to the + # (git-ignored) .cargo/config.toml. + ln -s "$GITHUB_WORKSPACE/target" "$RUNNER_TEMP/base/target" + mkdir -p "$RUNNER_TEMP/base/.cargo" + cp .cargo/config.toml "$RUNNER_TEMP/base/.cargo/config.toml" + + - name: Build head + run: python3 scripts/verify/perf-build.py . "$RUNNER_TEMP/perf/head" + + - name: Build base + run: python3 scripts/verify/perf-build.py "$RUNNER_TEMP/base" "$RUNNER_TEMP/perf/base" + + - name: Measure + run: | + python3 scripts/verify/perf-ab.py \ + --base "$RUNNER_TEMP/perf/base/binaries.json" \ + --head "$RUNNER_TEMP/perf/head/binaries.json" \ + --fixtures "$RUNNER_TEMP/perf/fixtures" \ + --out "$RUNNER_TEMP/perf/runs" \ + --summary "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: perf-budget + path: ${{ runner.temp }}/perf/runs/**/perf-budget.md + if-no-files-found: ignore + + verdict: + name: Perf budget + needs: [scope, measure] + if: always() + runs-on: blacksmith-2vcpu-ubuntu-2404 + steps: + - env: + RUN: ${{ needs.scope.outputs.run }} + SCOPE: ${{ needs.scope.result }} + MEASURE: ${{ needs.measure.result }} + run: | + if [[ "$SCOPE" != success ]]; then echo "scope failed"; exit 1; fi + if [[ "$RUN" != true ]]; then echo "not measured"; exit 0; fi + [[ "$MEASURE" == success ]] diff --git a/scripts/verify/__pycache__/perf-ab.cpython-314.pyc b/scripts/verify/__pycache__/perf-ab.cpython-314.pyc new file mode 100644 index 0000000000..6d55493591 Binary files /dev/null and b/scripts/verify/__pycache__/perf-ab.cpython-314.pyc differ diff --git a/scripts/verify/__pycache__/perf-build.cpython-314.pyc b/scripts/verify/__pycache__/perf-build.cpython-314.pyc new file mode 100644 index 0000000000..8f5e413151 Binary files /dev/null and b/scripts/verify/__pycache__/perf-build.cpython-314.pyc differ diff --git a/scripts/verify/__pycache__/perf_ab.cpython-314.pyc b/scripts/verify/__pycache__/perf_ab.cpython-314.pyc new file mode 100644 index 0000000000..672b6f1fd8 Binary files /dev/null and b/scripts/verify/__pycache__/perf_ab.cpython-314.pyc differ diff --git a/scripts/verify/__pycache__/perf_build.cpython-314.pyc b/scripts/verify/__pycache__/perf_build.cpython-314.pyc new file mode 100644 index 0000000000..1d6c453fb7 Binary files /dev/null and b/scripts/verify/__pycache__/perf_build.cpython-314.pyc differ diff --git a/scripts/verify/perf-ab.py b/scripts/verify/perf-ab.py new file mode 100644 index 0000000000..2d48a702ca --- /dev/null +++ b/scripts/verify/perf-ab.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""Paired performance comparison of two builds of the engine suite binaries. + +Runs each workload alternately with the base and head binaries on the same +machine and compares the medians of the test process's CPU time (user + +system, from the kernel's rusage) and wall time. CPU time ignores scheduling +noise better than wall time; pairing cancels most of the machine's state. + + perf-ab.py --base base.json --head head.json --fixtures DIR --out DIR + +base.json / head.json map a test target ("export", "seek") to its binary and +the crate directory to run it from: + {"worktree": "...", "binaries": {"export": "...", "seek": "..."}} +""" +import argparse +import json +import os +import statistics +import subprocess +import sys +import time +from pathlib import Path + +# A head build fails the budget when the median of its per-pair CPU ratios to +# the base is more than FAIL_LIMIT above 1 and it is slower in most pairs; +# above WARN_LIMIT it is reported as a warning. Adjacent runs of a pair share +# the machine's state, but a Blacksmith VM's speed drifts within a job and +# between builds: identical code measured up to +10.8% (export-jitter-camera- +# 40s, run 39000364369, pairs +13%, +11%, +0.3%) and seek-storm runs ranged +# 10.1 to 13.2 s (run 38998485953). The fail limit sits well above that, so +# the check stops large slowdowns without failing on the machine. Real-time +# playback's CPU varied by 15% and is only reported. +WORKLOADS = [ + ("export-basic", "export", "export/basic/*", "fast", True), + ("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast", True), + ("seek-storm", "seek", "seek/*/storm30/*", "fast", True), + ("playback-basic", "playback", "playback/basic/*", "fast", False), +] + +FAIL_LIMIT = 0.25 +WARN_LIMIT = 0.08 + + +def run_once(side, target, case, tier, fixtures, report): + binary = side["binaries"][target] + env = dict( + os.environ, + CAP_VERIFY_FIXTURE_DIR=str(fixtures), + CAP_VERIFY_REPORT_DIR=str(report), + CAP_VERIFY_CASE=case, + CAP_VERIFY_TIER=tier, + ) + report.mkdir(parents=True, exist_ok=True) + started = time.monotonic() + with (report / "test.log").open("w") as log: + process = subprocess.Popen( + [binary, "--test-threads", "1", "--nocapture"], + cwd=Path(side["worktree"]) / "crates/verify", + env=env, + stdout=log, + stderr=subprocess.STDOUT, + ) + _, status, usage = os.wait4(process.pid, 0) + wall = time.monotonic() - started + outcomes = {} + for path in report.glob("cap-verify-*.jsonl"): + for line in path.read_text().splitlines(): + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if row.get("case"): + # A case that passed, failed as expected or passed against + # an expectation ran to the end: the same work either way. + outcome = row.get("outcome") + outcomes[row["case"]] = "ran" if outcome in ("pass", "xfail", "xpass") else outcome + return { + "cpu": usage.ru_utime + usage.ru_stime, + "wall": wall, + "exit": os.waitstatus_to_exitcode(status), + "outcomes": outcomes, + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--base", required=True) + parser.add_argument("--head", required=True) + parser.add_argument("--fixtures", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--pairs", type=int, default=3) + parser.add_argument("--summary") + args = parser.parse_args() + + sides = { + "base": json.loads(Path(args.base).read_text()), + "head": json.loads(Path(args.head).read_text()), + } + fixtures = Path(args.fixtures) + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + + rows = [] + failed = [] + broken = [] + + def check(name, label, result, report): + # A test binary exits 0, or 101 when a case fails. Anything else (a + # signal, a missing library) means nothing was measured. + if result["exit"] in (0, 101) and result["outcomes"]: + return True + log = report / "test.log" + tail = log.read_text(errors="replace").splitlines()[-15:] if log.exists() else [] + print(f"::error::{name} {label}: exit {result['exit']}, {len(result['outcomes'])} cases reported") + print("\n".join(tail)) + return False + for name, target, case, tier, gated in WORKLOADS: + # The first run of a side generates fixtures and fills caches. + usable = True + for label, side in sides.items(): + report = out / name / f"warm-{label}" + usable &= check(name, label, run_once(side, target, case, tier, fixtures, report), report) + if not usable: + rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases", None)) + broken.append(name) + continue + samples = {"base": [], "head": []} + for pair in range(args.pairs): + order = ("base", "head") if pair % 2 == 0 else ("head", "base") + for label in order: + report = out / name / f"{pair}-{label}" + result = run_once(sides[label], target, case, tier, fixtures, report) + if not check(name, label, result, report): + broken.append(name) + samples[label].append(result) + print( + f"{name} pair {pair} {label}: cpu {result['cpu']:.2f}s wall {result['wall']:.2f}s exit {result['exit']}", + flush=True, + ) + if name in broken: + rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases", None)) + continue + outcomes = [s["outcomes"] for runs in samples.values() for s in runs] + if any(o != outcomes[0] for o in outcomes): + # Correctness is the suite's job; a case that now passes or fails + # can change how much work runs, so its time is not comparable. + rows.append((name, None, None, None, None, "case outcomes differ between runs: timing not compared", None)) + continue + cpu = {label: statistics.median(s["cpu"] for s in runs) for label, runs in samples.items()} + wall = {label: statistics.median(s["wall"] for s in runs) for label, runs in samples.items()} + cpu_delta = statistics.median( + h["cpu"] / b["cpu"] for b, h in zip(samples["base"], samples["head"]) + ) - 1 + wall_delta = wall["head"] / wall["base"] - 1 + slower_pairs = sum( + h["cpu"] > b["cpu"] for b, h in zip(samples["base"], samples["head"]) + ) + mostly_slower = slower_pairs * 4 >= args.pairs * 3 + verdict = "ok" if gated else "reported only" + if gated and cpu_delta > FAIL_LIMIT and mostly_slower: + verdict = f"SLOWER: CPU +{cpu_delta:.1%}, slower in {slower_pairs} of {args.pairs} pairs" + failed.append(name) + elif cpu_delta > WARN_LIMIT and mostly_slower: + verdict = f"CPU +{cpu_delta:.1%} (warning; fails past +{FAIL_LIMIT:.0%})" + elif wall_delta > WARN_LIMIT: + verdict = f"wall +{wall_delta:.1%} (warning only)" + rows.append((name, cpu["base"], cpu["head"], wall["base"], wall["head"], verdict, cpu_delta)) + + lines = [ + "## Perf budget", + "", + f"{args.pairs} interleaved pairs: median test process CPU time (user + system) and wall time per side, and the median of the per-pair CPU ratios.", + "", + "| Workload | CPU base | CPU head | CPU change (median of pairs) | Wall base | Wall head | Wall change | Verdict |", + "| --- | --- | --- | --- | --- | --- | --- | --- |", + ] + for name, cb, ch, wb, wh, verdict, cpu_delta in rows: + if cb is None: + lines.append(f"| {name} | | | | | | | {verdict} |") + else: + lines.append( + f"| {name} | {cb:.2f} s | {ch:.2f} s | {cpu_delta:+.1%} | {wb:.2f} s | {wh:.2f} s | {wh / wb - 1:+.1%} | {verdict} |" + ) + text = "\n".join(lines) + "\n" + print(text) + (out / "perf-budget.md").write_text(text) + if args.summary: + with open(args.summary, "a") as summary: + summary.write(text) + if broken: + print(f"::error::not measured: {', '.join(sorted(set(broken)))}") + if failed: + print(f"::error::slower than the base build: {', '.join(failed)}") + return 1 if broken or failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/verify/perf-build.py b/scripts/verify/perf-build.py new file mode 100644 index 0000000000..e970415eb7 --- /dev/null +++ b/scripts/verify/perf-build.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Builds the engine suite's export, seek and playback test binaries of one +worktree with the perf-verify profile and copies them out, so a second build +in the same target directory cannot replace them. + + perf-build.py WORKTREE OUT_DIR -> OUT_DIR/binaries.json +""" +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +TARGETS = ["export", "seek", "playback"] +# Release optimisation without link-time optimisation or single-unit codegen, +# so two builds per pull request stay affordable. Passed as --config so a base +# commit without this profile builds it too. +PROFILE = [ + "--config", 'profile.perf-verify.inherits="release"', + "--config", "profile.perf-verify.lto=false", + "--config", "profile.perf-verify.codegen-units=16", + "--config", "profile.perf-verify.incremental=false", + "--config", "profile.perf-verify.debug=0", +] + + +def main(): + worktree, out = Path(sys.argv[1]).resolve(), Path(sys.argv[2]).resolve() + out.mkdir(parents=True, exist_ok=True) + command = ["cargo", *PROFILE, "test", "--locked", "-p", "cap-verify", "--profile", "perf-verify", "--no-run", "--message-format", "json"] + for target in TARGETS: + command += ["--test", target] + result = subprocess.run(command, cwd=worktree, stdout=subprocess.PIPE, text=True, check=True) + binaries = {} + for line in result.stdout.splitlines(): + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if row.get("reason") == "compiler-artifact" and row.get("executable") and row["target"]["name"] in TARGETS: + source = Path(row["executable"]) + deps = out / "deps" + deps.mkdir(exist_ok=True) + dest = deps / source.name + shutil.copy2(source, dest) + binaries[row["target"]["name"]] = str(dest) + # macOS binaries load FFmpeg from @executable_path/../Frameworks. + # scripts/setup.js puts the framework in target/Frameworks; local + # builds may also have one per profile. + target_dir = source.parent.parent.parent + link = out / "Frameworks" + candidates = [target_dir / source.parent.parent.name / "Frameworks", target_dir / "Frameworks"] + candidates += [target_dir / profile / "Frameworks" for profile in ("release", "debug")] + for frameworks in candidates: + if frameworks.exists() and not link.exists(): + link.symlink_to(frameworks, target_is_directory=True) + break + missing = [t for t in TARGETS if t not in binaries] + if missing: + sys.exit(f"missing test binaries: {missing}") + if sys.platform == "darwin" and not (out / "Frameworks").exists(): + sys.exit("no FFmpeg Frameworks directory next to the target dir: run scripts/setup.js") + (out / "binaries.json").write_text(json.dumps({"worktree": str(worktree), "binaries": binaries}, indent=2)) + print(json.dumps(binaries, indent=2)) + + +if __name__ == "__main__": + main()