diff --git a/.github/justfile b/.github/justfile index 9ddb1a22958..696bd87a948 100644 --- a/.github/justfile +++ b/.github/justfile @@ -33,6 +33,7 @@ test: python3 sh/rs/test-lints.test.py python3 sh/rs/model-markers.test.py python3 sh/gh/nightly.test.py + python3 sh/gh/bench-report.test.py sh/go/publish-wrapper.test.sh bash .claude/hooks/direnv.test.sh python3 sh/rs/uring.test.py @@ -84,7 +85,22 @@ gst-formula *args: nightly group: sh/gh/nightly.sh {{ quote(group) }} +# Measure selected Criterion targets through the existing benchmark lifecycle. +[positional-arguments] +criterion *args: + bench/run.sh --criterion "$@" + +# Convert, validate, and stage nightly benchmark history. +[positional-arguments] +benchmark-report *args: + python3 sh/gh/bench-report.py "$@" + # Select shared native artifacts or check an upstream FFI publisher. [positional-arguments] ffi *args: python3 sh/gh/ffi.py "$@" + +# Prepare the shared history branch or publish its generated charts. +[positional-arguments] +benchmark-publish *args: + sh/gh/bench-publish.sh "$@" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index fd19fe150bb..23fff24c281 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -67,6 +67,7 @@ jobs: recipe: bench-ffi artifact: bench-ffi artifact-path: target/bench/ffi + benchmark: true # Keeps the BoltFFI and UniFFI slice building and running for the # browser generator comparison, which reuses it. It skips the size # step, whose moq-ffi and moq-wasm builds double the time and are @@ -91,8 +92,14 @@ jobs: recipe: gh nightly rust-test - name: Rust bench targets recipe: gh nightly rust-bench + artifact: bench-rust + artifact-path: target/bench/nightly/rust-bench + benchmark: true - name: JS benchmarks recipe: gh nightly js + artifact: bench-js + artifact-path: target/bench/nightly/js + benchmark: true uncached: true - name: BBB rendition alignment recipe: pub check-bbb @@ -160,10 +167,15 @@ jobs: run: nix develop --command bash -c "bun install --frozen-lockfile && bunx playwright install --with-deps chromium" - name: Run workload + id: workload run: nix develop ${{ matrix.shell }} --command just ${{ matrix.recipe }} env: NEXTEST_PROFILE: ${{ matrix.profile || 'default' }} + - name: Record benchmark outcome + if: ${{ matrix.benchmark && !cancelled() }} + run: nix develop --command just gh benchmark-report status "${{ steps.workload.outcome }}" "${{ matrix.artifact-path }}" + - name: Preserve results if: ${{ matrix.artifact && !cancelled() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -172,6 +184,78 @@ jobs: path: ${{ matrix.artifact-path }} retention-days: 45 + # One writer owns gh-pages, including future Criterion regression history. + # Explicit Pages deployment works with GITHUB_TOKEN; a branch push alone does not. + benchmark-site: + name: Benchmark site + needs: nightly + if: ${{ !cancelled() && github.ref == 'refs/heads/main' }} + runs-on: ubuntu-24.04-arm + timeout-minutes: 15 + concurrency: + group: benchmark-pages + cancel-in-progress: false + permissions: + contents: write + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deploy.outputs.page_url }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # Only this main-branch publisher pushes the shared history branch. + persist-credentials: true + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + pattern: bench-* + path: benchmark-inputs + - name: Validate completed reports + id: reports + run: nix develop --command just gh benchmark-report prepare benchmark-inputs benchmark-reports + - name: Prepare history + run: nix develop --command just gh benchmark-publish init + - name: Store time, CPU and ratio history + if: steps.reports.outputs.costs == 'true' + uses: benchmark-action/github-action-benchmark@4322e5726e6334590d251fc4f92bec0efafc45dc # v1.22.2 + with: + name: MoQ nightly costs + tool: customSmallerIsBetter + output-file-path: benchmark-reports/costs.json + benchmark-data-dir-path: bench/costs + skip-fetch-gh-pages: true + github-token: ${{ github.token }} + ref: ${{ github.sha }} + auto-push: false + comment-on-alert: false + fail-on-alert: false + - name: Store throughput history + if: steps.reports.outputs.rates == 'true' + uses: benchmark-action/github-action-benchmark@4322e5726e6334590d251fc4f92bec0efafc45dc # v1.22.2 + with: + name: MoQ nightly throughput + tool: customBiggerIsBetter + output-file-path: benchmark-reports/rates.json + benchmark-data-dir-path: bench/rates + skip-fetch-gh-pages: true + github-token: ${{ github.token }} + ref: ${{ github.sha }} + auto-push: false + comment-on-alert: false + fail-on-alert: false + - name: Publish shared history + run: nix develop --command just gh benchmark-publish finish benchmark-site + - uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4 + with: + path: benchmark-site + - name: Deploy charts + id: deploy + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 + tests: name: Tests (${{ matrix.recipe }}) runs-on: ubuntu-24.04-arm diff --git a/bench/README.md b/bench/README.md index bc3431211a5..abd7f6ecba2 100644 --- a/bench/README.md +++ b/bench/README.md @@ -340,5 +340,61 @@ Rust Criterion coverage includes `moq-pattern`'s and payload size; CMAF also varies track count. Separate frame and byte rates use the same fixtures, with byte rates counting the encoded stream size. Fixture generation, publisher setup and destruction stay outside timing. -The existing shared Rust benchmark nightly invocation runs both targets; their `--test` mode -exercises every case without making timing claims. +The shared Rust benchmark nightly invocation measures both targets in quick mode. + +## Nightly trends + +The trend site will be at once the +maintainer enables GitHub Pages with **Source: GitHub Actions**. The Nightly +workflow keeps history on one `gh-pages` branch and deploys the generated +`github-action-benchmark` charts explicitly. The publisher runs only on `main`, +serializes history writes, and leaves regression alerts and comments disabled. +Enabling Pages is a separate repository setting; merging the workflow does not +change it. Until it is enabled, the Nightly's Benchmark site job still records +history on `gh-pages`, then fails at deployment. + +The charts include these measurements: + +- Binding delivery in MB/s, process CPU in ms/MB, sync calls in ns, and each + wrapper's ratio to `rust-ffi`. Ratios above 1 mean more cost than Rust on the + same host, including for throughput, where the ratio is Rust/wrapper. +- Rust Criterion mean ns per iteration, with the measured confidence interval. + The `net-cpu` source and the moq-quic targets time thread CPU; the others + time elapsed time. The Rust bench targets job measures them with a short warmup + and Criterion's quick mode. Per-group benchmark settings can override the default warmup. Quick estimates + trade precision for nightly runtime; use full paired runs to assess a change. +- JS benchmark CSV time columns (`ns`, `us`, `ms`, and their per-operation + variants), retaining every workload dimension in each series name. + +Hosted runners vary across nights. Raw trends are informational, and binding +ratios compare languages within one run. A failed suite contributes no point; +malformed, empty, nonfinite, or duplicate measurements fail validation. The +nightly artifacts retain the reports, raw JS output, and each Criterion run's +metadata and estimates for 45 days. Relay load, audio quality, and the Rust +benchmark tests job's nextest output do not enter these charts. + +Reproduce the chart inputs locally: + +```bash +nix develop --command just bench-ffi +nix develop --command just gh nightly rust-bench +nix develop --command just gh nightly js +``` + +The latter two write `target/bench/nightly/{rust-bench,js}/report.json`. For a focused +measured Rust run using the same evidence lifecycle: + +```bash +nix develop --command env MOQ_BENCH_LABEL=pattern MOQ_BENCH_RUNS=/tmp/criterion-runs.txt just gh criterion -p moq-pattern --bench matching +nix develop --command just gh benchmark-report rust /tmp /tmp/rust-report.json +``` + +A new source uploads a `bench-*` artifact containing `report.json`, a nonempty +JSON array of `{ "name": "case/dimensions", "unit": "ns", "value": 123 }`, +and `outcome.json` containing `{ "status": "success" }`. Names must be unique +within a source; dimensions belong in the name. Supported units are `ns`, `us`, +`µs`, `ms`, `s`, `ms/MB`, `x`, `bytes`, `B`, `MiB`, `MB/s`, `MiB/s`, `B/s`, +`ops/s`, and `frames/s`. Optional `range` and `extra` fields retain uncertainty +and context. Validate and split sources into cost and throughput charts with +`just gh benchmark-report prepare INPUT_DIRECTORY OUTPUT_DIRECTORY`; extending +the site requires no new chart renderer or data branch. diff --git a/bench/metadata.py b/bench/metadata.py index 5be41f76556..5500abe8c2f 100644 --- a/bench/metadata.py +++ b/bench/metadata.py @@ -27,15 +27,16 @@ def command(*args): "rustc": command("rustc", "-vV"), "cargo": command("cargo", "-V"), "os": platform.platform(), "machine": platform.machine(), "processor": platform.processor(), "affinity": sorted(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else None, - "environment": {key: value for key, value in os.environ.items() if key.startswith(("MOQ_BENCH_", "CARGO_PROFILE_")) or key in ("RUSTFLAGS", "CARGO_BUILD_TARGET", "CARGO_BUILD_RUSTFLAGS", "TOKIO_WORKER_THREADS", "LD_PRELOAD", "MALLOC_CONF", "GLIBC_TUNABLES", "RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER")}, + "environment": {key: value for key, value in os.environ.items() if key.startswith(("MOQ_BENCH_", "CARGO_PROFILE_")) or key in ("RUSTFLAGS", "CARGO_BUILD_TARGET", "CARGO_BUILD_RUSTFLAGS", "TOKIO_WORKER_THREADS", "LD_PRELOAD", "MALLOC_CONF", "GLIBC_TUNABLES", "RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER", "SESSION_CPU")}, "profile": "profiling" if mode == "--profile" else "release", - "features": {"--runtime": "default + moq-relay/io-uring,moq-bench/uring", "--profile": "default; heap adds moq-relay/jemalloc"}.get(mode, "default"), + "features": {"--runtime": "default + moq-relay/io-uring,moq-bench/uring", "--profile": "default; heap adds moq-relay/jemalloc", + "--criterion": "see criterion-command.json"}.get(mode, "default"), "mode": mode, "runtime_workers": sys.argv[4] if len(sys.argv) > 4 else None, "rounds": int(sys.argv[5]) if len(sys.argv) > 5 else None, "allocator": "system by build; preload and allocator tunables are recorded in environment" + ("; heap runs jemalloc, profiled per the MALLOC_CONF in harness/relay.sh" if mode == "--profile" else ""), "profilers": {tool: command(tool, "--version").splitlines()[0] for tool in ("perf", "jeprof") if shutil.which(tool)} if mode == "--profile" else None, - "workload": {"startup": "2s", "duration": "10s", "report": "500ms"}} + "workload": None if mode == "--criterion" else {"startup": "2s", "duration": "10s", "report": "500ms"}} (out / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") subprocess.run(["cargo", "tree", "--locked", "-p", "moq-relay", "-e", "features"], stdout=(out / "features.txt").open("w"), check=True) if Path("/proc/cpuinfo").exists(): diff --git a/bench/run.sh b/bench/run.sh index 297400e4403..86765125844 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -47,6 +47,21 @@ usable_cpus() { env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc } case $MODE in + --criterion) + BASE= + CRITERION_CARGO_ARGS=() + CRITERION_FILTER_ARGS=() + parsing_filters=false + for argument in "${@:2}"; do + if [[ $argument == -- && $parsing_filters == false ]]; then + parsing_filters=true + elif [[ $parsing_filters == true ]]; then + CRITERION_FILTER_ARGS+=("$argument") + else + CRITERION_CARGO_ARGS+=("$argument") + fi + done + ;; --audio) BASE= ;; @@ -710,7 +725,17 @@ run_profile() { cd "$ROOT" -if [[ $MODE == --audio ]]; then +if [[ $MODE == --criterion ]]; then + # Each invocation owns fresh results; previous runs cannot become new points. + python3 -c 'import json,sys; from pathlib import Path; Path(sys.argv[1]).write_text(json.dumps(sys.argv[2:]))' \ + "$RUN/criterion-command.json" "${@:2}" + if [[ -n ${MOQ_BENCH_RUNS:-} ]]; then + printf '%s\n' "$RUN" >>"$MOQ_BENCH_RUNS" + fi + CARGO_TARGET_DIR=$CURRENT_TARGET cargo bench --locked ${CRITERION_CARGO_ARGS[@]+"${CRITERION_CARGO_ARGS[@]}"} -- \ + --quick --warm-up-time 0.05 --measurement-time 0.2 --noplot ${CRITERION_FILTER_ARGS[@]+"${CRITERION_FILTER_ARGS[@]}"} + exit 0 +elif [[ $MODE == --audio ]]; then run_audio_comparison exit 0 elif [[ $MODE == --runtime ]]; then diff --git a/js/net/bench/frames.ts b/js/net/bench/frames.ts index bff942ce72c..eb2ac881bf4 100644 --- a/js/net/bench/frames.ts +++ b/js/net/bench/frames.ts @@ -13,6 +13,9 @@ import * as Path from "../src/path.ts"; import { Reader, Stream } from "../src/stream.ts"; import * as Varint from "../src/varint.ts"; +// Keep protocol diagnostics out of the measured CSV on stdout. +console.debug = console.error; + const frameSizes = [16, 100, 1000]; const chunkSizes = [1200, 16 * 1024, 64 * 1024]; const framesPerGroup = 50; diff --git a/js/net/bench/requests.ts b/js/net/bench/requests.ts index 0daf1b294ed..2f16119e7ec 100644 --- a/js/net/bench/requests.ts +++ b/js/net/bench/requests.ts @@ -16,6 +16,9 @@ import * as Path from "../src/path"; import { Stream } from "../src/stream"; import { wireOf } from "../src/wire"; +// Keep protocol diagnostics out of the measured CSV on stdout. +console.debug = console.error; + const settle = () => new Promise((resolve) => setImmediate(resolve)); console.log("touched,routes,followers,update_us"); for (const touched of ["watched", "unrelated"]) diff --git a/js/publish/bench/catalog.ts b/js/publish/bench/catalog.ts index 75e3cc8ee92..623a64694d3 100644 --- a/js/publish/bench/catalog.ts +++ b/js/publish/bench/catalog.ts @@ -1,26 +1,13 @@ /** Sweep tracks x viewers for 1000 estimate rises inside one second. */ -import { Milli } from "@moq/clock"; +import { Manual, Milli, Nano } from "@moq/clock"; import * as Catalog from "@moq/hang/catalog"; import * as Json from "@moq/json"; import { Track } from "@moq/net"; import { Effect } from "@moq/signals"; import { CatalogProducer } from "../src/catalog"; -const realNow = performance.now.bind(performance); -const realTimer = Effect.prototype.timer; -let now = 0; -let timers = new Set<{ at: number; run: () => void }>(); -performance.now = () => now; -Effect.prototype.timer = function (fn, ms) { - const timer = { at: now + ms, run: fn }; - timers.add(timer); - this.cleanup(() => timers.delete(timer)); -}; - -async function measure(tracks: number, viewers: number) { - now = 0; - timers = new Set(); +async function measure(tracks: number, viewers: number, clock: Manual) { const scope = new Effect(); const catalog = new CatalogProducer(); const media = Array.from({ length: tracks }, (_, index) => new Track.Producer(`v${index}`)); @@ -68,19 +55,18 @@ async function measure(tracks: number, viewers: number) { } } }; - const start = realNow(); + const start = performance.now(); for (let step = 0; step < 1000; step++) { - now = step; + if (step > 0) clock.advance(Nano.fromMilli(Milli(1))); catalog.mutate((root) => { if (!root.video) throw new Error("video missing"); for (const config of Object.values(root.video.renditions)) config.jitter = Catalog.u53(step + 1); }); await receive(); } - now = 1000; - for (const timer of [...timers]) if (timer.at <= now) timer.run(); + await clock.advanceAsync(Nano.fromMilli(Milli(1))); await receive(); - const elapsed = realNow() - start; + const elapsed = performance.now() - start; if (publishes !== 2 || updates !== 2 * tracks * viewers) { throw new Error(`estimate churn regression: ${publishes} publishes, ${updates} subscribe updates`); } @@ -94,10 +80,15 @@ async function measure(tracks: number, viewers: number) { for (const track of media) track.close(); } -try { - console.log("tracks,viewers,catalog_publishes,subscribe_updates,unlimited_subscribe_updates,elapsed_ms"); - for (const tracks of [1, 4, 16]) for (const viewers of [1, 4, 16]) await measure(tracks, viewers); -} finally { - performance.now = realNow; - Effect.prototype.timer = realTimer; +console.log("tracks,viewers,catalog_publishes,subscribe_updates,unlimited_subscribe_updates,elapsed_ms"); +for (const tracks of [1, 4, 16]) { + for (const viewers of [1, 4, 16]) { + const clock = new Manual(); + const restore = clock.install(); + try { + await measure(tracks, viewers, clock); + } finally { + restore(); + } + } } diff --git a/quest/m2/bench-site.md b/quest/m2/bench-site.md index ec86b89da4c..3dc29a290f6 100644 --- a/quest/m2/bench-site.md +++ b/quest/m2/bench-site.md @@ -10,6 +10,12 @@ site. ## Plan +The pipeline is in review. Remaining: the maintainer enables repository Pages +with **Source: GitHub Actions** (until then the Nightly's Benchmark site job +fails at deployment), then verifies the first Nightly deployment at +. The "Rust benchmark tests" job is not +charted yet: its nextest benchmarks print no structured measurements. + Decided with the maintainer on 2026-10-15: - **GitHub Pages via github-action-benchmark**: the nightly pushes each diff --git a/rs/moq-net/benches/origin.rs b/rs/moq-net/benches/origin.rs index 566c93be9c5..1f602cdb133 100644 --- a/rs/moq-net/benches/origin.rs +++ b/rs/moq-net/benches/origin.rs @@ -255,7 +255,10 @@ fn bench_announce_duplicate(c: &mut Criterion) { for (duplicates, subscribers) in CONTENDED { let id = BenchmarkId::from_parameter(format!("{duplicates}d_{subscribers}s")); group.bench_function(id, |b| { - let (producer, _driver) = origin::Producer::new(origin::Config::default()); + // No hold, so an iteration times the ranking rather than the origin batching it. + let mut config = origin::Config::default(); + config.update_hold = Duration::ZERO; + let (producer, _driver) = origin::Producer::new(config); let consumer = producer.consume(); // Every peer announces the one path, each under its own hop chain so // the entries are distinct routes rather than one re-priced in place. @@ -295,7 +298,10 @@ fn bench_reprice_duplicate(c: &mut Criterion) { for (duplicates, subscribers) in CONTENDED { let id = BenchmarkId::from_parameter(format!("{duplicates}d_{subscribers}s_cost{incumbent_cost}")); group.bench_function(id, |b| { - let (producer, _driver) = origin::Producer::new(origin::Config::default()); + // No hold, so an iteration times the ranking rather than the origin batching it. + let mut config = origin::Config::default(); + config.update_hold = Duration::ZERO; + let (producer, _driver) = origin::Producer::new(config); let _routes: Vec<_> = (1..=duplicates) .map(|peer| { producer diff --git a/sh/gh/bench-publish.sh b/sh/gh/bench-publish.sh new file mode 100755 index 00000000000..6ee65b8e41f --- /dev/null +++ b/sh/gh/bench-publish.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +case ${1:-} in + init) + # ls-remote distinguishes an absent history branch from an authentication failure. + if git ls-remote --exit-code --heads origin gh-pages; then + git fetch origin gh-pages:gh-pages + else + status=$? + if [[ $status != 2 ]]; then exit "$status"; fi + tree=$(git hash-object -t tree /dev/null) + commit=$(git -c user.name=github-actions -c user.email=github-actions@users.noreply.github.com \ + commit-tree "$tree" -m 'Initialize benchmark history') + git branch gh-pages "$commit" + fi + ;; + finish) + if [[ $# != 2 ]]; then + echo 'usage: benchmark-publish finish DIRECTORY' >&2 + exit 2 + fi + git worktree add "$2" gh-pages + just gh benchmark-report index "$2" + git -C "$2" add bench/index.html .nojekyll + if ! git -C "$2" diff --cached --quiet; then + git -C "$2" -c user.name=github-actions -c user.email=github-actions@users.noreply.github.com \ + commit -m 'Update benchmark landing page' + fi + git -C "$2" push origin HEAD:gh-pages + ;; + *) + echo 'usage: benchmark-publish {init|finish DIRECTORY}' >&2 + exit 2 + ;; +esac diff --git a/sh/gh/bench-report.py b/sh/gh/bench-report.py new file mode 100644 index 00000000000..5dc89fc973c --- /dev/null +++ b/sh/gh/bench-report.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Convert measured nightly results to github-action-benchmark's JSON format.""" + +import argparse +import csv +import html +import json +import math +import os +from pathlib import Path +import re +from urllib.parse import quote + + +BIGGER = {'MB/s', 'MiB/s', 'B/s', 'ops/s', 'frames/s'} +SMALLER = {'ns', 'us', 'µs', 'ms', 's', 'ms/MB', 'x', 'bytes', 'B', 'MiB'} + + +def value(number): + if isinstance(number, bool) or not isinstance(number, (int, float)) or not math.isfinite(number) or number < 0: + raise ValueError(f'invalid measurement: {number!r}') + return number + + +def checked(entries): + if not isinstance(entries, list) or not entries: + raise ValueError('a benchmark report must contain measured entries') + names = set() + for entry in entries: + if not isinstance(entry, dict) or not isinstance(entry.get('name'), str) or not entry['name']: + raise ValueError(f'invalid benchmark entry: {entry!r}') + if entry['name'] in names: + raise ValueError(f'duplicate benchmark: {entry["name"]}') + names.add(entry['name']) + if entry.get('unit') not in BIGGER | SMALLER: + raise ValueError(f'unsupported measurement unit: {entry.get("unit")!r}') + value(entry.get('value')) + return entries + + +def write(path, entries): + checked(entries) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(entries, indent=2) + '\n') + + +def rust(directory): + manifest = directory / 'criterion-runs.txt' + runs = [Path(line) for line in manifest.read_text().splitlines() if line] + if not runs: + raise ValueError(f'no measured Criterion runs in {manifest}') + entries = [] + for run in runs: + metadata = json.loads((run / 'metadata.json').read_text()) + if metadata['mode'] != '--criterion': + raise ValueError(f'not a measured Criterion run: {run}') + label = metadata['environment'].get('MOQ_BENCH_LABEL') + if not isinstance(label, str) or not label: + raise ValueError(f'missing Criterion source label: {run}') + estimates = sorted((run / 'criterion').glob('**/new/estimates.json')) + if not estimates: + raise ValueError(f'Criterion produced no measured estimates: {run}') + for path in estimates: + benchmark = json.loads(path.with_name('benchmark.json').read_text()) + estimate = json.loads(path.read_text())['mean'] + interval = estimate['confidence_interval'] + low, high = value(interval['lower_bound']), value(interval['upper_bound']) + entries.append({ + 'name': f'rust/{label}/{benchmark["full_id"]}', 'unit': 'ns', + 'value': value(estimate['point_estimate']), 'range': f'{low:.4g}..{high:.4g}', + # Criterion does not record a target's measurement, so the clock is left to the README. + 'extra': f'Mean nanoseconds per iteration; {metadata["revision"]}', + }) + return checked(entries) + + +def timing_unit(column): + for unit in ('ns', 'us', 'ms'): + if column == unit or column.endswith(f'_{unit}') or column.startswith(f'{unit}_per_'): + return unit + return None + + +def javascript_file(path): + entries, headers = [], None + for line_number, line in enumerate(path.read_text().splitlines(), 1): + # The interests sweep reports dimensions and elapsed time in one line. + interests = re.fullmatch(r'broadcasts=(\d+) tracks=(\d+) acquire\+release=([\d.eE+-]+) us', line) + if interests: + broadcasts, tracks, elapsed = interests.groups() + entries.append({'name': f'js/{path.stem}/broadcasts={broadcasts}/tracks={tracks}/acquire+release', + 'unit': 'us', 'value': value(float(elapsed))}) + continue + row = [field.strip() for field in next(csv.reader([line]))] + if len(row) > 1 and any(timing_unit(field) for field in row): + if len(set(row)) != len(row): + raise ValueError(f'duplicate CSV headers: {path}:{line_number}') + headers = row + continue + if not headers or not line.strip(): + continue + if len(row) != len(headers): + raise ValueError(f'malformed benchmark CSV: {path}:{line_number}') + dimensions = '/'.join(f'{key}={quote(field, safe="-_.~")}' for key, field in zip(headers, row) + if timing_unit(key) is None) + for key, field in zip(headers, row): + if unit := timing_unit(key): + entries.append({'name': f'js/{path.stem}/{dimensions}/{key}', + 'unit': unit, 'value': value(float(field))}) + if not entries: + raise ValueError(f'JS benchmark produced no measurements: {path}') + return checked(entries) + + +def javascript(directory): + entries = [entry for path in sorted((directory / 'raw').glob('*.csv')) for entry in javascript_file(path)] + return checked(entries) + + +def prepare(directory, output): + costs, rates = [], [] + for source in sorted(directory.glob('bench-*')): + # Other nightly artifacts share the prefix; only a recorded outcome marks a chart source. + status = source / 'outcome.json' + if not status.exists(): + continue + outcome = json.loads(status.read_text())['status'] + if outcome not in {'success', 'failure', 'cancelled', 'skipped'}: + raise ValueError(f'invalid benchmark outcome: {source}') + if outcome != 'success': + print(f'{source.name}: {outcome}; no history point') + continue + report = source / 'report.json' + if not report.exists(): + raise ValueError(f'successful benchmark has no report: {source}') + entries = checked(json.loads(report.read_text())) + for entry in entries: + target = rates if entry['unit'] in BIGGER else costs + target.append({**entry, 'name': f'{source.name}/{entry["name"]}'}) + print(f'{source.name}: {len(entries)} measured entries') + if not costs and not rates: + raise ValueError('no completed benchmark reports to publish') + output.mkdir(parents=True, exist_ok=True) + for name, entries in [('costs', costs), ('rates', rates)]: + (output / f'{name}.json').write_text(json.dumps(entries, indent=2) + '\n') + if entries: + checked(entries) + if target := os.environ.get('GITHUB_OUTPUT'): + with open(target, 'a') as stream: + stream.write(f'{name}={str(bool(entries)).lower()}\n') + + +def index(directory): + destination = directory / 'bench' + links = [] + for name, label in [('costs', 'Time, CPU, size and Rust ratios'), ('rates', 'Delivered throughput')]: + if (destination / name / 'index.html').exists(): + links.append(f'
  • {html.escape(label)}
    • ') + if not links: + raise ValueError('no generated benchmark charts to deploy') + (destination / 'index.html').write_text(''' + +MoQ benchmarks + +

    MoQ nightly benchmarks

    +

    Hosted runners vary. These charts are informational. Binding ratios compare each wrapper +with Rust on the same host: above 1x means slower or more CPU per delivered byte.

    +
      ''' + ''.join(links) + '''
    +

    Each series keeps its source and payload shape. A failed suite contributes no point. +Metrics and local reproduction

    + +''') + (directory / '.nojekyll').touch() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + for command in ('rust', 'js', 'prepare'): + sub = commands.add_parser(command) + sub.add_argument('directory', type=Path) + sub.add_argument('output', type=Path) + sub = commands.add_parser('status') + sub.add_argument('status', choices=('success', 'failure', 'cancelled', 'skipped')) + sub.add_argument('directory', type=Path) + commands.add_parser('index').add_argument('directory', type=Path) + args = parser.parse_args() + if args.command == 'rust': + write(args.output, rust(args.directory)) + elif args.command == 'js': + write(args.output, javascript(args.directory)) + elif args.command == 'prepare': + prepare(args.directory, args.output) + elif args.command == 'status': + args.directory.mkdir(parents=True, exist_ok=True) + (args.directory / 'outcome.json').write_text(json.dumps({'status': args.status}) + '\n') + else: + index(args.directory) + + +if __name__ == '__main__': + main() diff --git a/sh/gh/bench-report.test.py b/sh/gh/bench-report.test.py new file mode 100644 index 00000000000..ea5da4426c7 --- /dev/null +++ b/sh/gh/bench-report.test.py @@ -0,0 +1,139 @@ +"""Nightly charts accept measurements, refuse malformed data, and share one history branch.""" +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location('report', Path(__file__).with_name('bench-report.py')) +report = importlib.util.module_from_spec(spec) +spec.loader.exec_module(report) + + +class ReportTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + + def test_values_refuse_smoke_and_malformed_reports(self): + for entries in ([], {}, [None], [{'name': '', 'unit': 'ns', 'value': 1}]): + with self.assertRaises(ValueError): + report.checked(entries) + for number in (float('nan'), float('inf'), -1, None, True, '2'): + with self.assertRaises(ValueError): + report.value(number) + for entries in ([{'name': 'x', 'unit': 'unknown', 'value': 1}], + [{'name': 'x', 'unit': 'ns', 'value': 1}] * 2): + with self.assertRaises(ValueError): + report.checked(entries) + + def test_js_multiple_schemas_and_interests(self): + source = self.root / 'requests.csv' + source.write_text('bun install v1.3\npaths,active_us,settled_us\n4,1.25,2.5\noperation,ns_per_op\nclose,0\n') + entries = report.javascript_file(source) + self.assertEqual([entry['value'] for entry in entries], [1.25, 2.5, 0]) + self.assertEqual(entries[0]['name'], 'js/requests/paths=4/active_us') + source.write_text('broadcasts=32 tracks=8 acquire+release=1.250 us\n') + self.assertEqual(report.javascript_file(source)[0]['value'], 1.25) + for text in ('Testing case\nSuccess\n', 'paths,time_us\n4\n', 'paths,time_us\n4,nan\n'): + source.write_text(text) + with self.assertRaises(ValueError): + report.javascript_file(source) + + def test_criterion_requires_fresh_measured_estimates(self): + run = self.root / 'runs' / 'one' + run.mkdir(parents=True) + (self.root / 'criterion-runs.txt').write_text(f'{run}\n') + (run / 'metadata.json').write_text(json.dumps({'mode': '--criterion', 'revision': 'abc', + 'environment': {'MOQ_BENCH_LABEL': 'net-cpu', 'SESSION_CPU': '1'}})) + with self.assertRaises(ValueError): + report.rust(self.root) + case = run / 'criterion' / 'delivery' / 'new' + case.mkdir(parents=True) + (case / 'benchmark.json').write_text(json.dumps({'full_id': 'delivery/4/8'})) + (case / 'estimates.json').write_text(json.dumps({'mean': {'point_estimate': 20, + 'confidence_interval': {'lower_bound': 18, 'upper_bound': 22}}})) + entry = report.rust(self.root)[0] + self.assertEqual(entry['name'], 'rust/net-cpu/delivery/4/8') + self.assertEqual(entry['value'], 20) + self.assertIn('abc', entry['extra']) + + def test_only_successful_sources_are_published(self): + source = self.root / 'bench-ffi' + source.mkdir() + entries = [{'name': 'go/round', 'unit': 'MB/s', 'value': 4}, + {'name': 'go/round vs rust-ffi', 'unit': 'x', 'value': 2}] + (source / 'report.json').write_text(json.dumps(entries)) + (source / 'outcome.json').write_text('{"status":"success"}') + # Another nightly artifact with the same prefix and its own report schema. + unrelated = self.root / 'bench-browser' + unrelated.mkdir() + (unrelated / 'report.json').write_text('{"samples": []}') + failed = self.root / 'bench-rust' + failed.mkdir() + (failed / 'outcome.json').write_text('{"status":"failure"}') + (failed / 'report.json').write_text('not a usable report') + output = self.root / 'output' + report.prepare(self.root, output) + self.assertEqual(json.loads((output / 'rates.json').read_text())[0]['value'], 4) + self.assertEqual(json.loads((output / 'costs.json').read_text())[0]['value'], 2) + (source / 'report.json').unlink() + with self.assertRaises(ValueError): + report.prepare(self.root, output) + + def test_landing_page_keeps_existing_charts_and_history(self): + chart = self.root / 'bench' / 'costs' + chart.mkdir(parents=True) + (chart / 'index.html').write_text('chart') + (chart / 'data.js').write_text('history') + report.index(self.root) + self.assertIn('costs/', (chart.parent / 'index.html').read_text()) + self.assertEqual((chart / 'data.js').read_text(), 'history') + self.assertTrue((self.root / '.nojekyll').exists()) + + def test_first_and_subsequent_history_publications(self): + remote, checkout = self.root / 'remote.git', self.root / 'checkout' + def git(*args, cwd=None): + return subprocess.run(['git', *args], cwd=cwd, check=True, capture_output=True, text=True) + git('init', '--bare', str(remote)) + git('clone', str(remote), str(checkout)) + git('config', 'user.name', 'test', cwd=checkout) + git('config', 'user.email', 'test@example.com', cwd=checkout) + (checkout / 'source').write_text('source tree') + git('add', '.', cwd=checkout) + git('commit', '-m', 'Source', cwd=checkout) + commands = self.root / 'commands' + commands.mkdir() + just = commands / 'just' + just.write_text(f'#!/bin/sh\nexec python3 {ROOT}/sh/gh/bench-report.py "$3" "$4"\n') + just.chmod(0o755) + env = dict(os.environ, PATH=f'{commands}:{os.environ["PATH"]}') + publisher = ROOT / 'sh/gh/bench-publish.sh' + for iteration in (1, 2): + subprocess.run(['bash', str(publisher), 'init'], cwd=checkout, env=env, check=True, + capture_output=True) + history = self.root / f'history-{iteration}' + git('worktree', 'add', str(history), 'gh-pages', cwd=checkout) + chart = history / 'bench' / 'costs' + chart.mkdir(parents=True, exist_ok=True) + (chart / 'index.html').write_text('chart') + (chart / 'data.js').write_text(f'run {iteration}') + git('add', '.', cwd=history) + git('commit', '-m', 'Measured result', cwd=history) + git('worktree', 'remove', str(history), cwd=checkout) + site = self.root / f'site-{iteration}' + subprocess.run(['bash', str(publisher), 'finish', str(site)], cwd=checkout, env=env, + check=True, capture_output=True) + self.assertFalse((site / 'source').exists()) + self.assertEqual((site / 'bench/costs/data.js').read_text(), f'run {iteration}') + git('worktree', 'remove', str(site), cwd=checkout) + git('branch', '-D', 'gh-pages', cwd=checkout) + self.assertEqual(git('--git-dir', str(remote), 'show', 'gh-pages:bench/costs/data.js').stdout, 'run 2') + + +if __name__ == '__main__': + unittest.main() diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 7be7804d203..151a45ad001 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -7,6 +7,25 @@ if [[ $# != 1 ]]; then echo "$usage" >&2 exit 2 fi +# The measured groups write a report for the benchmark site; rust-test only runs. +case "$1" in + rust-test) report= ;; + rust-bench) report=rust ;; + js) report=js ;; + *) + echo "$usage" >&2 + exit 2 + ;; +esac + +if [[ -n $report ]]; then + root=$(git rev-parse --show-toplevel) + out=${MOQ_BENCH_RESULTS:-$root/target/bench/nightly/$1} + mkdir -p "$out" + export MOQ_BENCH_RUNS="$out/criterion-runs.txt" + : >"$MOQ_BENCH_RUNS" + rm -f "$out/report.json" +fi failed=() run() { @@ -14,7 +33,21 @@ run() { if ! "$@"; then failed+=("$*"); fi } -case "${1:-}" in +criterion() { + local label=$1 + shift + run env MOQ_BENCH_LABEL="$label" MOQ_BENCH_ARTIFACTS="$out/runs" just gh criterion "$@" +} + +javascript() { + local label=$1 + shift + mkdir -p "$out/raw" + echo "nightly: $*" + if ! "$@" | tee "$out/raw/$label.csv"; then failed+=("$*"); fi +} + +case "$1" in # Ignored benchmark tests under nextest: the release builds first, then the test-profile ones. rust-test) run just rs bench-latency @@ -34,38 +67,34 @@ case "${1:-}" in --run-ignored only --success-output immediate run just rs test -p moq-cli -E 'test(benchmark_stream_import)' --run-ignored only --success-output immediate ;; - # Cargo bench targets, all in the bench profile. + # Measured Cargo bench targets, all in the bench profile. rust-bench) # Fuzz exposes moq-net's codec benches; it is not a feature of the other crates. - run cargo bench --locked --no-fail-fast -p moq-net --features fuzz --bench '*' -- --test - # Exercise the optional thread-CPU clock as well as the normal elapsed-time smoke. - run env SESSION_CPU=1 cargo bench --locked -p moq-net --bench session -- session_delivery_group_cost_cpu --test - # Each kio routine once, including the contended bench's worker threads. - run cargo bench --locked --no-fail-fast -p kio --bench '*' -- --test + criterion net --no-fail-fast -p moq-net --features fuzz --bench '*' + # Measure the optional thread-CPU clock as well as the normal elapsed time. + SESSION_CPU=1 criterion net-cpu -p moq-net --bench session -- session_delivery_group_cost_cpu + # Measure each kio routine, including the contended bench's worker threads. + criterion kio --no-fail-fast -p kio --bench '*' # One invocation unifies this release build instead of rebuilding shared crates # independently for each target. Keep the explicit targets: '*' would add coverage. - run cargo bench --locked --no-fail-fast -p moq-relay -p moq-stats -p moq-mux -p moq-rtmp -p moq-uring -p moq-pattern \ + criterion shared --no-fail-fast -p moq-relay -p moq-stats -p moq-mux -p moq-rtmp -p moq-uring -p moq-pattern \ -p moq-quic --features moq-stats/bench \ --bench http_idle --bench auth --bench producer_tick --bench decode --bench aggregate \ --bench catalog --bench ts_schedule --bench export --bench containers --bench matching \ --bench chunk_interleaving --bench worker_turns \ - --bench streams --bench receive --bench endpoint --bench wire -- --test + --bench streams --bench receive --bench endpoint --bench wire # moq-gst's catalog bench shares a target name with moq-mux's, so it runs on its own. - run just rs bench-gst-catalog -- --quick + criterion gst -p moq-gst --bench catalog ;; js) for benchmark in broadcasts request-ids requests forward reader frames varint lite-varint track interests priority; do - run just js bench "$benchmark" + javascript "$benchmark" just js bench "$benchmark" done - run just js catalog-bench - run just js arrivals-bench - run just js p2p-bench - run just js e2ee-bench - run just js signals-bench - ;; - *) - echo "$usage" >&2 - exit 2 + javascript catalog just js catalog-bench + javascript arrivals just js arrivals-bench + javascript p2p just js p2p-bench + javascript e2ee just js e2ee-bench + javascript signals just js signals-bench ;; esac @@ -73,3 +102,7 @@ if ((${#failed[@]})); then printf 'nightly: failed: %s\n' "${failed[@]}" >&2 exit 1 fi + +if [[ -n $report ]]; then + just gh benchmark-report "$report" "$out" "$out/report.json" +fi diff --git a/sh/gh/nightly.test.py b/sh/gh/nightly.test.py index eef6e450997..1d5df2cfbe8 100644 --- a/sh/gh/nightly.test.py +++ b/sh/gh/nightly.test.py @@ -24,7 +24,7 @@ def run_group(self, group, fail=""): ) stub.chmod(0o755) env = dict(os.environ, PATH=f"{scratch}:{os.environ['PATH']}", - NIGHTLY_CALLS=str(calls), NIGHTLY_FAIL=fail, BASH_ENV="/dev/null") + MOQ_BENCH_RESULTS=str(scratch / "results"), NIGHTLY_CALLS=str(calls), NIGHTLY_FAIL=fail, BASH_ENV="/dev/null") result = subprocess.run(["bash", "sh/gh/nightly.sh", group], cwd=ROOT, env=env, capture_output=True, text=True) return result, calls.read_text().splitlines() if calls.exists() else [] @@ -39,9 +39,9 @@ def test_failed_js_benchmark_does_not_hide_later_work(self): def test_bench_targets_share_one_invocation(self): result, calls = self.run_group("rust-bench") self.assertEqual(result.returncode, 0, result.stderr) - release = [call for call in calls if call.startswith("cargo bench")] - self.assertEqual(len(release), 4) # moq-net, its CPU clock, kio, then the shared build. - self.assertTrue(any("--bench session -- session_delivery_group_cost_cpu --test" in call for call in release)) + release = [call for call in calls if call.startswith("just gh criterion")] + self.assertEqual(len(release), 5) # moq-net, its CPU clock, kio, the shared build, then GStreamer. + self.assertTrue(any("--bench session -- session_delivery_group_cost_cpu" in call for call in release)) shared = [call for call in release if "-p moq-relay" in call] self.assertEqual(len(shared), 1) self.assertIn("--no-fail-fast", shared[0]) @@ -53,12 +53,13 @@ def test_bench_targets_share_one_invocation(self): self.assertIn("--bench containers", shared[0]) self.assertIn("-p moq-quic", shared[0]) self.assertIn("--bench wire", shared[0]) - self.assertEqual(calls[-1], "just rs bench-gst-catalog -- --quick") + self.assertEqual(release[-1], "just gh criterion -p moq-gst --bench catalog") + self.assertTrue(calls[-1].startswith("just gh benchmark-report rust ")) def test_benchmark_tests_stay_out_of_cargo_bench(self): result, calls = self.run_group("rust-test") self.assertEqual(result.returncode, 0, result.stderr) - self.assertFalse(any(call.startswith("cargo bench") for call in calls)) + self.assertFalse(any(call.startswith(("cargo bench", "just gh")) for call in calls)) # Release builds run before the test-profile ones. profiles = ["--release" in call or " bench-" in call for call in calls] self.assertEqual(profiles, sorted(profiles, reverse=True)) @@ -67,11 +68,12 @@ def test_benchmark_tests_stay_out_of_cargo_bench(self): def test_failed_release_build_does_not_hide_later_work(self): _, calls = self.run_group("rust-bench") - build = next(call.removeprefix("cargo ") for call in calls - if call.startswith("cargo bench") and "-p moq-relay" in call) + build = next(call.removeprefix("just ") for call in calls + if call.startswith("just gh criterion") and "-p moq-relay" in call) result, calls = self.run_group("rust-bench", build) self.assertEqual(result.returncode, 1) - self.assertEqual(calls[-1], "just rs bench-gst-catalog -- --quick") + # A failed group publishes no report, but every later target still runs. + self.assertEqual(calls[-1], "just gh criterion -p moq-gst --bench catalog") def test_unknown_group_fails_before_running_anything(self): result, calls = self.run_group("unknown")