From b4b5984a8f7dfea869d91857dbf3b6c122612cb3 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:32:41 -0700 Subject: [PATCH 1/5] chore: claim benchmark trend site quest Co-Authored-By: GPT-6 From 80a9ab15fc804ca525dd9a28862b6a9f29d04fd3 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:01:46 -0700 Subject: [PATCH 2/5] ci(bench): publish measured nightly benchmark trends Co-Authored-By: GPT-6 --- .github/justfile | 16 +++ .github/workflows/nightly.yml | 84 ++++++++++++++ bench/README.md | 60 +++++++++- bench/metadata.py | 6 +- bench/run.sh | 27 ++++- js/net/bench/frames.ts | 3 + js/net/bench/lite-varint.ts | 2 +- js/net/bench/requests.ts | 3 + js/publish/bench/catalog.ts | 43 +++----- quest/m1/bench-site.md | 5 + sh/gh/bench-publish.sh | 36 ++++++ sh/gh/bench-report.py | 202 ++++++++++++++++++++++++++++++++++ sh/gh/bench-report.test.py | 135 +++++++++++++++++++++++ sh/gh/nightly.sh | 54 +++++++-- sh/gh/nightly.test.py | 20 ++-- 15 files changed, 641 insertions(+), 55 deletions(-) create mode 100755 sh/gh/bench-publish.sh create mode 100644 sh/gh/bench-report.py create mode 100644 sh/gh/bench-report.test.py diff --git a/.github/justfile b/.github/justfile index 209acb7000e..04db16643c0 100644 --- a/.github/justfile +++ b/.github/justfile @@ -32,6 +32,7 @@ test: python3 sh/js/workspace.test.py python3 sh/rs/test-lints.test.py python3 sh/gh/nightly.test.py + python3 sh/gh/bench-report.test.py sh/go/publish-wrapper.test.sh bash .claude/hooks/direnv.test.sh python3 sh/rs/uring.test.py @@ -83,7 +84,22 @@ gst-formula *args: nightly group: sh/gh/nightly.sh {{ quote(group) }} +# Measure selected Criterion targets through the existing benchmark lifecycle. +[positional-arguments] +criterion *args: + bench/run.sh --criterion "$@" + +# Convert, validate, and stage nightly benchmark history. +[positional-arguments] +benchmark-report *args: + python3 sh/gh/bench-report.py "$@" + # Select shared native artifacts or check an upstream FFI publisher. [positional-arguments] ffi *args: python3 sh/gh/ffi.py "$@" + +# Prepare the shared history branch or publish its generated charts. +[positional-arguments] +benchmark-publish *args: + sh/gh/bench-publish.sh "$@" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c5920985377..83741bea5b8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -66,6 +66,7 @@ jobs: recipe: bench-ffi artifact: bench-ffi artifact-path: target/bench/ffi + benchmark: true - name: Audio grouping relay benchmark recipe: bench-audio # Smoke-tests the capture path: exact off-CPU waits, resolved worker stacks and CPU. @@ -74,8 +75,14 @@ jobs: perf: true - name: Rust benchmarks recipe: gh nightly rust + artifact: bench-rust + artifact-path: target/bench/nightly/rust + benchmark: true - name: JS benchmarks recipe: gh nightly js + artifact: bench-js + artifact-path: target/bench/nightly/js + benchmark: true uncached: true - name: BBB rendition alignment recipe: pub check-bbb @@ -136,10 +143,15 @@ jobs: run: nix develop --command bash -c "bun install --frozen-lockfile && bunx playwright install --with-deps chromium" - name: Run workload + id: workload run: nix develop ${{ matrix.shell }} --command just ${{ matrix.recipe }} env: NEXTEST_PROFILE: ${{ matrix.profile || 'default' }} + - name: Record benchmark outcome + if: ${{ matrix.benchmark && !cancelled() }} + run: nix develop --command just gh benchmark-report status "${{ steps.workload.outcome }}" "${{ matrix.artifact-path }}" + - name: Preserve results if: ${{ matrix.artifact && !cancelled() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -148,6 +160,78 @@ jobs: path: ${{ matrix.artifact-path }} retention-days: 45 + # One writer owns gh-pages, including future Criterion regression history. + # Explicit Pages deployment works with GITHUB_TOKEN; a branch push alone does not. + benchmark-site: + name: Benchmark site + needs: nightly + if: ${{ !cancelled() && github.ref == 'refs/heads/main' }} + runs-on: ubuntu-24.04-arm + timeout-minutes: 15 + concurrency: + group: benchmark-pages + cancel-in-progress: false + permissions: + contents: write + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deploy.outputs.page_url }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # Only this main-branch publisher pushes the shared history branch. + persist-credentials: true + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + pattern: bench-* + path: benchmark-inputs + - name: Validate completed reports + id: reports + run: nix develop --command just gh benchmark-report prepare benchmark-inputs benchmark-reports + - name: Prepare history + run: nix develop --command just gh benchmark-publish init + - name: Store time, CPU and ratio history + if: steps.reports.outputs.costs == 'true' + uses: benchmark-action/github-action-benchmark@4322e5726e6334590d251fc4f92bec0efafc45dc # v1.22.2 + with: + name: MoQ nightly costs + tool: customSmallerIsBetter + output-file-path: benchmark-reports/costs.json + benchmark-data-dir-path: bench/costs + skip-fetch-gh-pages: true + github-token: ${{ github.token }} + ref: ${{ github.sha }} + auto-push: false + comment-on-alert: false + fail-on-alert: false + - name: Store throughput history + if: steps.reports.outputs.rates == 'true' + uses: benchmark-action/github-action-benchmark@4322e5726e6334590d251fc4f92bec0efafc45dc # v1.22.2 + with: + name: MoQ nightly throughput + tool: customBiggerIsBetter + output-file-path: benchmark-reports/rates.json + benchmark-data-dir-path: bench/rates + skip-fetch-gh-pages: true + github-token: ${{ github.token }} + ref: ${{ github.sha }} + auto-push: false + comment-on-alert: false + fail-on-alert: false + - name: Publish shared history + run: nix develop --command just gh benchmark-publish finish benchmark-site + - uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4 + with: + path: benchmark-site + - name: Deploy charts + id: deploy + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 + tests: name: Tests (${{ matrix.recipe }}) runs-on: ubuntu-24.04-arm diff --git a/bench/README.md b/bench/README.md index 36b665f6282..f1505324839 100644 --- a/bench/README.md +++ b/bench/README.md @@ -189,5 +189,61 @@ Rust Criterion coverage includes `moq-pattern`'s and payload size; CMAF also varies track count. Separate frame and byte rates use the same fixtures, with byte rates counting the encoded stream size. Fixture generation, publisher setup and destruction stay outside timing. -The existing shared Rust benchmark nightly invocation runs both targets; their `--test` mode -exercises every case without making timing claims. +The shared Rust benchmark nightly invocation measures both targets in quick mode. + +## Nightly trends + +The trend site will be at once the +maintainer enables GitHub Pages with **Source: GitHub Actions**. The Nightly +workflow keeps history on one `gh-pages` branch and deploys the generated +`github-action-benchmark` charts explicitly. The publisher runs only on `main`, +serializes history writes, and leaves regression alerts and comments disabled. +Enabling Pages is a separate repository setting; merging the workflow does not +change it. + +The charts include these measurements: + +- Binding delivery in MB/s, process CPU in ms/MB, sync calls in ns, and each + wrapper's ratio to `rust-ffi`. Ratios above 1 mean more cost than Rust on the + same host, including for throughput, where the ratio is Rust/wrapper. +- Rust Criterion mean ns per iteration, with the measured confidence interval. + The `net-cpu` source uses the optional thread CPU clock; other sources use + elapsed time. Nightly now measures the selected targets with a short warmup + and Criterion's quick mode instead of only exercising them with `--test`. + Per-group benchmark settings can override the default warmup. Quick estimates + trade precision for nightly runtime; use full paired runs to assess a change. +- JS benchmark CSV time columns (`ns`, `us`, `ms`, and their per-operation + variants), retaining every workload dimension in each series name. + +Hosted runners vary across nights. Raw trends are informational, and binding +ratios compare languages within one run. A failed suite contributes no point; +malformed, empty, nonfinite, or duplicate measurements fail validation. The +nightly artifacts retain the reports, raw JS output, and each Criterion run's +metadata and estimates for 45 days. Relay load and audio quality are separate +workloads and do not enter these charts. + +Reproduce the chart inputs locally: + +```bash +nix develop --command just bench-ffi +nix develop --command just gh nightly rust +nix develop --command just gh nightly js +``` + +The latter two write `target/bench/nightly/{rust,js}/report.json`. For a focused +measured Rust run using the same evidence lifecycle: + +```bash +nix develop --command env MOQ_BENCH_LABEL=pattern MOQ_BENCH_RUNS=/tmp/criterion-runs.txt just gh criterion -p moq-pattern --bench matching +nix develop --command just gh benchmark-report rust /tmp /tmp/rust-report.json +``` + +A new source uploads a `bench-*` artifact containing `report.json`, a nonempty +JSON array of `{ "name": "case/dimensions", "unit": "ns", "value": 123 }`, +and `outcome.json` containing `{ "status": "success" }`. Names must be unique +within a source; dimensions belong in the name. Supported units are `ns`, `us`, +`µs`, `ms`, `s`, `ms/MB`, `x`, `bytes`, `B`, `MiB`, `MB/s`, `MiB/s`, `B/s`, +`ops/s`, and `frames/s`. Optional `range` and `extra` fields retain uncertainty +and context. Validate and split sources into cost and throughput charts with +`just gh benchmark-report prepare INPUT_DIRECTORY OUTPUT_DIRECTORY`; extending +the site requires no new chart renderer or data branch. diff --git a/bench/metadata.py b/bench/metadata.py index 5497cacbf0c..aee7f71391e 100644 --- a/bench/metadata.py +++ b/bench/metadata.py @@ -27,12 +27,12 @@ def command(*args): "rustc": command("rustc", "-vV"), "cargo": command("cargo", "-V"), "os": platform.platform(), "machine": platform.machine(), "processor": platform.processor(), "affinity": sorted(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else None, - "environment": {key: value for key, value in os.environ.items() if key.startswith(("MOQ_BENCH_", "CARGO_PROFILE_")) or key in ("RUSTFLAGS", "CARGO_BUILD_TARGET", "CARGO_BUILD_RUSTFLAGS", "TOKIO_WORKER_THREADS", "LD_PRELOAD", "MALLOC_CONF", "GLIBC_TUNABLES", "RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER")}, - "profile": "profiling" if mode == "--lock" else "release", "features": "default + moq-relay/io-uring,moq-bench/uring" if mode == "--runtime" else "default", + "environment": {key: value for key, value in os.environ.items() if key.startswith(("MOQ_BENCH_", "CARGO_PROFILE_")) or key in ("RUSTFLAGS", "CARGO_BUILD_TARGET", "CARGO_BUILD_RUSTFLAGS", "TOKIO_WORKER_THREADS", "LD_PRELOAD", "MALLOC_CONF", "GLIBC_TUNABLES", "RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER", "SESSION_CPU")}, + "profile": "profiling" if mode == "--lock" else "release", "features": "see criterion-command.json" if mode == "--criterion" else "default + moq-relay/io-uring,moq-bench/uring" if mode == "--runtime" else "default", "mode": mode, "runtime_workers": sys.argv[4] if len(sys.argv) > 4 else None, "rounds": int(sys.argv[5]) if len(sys.argv) > 5 else None, "allocator": "system by build; preload and allocator tunables are recorded in environment", - "workload": {"startup": "2s", "duration": "10s", "report": "500ms"}} + "workload": None if mode == "--criterion" else {"startup": "2s", "duration": "10s", "report": "500ms"}} (out / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") subprocess.run(["cargo", "tree", "--locked", "-p", "moq-relay", "-e", "features"], stdout=(out / "features.txt").open("w"), check=True) if Path("/proc/cpuinfo").exists(): diff --git a/bench/run.sh b/bench/run.sh index d1fc701c7db..8e04dd461cb 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -17,6 +17,21 @@ if [[ ! $ROUNDS =~ ^[0-9]+$ ]] || ((10#$ROUNDS < 1)); then fi ROUNDS=$((10#$ROUNDS)) case $MODE in + --criterion) + BASE= + CRITERION_CARGO_ARGS=() + CRITERION_FILTER_ARGS=() + parsing_filters=false + for argument in "${@:2}"; do + if [[ $argument == -- && $parsing_filters == false ]]; then + parsing_filters=true + elif [[ $parsing_filters == true ]]; then + CRITERION_FILTER_ARGS+=("$argument") + else + CRITERION_CARGO_ARGS+=("$argument") + fi + done + ;; --audio) BASE= ;; @@ -534,7 +549,17 @@ run_lock_profile() { cd "$ROOT" -if [[ $MODE == --audio ]]; then +if [[ $MODE == --criterion ]]; then + # Each invocation owns fresh results; previous runs cannot become new points. + python3 -c 'import json,sys; from pathlib import Path; Path(sys.argv[1]).write_text(json.dumps(sys.argv[2:]))' \ + "$RUN/criterion-command.json" "${@:2}" + if [[ -n ${MOQ_BENCH_RUNS:-} ]]; then + printf '%s\n' "$RUN" >>"$MOQ_BENCH_RUNS" + fi + CARGO_TARGET_DIR=$CURRENT_TARGET cargo bench --locked ${CRITERION_CARGO_ARGS[@]+"${CRITERION_CARGO_ARGS[@]}"} -- \ + --quick --warm-up-time 0.05 --measurement-time 0.2 --noplot ${CRITERION_FILTER_ARGS[@]+"${CRITERION_FILTER_ARGS[@]}"} + exit 0 +elif [[ $MODE == --audio ]]; then run_audio_comparison exit 0 elif [[ $MODE == --runtime ]]; then diff --git a/js/net/bench/frames.ts b/js/net/bench/frames.ts index bff942ce72c..eb2ac881bf4 100644 --- a/js/net/bench/frames.ts +++ b/js/net/bench/frames.ts @@ -13,6 +13,9 @@ import * as Path from "../src/path.ts"; import { Reader, Stream } from "../src/stream.ts"; import * as Varint from "../src/varint.ts"; +// Keep protocol diagnostics out of the measured CSV on stdout. +console.debug = console.error; + const frameSizes = [16, 100, 1000]; const chunkSizes = [1200, 16 * 1024, 64 * 1024]; const framesPerGroup = 50; diff --git a/js/net/bench/lite-varint.ts b/js/net/bench/lite-varint.ts index 148fb937ff7..4f335e79355 100644 --- a/js/net/bench/lite-varint.ts +++ b/js/net/bench/lite-varint.ts @@ -79,7 +79,7 @@ const subscribe: Sample = { const datagram: Sample = { name: "Datagram", encode: (w, version) => w.write(new Datagram(3n, 1_234, 1_234_567_890, new Uint8Array()).encode(version)), - decode: (r, version) => r.readAll().then((data) => Datagram.decode(data, version)), + decode: (r, version) => r.readAll().then((data) => Datagram.decode(data, version, () => true)), }; const hop = randomHop(); diff --git a/js/net/bench/requests.ts b/js/net/bench/requests.ts index 0daf1b294ed..2f16119e7ec 100644 --- a/js/net/bench/requests.ts +++ b/js/net/bench/requests.ts @@ -16,6 +16,9 @@ import * as Path from "../src/path"; import { Stream } from "../src/stream"; import { wireOf } from "../src/wire"; +// Keep protocol diagnostics out of the measured CSV on stdout. +console.debug = console.error; + const settle = () => new Promise((resolve) => setImmediate(resolve)); console.log("touched,routes,followers,update_us"); for (const touched of ["watched", "unrelated"]) diff --git a/js/publish/bench/catalog.ts b/js/publish/bench/catalog.ts index 75e3cc8ee92..623a64694d3 100644 --- a/js/publish/bench/catalog.ts +++ b/js/publish/bench/catalog.ts @@ -1,26 +1,13 @@ /** Sweep tracks x viewers for 1000 estimate rises inside one second. */ -import { Milli } from "@moq/clock"; +import { Manual, Milli, Nano } from "@moq/clock"; import * as Catalog from "@moq/hang/catalog"; import * as Json from "@moq/json"; import { Track } from "@moq/net"; import { Effect } from "@moq/signals"; import { CatalogProducer } from "../src/catalog"; -const realNow = performance.now.bind(performance); -const realTimer = Effect.prototype.timer; -let now = 0; -let timers = new Set<{ at: number; run: () => void }>(); -performance.now = () => now; -Effect.prototype.timer = function (fn, ms) { - const timer = { at: now + ms, run: fn }; - timers.add(timer); - this.cleanup(() => timers.delete(timer)); -}; - -async function measure(tracks: number, viewers: number) { - now = 0; - timers = new Set(); +async function measure(tracks: number, viewers: number, clock: Manual) { const scope = new Effect(); const catalog = new CatalogProducer(); const media = Array.from({ length: tracks }, (_, index) => new Track.Producer(`v${index}`)); @@ -68,19 +55,18 @@ async function measure(tracks: number, viewers: number) { } } }; - const start = realNow(); + const start = performance.now(); for (let step = 0; step < 1000; step++) { - now = step; + if (step > 0) clock.advance(Nano.fromMilli(Milli(1))); catalog.mutate((root) => { if (!root.video) throw new Error("video missing"); for (const config of Object.values(root.video.renditions)) config.jitter = Catalog.u53(step + 1); }); await receive(); } - now = 1000; - for (const timer of [...timers]) if (timer.at <= now) timer.run(); + await clock.advanceAsync(Nano.fromMilli(Milli(1))); await receive(); - const elapsed = realNow() - start; + const elapsed = performance.now() - start; if (publishes !== 2 || updates !== 2 * tracks * viewers) { throw new Error(`estimate churn regression: ${publishes} publishes, ${updates} subscribe updates`); } @@ -94,10 +80,15 @@ async function measure(tracks: number, viewers: number) { for (const track of media) track.close(); } -try { - console.log("tracks,viewers,catalog_publishes,subscribe_updates,unlimited_subscribe_updates,elapsed_ms"); - for (const tracks of [1, 4, 16]) for (const viewers of [1, 4, 16]) await measure(tracks, viewers); -} finally { - performance.now = realNow; - Effect.prototype.timer = realTimer; +console.log("tracks,viewers,catalog_publishes,subscribe_updates,unlimited_subscribe_updates,elapsed_ms"); +for (const tracks of [1, 4, 16]) { + for (const viewers of [1, 4, 16]) { + const clock = new Manual(); + const restore = clock.install(); + try { + await measure(tracks, viewers, clock); + } finally { + restore(); + } + } } diff --git a/quest/m1/bench-site.md b/quest/m1/bench-site.md index 4d9e56f7228..24eb5c228a2 100644 --- a/quest/m1/bench-site.md +++ b/quest/m1/bench-site.md @@ -10,6 +10,11 @@ site. ## Plan +The implementation is prepared in a draft PR. The remaining maintainer action +is enabling repository Pages with **Source: GitHub Actions**, then verifying +the first Nightly deployment at . +Repository settings have not been changed. + Decided with the maintainer on 2026-10-15: - **GitHub Pages via github-action-benchmark**: the nightly pushes each diff --git a/sh/gh/bench-publish.sh b/sh/gh/bench-publish.sh new file mode 100755 index 00000000000..6ee65b8e41f --- /dev/null +++ b/sh/gh/bench-publish.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +case ${1:-} in + init) + # ls-remote distinguishes an absent history branch from an authentication failure. + if git ls-remote --exit-code --heads origin gh-pages; then + git fetch origin gh-pages:gh-pages + else + status=$? + if [[ $status != 2 ]]; then exit "$status"; fi + tree=$(git hash-object -t tree /dev/null) + commit=$(git -c user.name=github-actions -c user.email=github-actions@users.noreply.github.com \ + commit-tree "$tree" -m 'Initialize benchmark history') + git branch gh-pages "$commit" + fi + ;; + finish) + if [[ $# != 2 ]]; then + echo 'usage: benchmark-publish finish DIRECTORY' >&2 + exit 2 + fi + git worktree add "$2" gh-pages + just gh benchmark-report index "$2" + git -C "$2" add bench/index.html .nojekyll + if ! git -C "$2" diff --cached --quiet; then + git -C "$2" -c user.name=github-actions -c user.email=github-actions@users.noreply.github.com \ + commit -m 'Update benchmark landing page' + fi + git -C "$2" push origin HEAD:gh-pages + ;; + *) + echo 'usage: benchmark-publish {init|finish DIRECTORY}' >&2 + exit 2 + ;; +esac diff --git a/sh/gh/bench-report.py b/sh/gh/bench-report.py new file mode 100644 index 00000000000..f554fc07b1a --- /dev/null +++ b/sh/gh/bench-report.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Convert measured nightly results to github-action-benchmark's JSON format.""" + +import argparse +import csv +import html +import json +import math +import os +from pathlib import Path +import re +from urllib.parse import quote + + +BIGGER = {'MB/s', 'MiB/s', 'B/s', 'ops/s', 'frames/s'} +SMALLER = {'ns', 'us', 'µs', 'ms', 's', 'ms/MB', 'x', 'bytes', 'B', 'MiB'} + + +def value(number): + if isinstance(number, bool) or not isinstance(number, (int, float)) or not math.isfinite(number) or number < 0: + raise ValueError(f'invalid measurement: {number!r}') + return number + + +def checked(entries): + if not isinstance(entries, list) or not entries: + raise ValueError('a benchmark report must contain measured entries') + names = set() + for entry in entries: + if not isinstance(entry, dict) or not isinstance(entry.get('name'), str) or not entry['name']: + raise ValueError(f'invalid benchmark entry: {entry!r}') + if entry['name'] in names: + raise ValueError(f'duplicate benchmark: {entry["name"]}') + names.add(entry['name']) + if entry.get('unit') not in BIGGER | SMALLER: + raise ValueError(f'unsupported measurement unit: {entry.get("unit")!r}') + value(entry.get('value')) + return entries + + +def write(path, entries): + checked(entries) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(entries, indent=2) + '\n') + + +def rust(directory): + manifest = directory / 'criterion-runs.txt' + runs = [Path(line) for line in manifest.read_text().splitlines() if line] + if not runs: + raise ValueError(f'no measured Criterion runs in {manifest}') + entries = [] + for run in runs: + metadata = json.loads((run / 'metadata.json').read_text()) + if metadata['mode'] != '--criterion': + raise ValueError(f'not a measured Criterion run: {run}') + label = metadata['environment'].get('MOQ_BENCH_LABEL') + if not isinstance(label, str) or not label: + raise ValueError(f'missing Criterion source label: {run}') + estimates = sorted((run / 'criterion').glob('**/new/estimates.json')) + if not estimates: + raise ValueError(f'Criterion produced no measured estimates: {run}') + for path in estimates: + benchmark = json.loads(path.with_name('benchmark.json').read_text()) + estimate = json.loads(path.read_text())['mean'] + interval = estimate['confidence_interval'] + low, high = value(interval['lower_bound']), value(interval['upper_bound']) + clock = 'thread CPU' if metadata['environment'].get('SESSION_CPU') else 'elapsed time' + entries.append({ + 'name': f'rust/{label}/{benchmark["full_id"]}', 'unit': 'ns', + 'value': value(estimate['point_estimate']), 'range': f'{low:.4g}..{high:.4g}', + 'extra': f'Mean nanoseconds per iteration ({clock}); {metadata["revision"]}', + }) + return checked(entries) + + +def timing_unit(column): + for unit in ('ns', 'us', 'ms'): + if column == unit or column.endswith(f'_{unit}') or column.startswith(f'{unit}_per_'): + return unit + return None + + +def javascript_file(path): + entries, headers = [], None + for line_number, line in enumerate(path.read_text().splitlines(), 1): + # The interests sweep reports dimensions and elapsed time in one line. + interests = re.fullmatch(r'broadcasts=(\d+) tracks=(\d+) acquire\+release=([\d.eE+-]+) us', line) + if interests: + broadcasts, tracks, elapsed = interests.groups() + entries.append({'name': f'js/{path.stem}/broadcasts={broadcasts}/tracks={tracks}/acquire+release', + 'unit': 'us', 'value': value(float(elapsed))}) + continue + row = [field.strip() for field in next(csv.reader([line]))] + if len(row) > 1 and any(timing_unit(field) for field in row): + if len(set(row)) != len(row): + raise ValueError(f'duplicate CSV headers: {path}:{line_number}') + headers = row + continue + if not headers or not line.strip(): + continue + if len(row) != len(headers): + raise ValueError(f'malformed benchmark CSV: {path}:{line_number}') + dimensions = '/'.join(f'{key}={quote(field, safe="-_.~")}' for key, field in zip(headers, row) + if timing_unit(key) is None) + for key, field in zip(headers, row): + if unit := timing_unit(key): + entries.append({'name': f'js/{path.stem}/{dimensions}/{key}', + 'unit': unit, 'value': value(float(field))}) + if not entries: + raise ValueError(f'JS benchmark produced no measurements: {path}') + return checked(entries) + + +def javascript(directory): + entries = [entry for path in sorted((directory / 'raw').glob('*.csv')) for entry in javascript_file(path)] + return checked(entries) + + +def prepare(directory, output): + costs, rates = [], [] + for source in sorted(directory.glob('bench-*')): + status = source / 'outcome.json' + if status.exists(): + outcome = json.loads(status.read_text())['status'] + if outcome not in {'success', 'failure', 'cancelled', 'skipped'}: + raise ValueError(f'invalid benchmark outcome: {source}') + if outcome != 'success': + print(f'{source.name}: {outcome}; no history point') + continue + report = source / 'report.json' + if not report.exists(): + if status.exists(): + raise ValueError(f'successful benchmark has no report: {source}') + continue + entries = checked(json.loads(report.read_text())) + for entry in entries: + target = rates if entry['unit'] in BIGGER else costs + target.append({**entry, 'name': f'{source.name}/{entry["name"]}'}) + print(f'{source.name}: {len(entries)} measured entries') + if not costs and not rates: + raise ValueError('no completed benchmark reports to publish') + output.mkdir(parents=True, exist_ok=True) + for name, entries in [('costs', costs), ('rates', rates)]: + (output / f'{name}.json').write_text(json.dumps(entries, indent=2) + '\n') + if entries: + checked(entries) + if target := os.environ.get('GITHUB_OUTPUT'): + with open(target, 'a') as stream: + stream.write(f'{name}={str(bool(entries)).lower()}\n') + + +def index(directory): + destination = directory / 'bench' + links = [] + for name, label in [('costs', 'Time, CPU, size and Rust ratios'), ('rates', 'Delivered throughput')]: + if (destination / name / 'index.html').exists(): + links.append(f'
  • {html.escape(label)}
    • ') + if not links: + raise ValueError('no generated benchmark charts to deploy') + (destination / 'index.html').write_text(''' + +MoQ benchmarks + +

    MoQ nightly benchmarks

    +

    Hosted runners vary. These charts are informational. Binding ratios compare each wrapper +with Rust on the same host: above 1x means slower or more CPU per delivered byte.

    +
      ''' + ''.join(links) + '''
    +

    Each series keeps its source and payload shape. A failed suite contributes no point. +Metrics and local reproduction

    + +''') + (directory / '.nojekyll').touch() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + for command in ('rust', 'js', 'prepare'): + sub = commands.add_parser(command) + sub.add_argument('directory', type=Path) + sub.add_argument('output', type=Path) + sub = commands.add_parser('status') + sub.add_argument('status', choices=('success', 'failure', 'cancelled', 'skipped')) + sub.add_argument('directory', type=Path) + commands.add_parser('index').add_argument('directory', type=Path) + args = parser.parse_args() + if args.command == 'rust': + write(args.output, rust(args.directory)) + elif args.command == 'js': + write(args.output, javascript(args.directory)) + elif args.command == 'prepare': + prepare(args.directory, args.output) + elif args.command == 'status': + args.directory.mkdir(parents=True, exist_ok=True) + (args.directory / 'outcome.json').write_text(json.dumps({'status': args.status}) + '\n') + else: + index(args.directory) + + +if __name__ == '__main__': + main() diff --git a/sh/gh/bench-report.test.py b/sh/gh/bench-report.test.py new file mode 100644 index 00000000000..d387a82458f --- /dev/null +++ b/sh/gh/bench-report.test.py @@ -0,0 +1,135 @@ +"""Nightly charts accept measurements, refuse malformed data, and share one history branch.""" +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location('report', Path(__file__).with_name('bench-report.py')) +report = importlib.util.module_from_spec(spec) +spec.loader.exec_module(report) + + +class ReportTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + + def test_values_refuse_smoke_and_malformed_reports(self): + for entries in ([], {}, [None], [{'name': '', 'unit': 'ns', 'value': 1}]): + with self.assertRaises(ValueError): + report.checked(entries) + for number in (float('nan'), float('inf'), -1, None, True, '2'): + with self.assertRaises(ValueError): + report.value(number) + for entries in ([{'name': 'x', 'unit': 'unknown', 'value': 1}], + [{'name': 'x', 'unit': 'ns', 'value': 1}] * 2): + with self.assertRaises(ValueError): + report.checked(entries) + + def test_js_multiple_schemas_and_interests(self): + source = self.root / 'requests.csv' + source.write_text('bun install v1.3\npaths,active_us,settled_us\n4,1.25,2.5\noperation,ns_per_op\nclose,0\n') + entries = report.javascript_file(source) + self.assertEqual([entry['value'] for entry in entries], [1.25, 2.5, 0]) + self.assertEqual(entries[0]['name'], 'js/requests/paths=4/active_us') + source.write_text('broadcasts=32 tracks=8 acquire+release=1.250 us\n') + self.assertEqual(report.javascript_file(source)[0]['value'], 1.25) + for text in ('Testing case\nSuccess\n', 'paths,time_us\n4\n', 'paths,time_us\n4,nan\n'): + source.write_text(text) + with self.assertRaises(ValueError): + report.javascript_file(source) + + def test_criterion_requires_fresh_measured_estimates(self): + run = self.root / 'runs' / 'one' + run.mkdir(parents=True) + (self.root / 'criterion-runs.txt').write_text(f'{run}\n') + (run / 'metadata.json').write_text(json.dumps({'mode': '--criterion', 'revision': 'abc', + 'environment': {'MOQ_BENCH_LABEL': 'net-cpu', 'SESSION_CPU': '1'}})) + with self.assertRaises(ValueError): + report.rust(self.root) + case = run / 'criterion' / 'delivery' / 'new' + case.mkdir(parents=True) + (case / 'benchmark.json').write_text(json.dumps({'full_id': 'delivery/4/8'})) + (case / 'estimates.json').write_text(json.dumps({'mean': {'point_estimate': 20, + 'confidence_interval': {'lower_bound': 18, 'upper_bound': 22}}})) + entry = report.rust(self.root)[0] + self.assertEqual(entry['name'], 'rust/net-cpu/delivery/4/8') + self.assertEqual(entry['value'], 20) + self.assertIn('thread CPU', entry['extra']) + + def test_only_successful_sources_are_published(self): + source = self.root / 'bench-ffi' + source.mkdir() + entries = [{'name': 'go/round', 'unit': 'MB/s', 'value': 4}, + {'name': 'go/round vs rust-ffi', 'unit': 'x', 'value': 2}] + (source / 'report.json').write_text(json.dumps(entries)) + failed = self.root / 'bench-rust' + failed.mkdir() + (failed / 'outcome.json').write_text('{"status":"failure"}') + (failed / 'report.json').write_text('not a usable report') + output = self.root / 'output' + report.prepare(self.root, output) + self.assertEqual(json.loads((output / 'rates.json').read_text())[0]['value'], 4) + self.assertEqual(json.loads((output / 'costs.json').read_text())[0]['value'], 2) + (source / 'outcome.json').write_text('{"status":"success"}') + (source / 'report.json').unlink() + with self.assertRaises(ValueError): + report.prepare(self.root, output) + + def test_landing_page_keeps_existing_charts_and_history(self): + chart = self.root / 'bench' / 'costs' + chart.mkdir(parents=True) + (chart / 'index.html').write_text('chart') + (chart / 'data.js').write_text('history') + report.index(self.root) + self.assertIn('costs/', (chart.parent / 'index.html').read_text()) + self.assertEqual((chart / 'data.js').read_text(), 'history') + self.assertTrue((self.root / '.nojekyll').exists()) + + def test_first_and_subsequent_history_publications(self): + remote, checkout = self.root / 'remote.git', self.root / 'checkout' + def git(*args, cwd=None): + return subprocess.run(['git', *args], cwd=cwd, check=True, capture_output=True, text=True) + git('init', '--bare', str(remote)) + git('clone', str(remote), str(checkout)) + git('config', 'user.name', 'test', cwd=checkout) + git('config', 'user.email', 'test@example.com', cwd=checkout) + (checkout / 'source').write_text('source tree') + git('add', '.', cwd=checkout) + git('commit', '-m', 'Source', cwd=checkout) + commands = self.root / 'commands' + commands.mkdir() + just = commands / 'just' + just.write_text(f'#!/bin/sh\nexec python3 {ROOT}/sh/gh/bench-report.py "$3" "$4"\n') + just.chmod(0o755) + env = dict(os.environ, PATH=f'{commands}:{os.environ["PATH"]}') + publisher = ROOT / 'sh/gh/bench-publish.sh' + for iteration in (1, 2): + subprocess.run(['bash', str(publisher), 'init'], cwd=checkout, env=env, check=True, + capture_output=True) + history = self.root / f'history-{iteration}' + git('worktree', 'add', str(history), 'gh-pages', cwd=checkout) + chart = history / 'bench' / 'costs' + chart.mkdir(parents=True, exist_ok=True) + (chart / 'index.html').write_text('chart') + (chart / 'data.js').write_text(f'run {iteration}') + git('add', '.', cwd=history) + git('commit', '-m', 'Measured result', cwd=history) + git('worktree', 'remove', str(history), cwd=checkout) + site = self.root / f'site-{iteration}' + subprocess.run(['bash', str(publisher), 'finish', str(site)], cwd=checkout, env=env, + check=True, capture_output=True) + self.assertFalse((site / 'source').exists()) + self.assertEqual((site / 'bench/costs/data.js').read_text(), f'run {iteration}') + git('worktree', 'remove', str(site), cwd=checkout) + git('branch', '-D', 'gh-pages', cwd=checkout) + self.assertEqual(git('--git-dir', str(remote), 'show', 'gh-pages:bench/costs/data.js').stdout, 'run 2') + + +if __name__ == '__main__': + unittest.main() diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 40effa9a211..00e1752ca86 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -7,6 +7,20 @@ if [[ $# != 1 ]]; then echo "$usage" >&2 exit 2 fi +case "$1" in + rust | js) ;; + *) + echo "$usage" >&2 + exit 2 + ;; +esac + +root=$(git rev-parse --show-toplevel) +out=${MOQ_BENCH_RESULTS:-$root/target/bench/nightly/$1} +mkdir -p "$out" +export MOQ_BENCH_RUNS="$out/criterion-runs.txt" +: >"$MOQ_BENCH_RUNS" +rm -f "$out/report.json" failed=() run() { @@ -14,6 +28,20 @@ run() { if ! "$@"; then failed+=("$*"); fi } +criterion() { + local label=$1 + shift + run env MOQ_BENCH_LABEL="$label" MOQ_BENCH_ARTIFACTS="$out/runs" just gh criterion "$@" +} + +javascript() { + local label=$1 + shift + mkdir -p "$out/raw" + echo "nightly: $*" + if ! "$@" | tee "$out/raw/$label.csv"; then failed+=("$*"); fi +} + case "${1:-}" in rust) run just rs bench-latency @@ -25,18 +53,18 @@ case "${1:-}" in run just rs bench-feed-fetch run just rs test -p moq-relay -E 'test(benchmark_stun_sources)' --run-ignored only --success-output immediate # Fuzz exposes moq-net's codec benches; it is not a feature of the other crates. - run cargo bench --locked --no-fail-fast -p moq-net --features fuzz --bench '*' -- --test + criterion net --no-fail-fast -p moq-net --features fuzz --bench '*' # Exercise the optional thread-CPU clock as well as the normal elapsed-time smoke. - run env SESSION_CPU=1 cargo bench --locked -p moq-net --bench session -- session_delivery_group_cost_cpu --test - # Each kio routine once, including the contended bench's worker threads. - run cargo bench --locked --no-fail-fast -p kio --bench '*' -- --test + SESSION_CPU=1 criterion net-cpu -p moq-net --bench session -- session_delivery_group_cost_cpu + # Measure each kio routine, including the contended bench's worker threads. + criterion kio --no-fail-fast -p kio --bench '*' # One invocation unifies this release build instead of rebuilding shared crates # independently for each target. Keep the explicit targets: '*' would add coverage. - run cargo bench --locked --no-fail-fast -p moq-relay -p moq-stats -p moq-mux -p moq-rtmp -p moq-uring -p moq-pattern \ + criterion shared --no-fail-fast -p moq-relay -p moq-stats -p moq-mux -p moq-rtmp -p moq-uring -p moq-pattern \ --features moq-stats/bench \ --bench http_idle --bench auth --bench producer_tick --bench decode --bench aggregate \ --bench catalog --bench ts_schedule --bench export --bench containers --bench matching \ - --bench chunk_interleaving --bench worker_turns -- --test + --bench chunk_interleaving --bench worker_turns run just rs test -p moq-archive -p moq-mux -p moq-hls --no-fail-fast \ -E 'test(benchmark_timeline_fetch_fanout) | test(benchmark_damage_pid_fanout) | test(benchmark_pes_staging) | test(benchmark_reference_handoff_fanout)' \ --run-ignored only --success-output immediate @@ -44,17 +72,17 @@ case "${1:-}" in run just rs test -p moq-ffi --no-default-features -E 'test(benchmark_timebase_reservation_fanout)' \ --run-ignored only --success-output immediate # moq-gst's catalog bench shares a target name with moq-mux's, so it runs on its own. - run just rs bench-gst-catalog -- --quick + criterion gst -p moq-gst --bench catalog run just rs test -p moq-cli -E 'test(benchmark_stream_import)' --run-ignored only --success-output immediate ;; js) for benchmark in broadcasts request-ids requests forward reader frames varint lite-varint track interests priority; do - run just js bench "$benchmark" + javascript "$benchmark" just js bench "$benchmark" done - run just js catalog-bench - run just js arrivals-bench - run just js p2p-bench - run just js signals-bench + javascript catalog just js catalog-bench + javascript arrivals just js arrivals-bench + javascript p2p just js p2p-bench + javascript signals just js signals-bench ;; *) echo "$usage" >&2 @@ -66,3 +94,5 @@ if ((${#failed[@]})); then printf 'nightly: failed: %s\n' "${failed[@]}" >&2 exit 1 fi + +just gh benchmark-report "$1" "$out" "$out/report.json" diff --git a/sh/gh/nightly.test.py b/sh/gh/nightly.test.py index d2a96d1d8f5..99db7be4da6 100644 --- a/sh/gh/nightly.test.py +++ b/sh/gh/nightly.test.py @@ -18,13 +18,13 @@ def run_group(self, group, fail=""): for command in ("just", "cargo"): stub = scratch / command stub.write_text( - '#!/bin/bash\n' + '#!/usr/bin/env bash\n' 'printf "%s %s\n" "${0##*/}" "$*" >> "$NIGHTLY_CALLS"\n' '[[ "$*" != "$NIGHTLY_FAIL" ]]\n' ) stub.chmod(0o755) env = dict(os.environ, PATH=f"{scratch}:{os.environ['PATH']}", - NIGHTLY_CALLS=str(calls), NIGHTLY_FAIL=fail, BASH_ENV="/dev/null") + MOQ_BENCH_RESULTS=str(scratch / "results"), NIGHTLY_CALLS=str(calls), NIGHTLY_FAIL=fail, BASH_ENV="/dev/null") result = subprocess.run(["bash", "sh/gh/nightly.sh", group], cwd=ROOT, env=env, capture_output=True, text=True) return result, calls.read_text().splitlines() if calls.exists() else [] @@ -39,9 +39,9 @@ def test_failed_js_benchmark_does_not_hide_later_work(self): def test_release_benchmarks_share_one_invocation(self): result, calls = self.run_group("rust") self.assertEqual(result.returncode, 0, result.stderr) - release = [call for call in calls if call.startswith("cargo bench")] - self.assertEqual(len(release), 4) # moq-net, its CPU clock, kio, then the shared build. - self.assertTrue(any("--bench session -- session_delivery_group_cost_cpu --test" in call for call in release)) + release = [call for call in calls if call.startswith("just gh criterion")] + self.assertEqual(len(release), 5) # moq-net, its CPU clock, kio, the shared build, then GStreamer. + self.assertTrue(any("--bench session -- session_delivery_group_cost_cpu" in call for call in release)) shared = [call for call in release if "-p moq-relay" in call] self.assertEqual(len(shared), 1) self.assertIn("--no-fail-fast", shared[0]) @@ -51,16 +51,16 @@ def test_release_benchmarks_share_one_invocation(self): self.assertIn("-p moq-pattern", shared[0]) self.assertIn("--bench matching", shared[0]) self.assertIn("--bench containers", shared[0]) - self.assertIn("just rs bench-gst-catalog -- --quick", calls) - self.assertTrue(calls[-1].startswith("just rs test -p moq-cli")) + self.assertIn("just gh criterion -p moq-gst --bench catalog", calls) + self.assertTrue(any(call.startswith("just rs test -p moq-cli") for call in calls)) def test_failed_release_build_does_not_hide_later_work(self): _, calls = self.run_group("rust") - build = next(call.removeprefix("cargo ") for call in calls - if call.startswith("cargo bench") and "-p moq-relay" in call) + build = next(call.removeprefix("just ") for call in calls + if call.startswith("just gh criterion") and "-p moq-relay" in call) result, calls = self.run_group("rust", build) self.assertEqual(result.returncode, 1) - self.assertTrue(calls[-1].startswith("just rs test -p moq-cli")) + self.assertTrue(any(call.startswith("just rs test -p moq-cli") for call in calls)) def test_unknown_group_fails_before_running_anything(self): result, calls = self.run_group("unknown") From b50f481b7078235393e9855a976facc3ec81932e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 20:57:23 -0700 Subject: [PATCH 3/5] ci(bench): chart only sources that record an outcome bench-browser and bench-ffi-gen also match the bench-* download pattern; the browser report has its own schema. Co-Authored-By: Claude Opus 5.5 --- sh/gh/bench-report.py | 20 ++++++++++---------- sh/gh/bench-report.test.py | 6 +++++- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/sh/gh/bench-report.py b/sh/gh/bench-report.py index f554fc07b1a..d46507325e6 100644 --- a/sh/gh/bench-report.py +++ b/sh/gh/bench-report.py @@ -120,19 +120,19 @@ def javascript(directory): def prepare(directory, output): costs, rates = [], [] for source in sorted(directory.glob('bench-*')): + # Other nightly artifacts share the prefix; only a recorded outcome marks a chart source. status = source / 'outcome.json' - if status.exists(): - outcome = json.loads(status.read_text())['status'] - if outcome not in {'success', 'failure', 'cancelled', 'skipped'}: - raise ValueError(f'invalid benchmark outcome: {source}') - if outcome != 'success': - print(f'{source.name}: {outcome}; no history point') - continue + if not status.exists(): + continue + outcome = json.loads(status.read_text())['status'] + if outcome not in {'success', 'failure', 'cancelled', 'skipped'}: + raise ValueError(f'invalid benchmark outcome: {source}') + if outcome != 'success': + print(f'{source.name}: {outcome}; no history point') + continue report = source / 'report.json' if not report.exists(): - if status.exists(): - raise ValueError(f'successful benchmark has no report: {source}') - continue + raise ValueError(f'successful benchmark has no report: {source}') entries = checked(json.loads(report.read_text())) for entry in entries: target = rates if entry['unit'] in BIGGER else costs diff --git a/sh/gh/bench-report.test.py b/sh/gh/bench-report.test.py index d387a82458f..8690c400eae 100644 --- a/sh/gh/bench-report.test.py +++ b/sh/gh/bench-report.test.py @@ -68,6 +68,11 @@ def test_only_successful_sources_are_published(self): entries = [{'name': 'go/round', 'unit': 'MB/s', 'value': 4}, {'name': 'go/round vs rust-ffi', 'unit': 'x', 'value': 2}] (source / 'report.json').write_text(json.dumps(entries)) + (source / 'outcome.json').write_text('{"status":"success"}') + # Another nightly artifact with the same prefix and its own report schema. + unrelated = self.root / 'bench-browser' + unrelated.mkdir() + (unrelated / 'report.json').write_text('{"samples": []}') failed = self.root / 'bench-rust' failed.mkdir() (failed / 'outcome.json').write_text('{"status":"failure"}') @@ -76,7 +81,6 @@ def test_only_successful_sources_are_published(self): report.prepare(self.root, output) self.assertEqual(json.loads((output / 'rates.json').read_text())[0]['value'], 4) self.assertEqual(json.loads((output / 'costs.json').read_text())[0]['value'], 2) - (source / 'outcome.json').write_text('{"status":"success"}') (source / 'report.json').unlink() with self.assertRaises(ValueError): report.prepare(self.root, output) From 9065d850c4e615367c5b270e950ec5e063e13275 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 21:02:00 -0700 Subject: [PATCH 4/5] bench(net): time duplicate-route ranking without the update hold The origin now holds route updates and restarts for 300 ms against its clock, so announce_duplicate and reprice_duplicate saw nothing pending and panicked. Both now run with a zero hold, as the auth renewal bench does. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/benches/origin.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/benches/origin.rs b/rs/moq-net/benches/origin.rs index 566c93be9c5..1f602cdb133 100644 --- a/rs/moq-net/benches/origin.rs +++ b/rs/moq-net/benches/origin.rs @@ -255,7 +255,10 @@ fn bench_announce_duplicate(c: &mut Criterion) { for (duplicates, subscribers) in CONTENDED { let id = BenchmarkId::from_parameter(format!("{duplicates}d_{subscribers}s")); group.bench_function(id, |b| { - let (producer, _driver) = origin::Producer::new(origin::Config::default()); + // No hold, so an iteration times the ranking rather than the origin batching it. + let mut config = origin::Config::default(); + config.update_hold = Duration::ZERO; + let (producer, _driver) = origin::Producer::new(config); let consumer = producer.consume(); // Every peer announces the one path, each under its own hop chain so // the entries are distinct routes rather than one re-priced in place. @@ -295,7 +298,10 @@ fn bench_reprice_duplicate(c: &mut Criterion) { for (duplicates, subscribers) in CONTENDED { let id = BenchmarkId::from_parameter(format!("{duplicates}d_{subscribers}s_cost{incumbent_cost}")); group.bench_function(id, |b| { - let (producer, _driver) = origin::Producer::new(origin::Config::default()); + // No hold, so an iteration times the ranking rather than the origin batching it. + let mut config = origin::Config::default(); + config.update_hold = Duration::ZERO; + let (producer, _driver) = origin::Producer::new(config); let _routes: Vec<_> = (1..=duplicates) .map(|peer| { producer From 1bfcd7e9bdb385a8ecd558b3cdc4f175aa649528 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 21:22:38 -0700 Subject: [PATCH 5/5] ci(bench): stop guessing each Criterion target's clock The moq-quic targets time thread CPU through their own Measurement, which Criterion does not record, so the report labelled them elapsed time. Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 4 ++-- sh/gh/bench-report.py | 4 ++-- sh/gh/bench-report.test.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bench/README.md b/bench/README.md index 0ca19a9b095..abd7f6ecba2 100644 --- a/bench/README.md +++ b/bench/README.md @@ -359,8 +359,8 @@ The charts include these measurements: wrapper's ratio to `rust-ffi`. Ratios above 1 mean more cost than Rust on the same host, including for throughput, where the ratio is Rust/wrapper. - Rust Criterion mean ns per iteration, with the measured confidence interval. - The `net-cpu` source uses the optional thread CPU clock; other sources use - elapsed time. The Rust bench targets job measures them with a short warmup + The `net-cpu` source and the moq-quic targets time thread CPU; the others + time elapsed time. The Rust bench targets job measures them with a short warmup and Criterion's quick mode. Per-group benchmark settings can override the default warmup. Quick estimates trade precision for nightly runtime; use full paired runs to assess a change. - JS benchmark CSV time columns (`ns`, `us`, `ms`, and their per-operation diff --git a/sh/gh/bench-report.py b/sh/gh/bench-report.py index d46507325e6..5dc89fc973c 100644 --- a/sh/gh/bench-report.py +++ b/sh/gh/bench-report.py @@ -65,11 +65,11 @@ def rust(directory): estimate = json.loads(path.read_text())['mean'] interval = estimate['confidence_interval'] low, high = value(interval['lower_bound']), value(interval['upper_bound']) - clock = 'thread CPU' if metadata['environment'].get('SESSION_CPU') else 'elapsed time' entries.append({ 'name': f'rust/{label}/{benchmark["full_id"]}', 'unit': 'ns', 'value': value(estimate['point_estimate']), 'range': f'{low:.4g}..{high:.4g}', - 'extra': f'Mean nanoseconds per iteration ({clock}); {metadata["revision"]}', + # Criterion does not record a target's measurement, so the clock is left to the README. + 'extra': f'Mean nanoseconds per iteration; {metadata["revision"]}', }) return checked(entries) diff --git a/sh/gh/bench-report.test.py b/sh/gh/bench-report.test.py index 8690c400eae..ea5da4426c7 100644 --- a/sh/gh/bench-report.test.py +++ b/sh/gh/bench-report.test.py @@ -60,7 +60,7 @@ def test_criterion_requires_fresh_measured_estimates(self): entry = report.rust(self.root)[0] self.assertEqual(entry['name'], 'rust/net-cpu/delivery/4/8') self.assertEqual(entry['value'], 20) - self.assertIn('thread CPU', entry['extra']) + self.assertIn('abc', entry['extra']) def test_only_successful_sources_are_published(self): source = self.root / 'bench-ffi'