ci(bench): publish measured nightly benchmark trends

.github/justfile

@@ -33,6 +33,7 @@ test:
     python3 sh/rs/test-lints.test.py
     python3 sh/rs/model-markers.test.py
     python3 sh/gh/nightly.test.py
+    python3 sh/gh/bench-report.test.py
     sh/go/publish-wrapper.test.sh
     bash .claude/hooks/direnv.test.sh
     python3 sh/rs/uring.test.py
@@ -84,7 +85,22 @@ gst-formula *args:
 nightly group:
     sh/gh/nightly.sh {{ quote(group) }}
 
+# Measure selected Criterion targets through the existing benchmark lifecycle.
+[positional-arguments]
+criterion *args:
+    bench/run.sh --criterion "$@"
+
+# Convert, validate, and stage nightly benchmark history.
+[positional-arguments]
+benchmark-report *args:
+    python3 sh/gh/bench-report.py "$@"
+
 # Select shared native artifacts or check an upstream FFI publisher.
 [positional-arguments]
 ffi *args:
     python3 sh/gh/ffi.py "$@"
+
+# Prepare the shared history branch or publish its generated charts.
+[positional-arguments]
+benchmark-publish *args:
+    sh/gh/bench-publish.sh "$@"

.github/workflows/nightly.yml

@@ -67,6 +67,7 @@ jobs:
             recipe: bench-ffi
             artifact: bench-ffi
             artifact-path: target/bench/ffi
+            benchmark: true
           # Keeps the BoltFFI and UniFFI slice building and running for the
           # browser generator comparison, which reuses it. It skips the size
           # step, whose moq-ffi and moq-wasm builds double the time and are
@@ -91,8 +92,14 @@ jobs:
             recipe: gh nightly rust-test
           - name: Rust bench targets
             recipe: gh nightly rust-bench
+            artifact: bench-rust
+            artifact-path: target/bench/nightly/rust-bench
+            benchmark: true
           - name: JS benchmarks
             recipe: gh nightly js
+            artifact: bench-js
+            artifact-path: target/bench/nightly/js
+            benchmark: true
             uncached: true
           - name: BBB rendition alignment
             recipe: pub check-bbb
@@ -160,10 +167,15 @@ jobs:
         run: nix develop --command bash -c "bun install --frozen-lockfile && bunx playwright install --with-deps chromium"
 
       - name: Run workload
+        id: workload
         run: nix develop ${{ matrix.shell }} --command just ${{ matrix.recipe }}
         env:
           NEXTEST_PROFILE: ${{ matrix.profile || 'default' }}
 
+      - name: Record benchmark outcome
+        if: ${{ matrix.benchmark && !cancelled() }}
+        run: nix develop --command just gh benchmark-report status "${{ steps.workload.outcome }}" "${{ matrix.artifact-path }}"
+
       - name: Preserve results
         if: ${{ matrix.artifact && !cancelled() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
@@ -172,6 +184,78 @@ jobs:
           path: ${{ matrix.artifact-path }}
           retention-days: 45
 
+  # One writer owns gh-pages, including future Criterion regression history.
+  # Explicit Pages deployment works with GITHUB_TOKEN; a branch push alone does not.
+  benchmark-site:
+    name: Benchmark site
+    needs: nightly
+    if: ${{ !cancelled() && github.ref == 'refs/heads/main' }}
+    runs-on: ubuntu-24.04-arm
+    timeout-minutes: 15
+    concurrency:
+      group: benchmark-pages
+      cancel-in-progress: false
+    permissions:
+      contents: write
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deploy.outputs.page_url }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          # Only this main-branch publisher pushes the shared history branch.
+          persist-credentials: true
+      - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main
+        with:
+          determinate: false
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
+        with:
+          pattern: bench-*
+          path: benchmark-inputs
+      - name: Validate completed reports
+        id: reports
+        run: nix develop --command just gh benchmark-report prepare benchmark-inputs benchmark-reports
+      - name: Prepare history
+        run: nix develop --command just gh benchmark-publish init
+      - name: Store time, CPU and ratio history
+        if: steps.reports.outputs.costs == 'true'
+        uses: benchmark-action/github-action-benchmark@4322e5726e6334590d251fc4f92bec0efafc45dc # v1.22.2
+        with:
+          name: MoQ nightly costs
+          tool: customSmallerIsBetter
+          output-file-path: benchmark-reports/costs.json
+          benchmark-data-dir-path: bench/costs
+          skip-fetch-gh-pages: true
+          github-token: ${{ github.token }}
+          ref: ${{ github.sha }}
+          auto-push: false
+          comment-on-alert: false
+          fail-on-alert: false
+      - name: Store throughput history
+        if: steps.reports.outputs.rates == 'true'
+        uses: benchmark-action/github-action-benchmark@4322e5726e6334590d251fc4f92bec0efafc45dc # v1.22.2
+        with:
+          name: MoQ nightly throughput
+          tool: customBiggerIsBetter
+          output-file-path: benchmark-reports/rates.json
+          benchmark-data-dir-path: bench/rates
+          skip-fetch-gh-pages: true
+          github-token: ${{ github.token }}
+          ref: ${{ github.sha }}
+          auto-push: false
+          comment-on-alert: false
+          fail-on-alert: false
+      - name: Publish shared history
+        run: nix develop --command just gh benchmark-publish finish benchmark-site
+      - uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4
+        with:
+          path: benchmark-site
+      - name: Deploy charts
+        id: deploy
+        uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4
+
   tests:
     name: Tests (${{ matrix.recipe }})
     runs-on: ubuntu-24.04-arm

bench/README.md

@@ -340,5 +340,61 @@ Rust Criterion coverage includes `moq-pattern`'s
 and payload size; CMAF also varies track count. Separate frame and byte rates
 use the same fixtures, with byte rates counting the encoded stream size.
 Fixture generation, publisher setup and destruction stay outside timing.
-The existing shared Rust benchmark nightly invocation runs both targets; their `--test` mode
-exercises every case without making timing claims.
+The shared Rust benchmark nightly invocation measures both targets in quick mode.
+
+## Nightly trends
+
+The trend site will be at <https://moq-dev.github.io/moq/bench/> once the
+maintainer enables GitHub Pages with **Source: GitHub Actions**. The Nightly
+workflow keeps history on one `gh-pages` branch and deploys the generated
+`github-action-benchmark` charts explicitly. The publisher runs only on `main`,
+serializes history writes, and leaves regression alerts and comments disabled.
+Enabling Pages is a separate repository setting; merging the workflow does not
+change it. Until it is enabled, the Nightly's Benchmark site job still records
+history on `gh-pages`, then fails at deployment.
+
+The charts include these measurements:
+
+- Binding delivery in MB/s, process CPU in ms/MB, sync calls in ns, and each
+  wrapper's ratio to `rust-ffi`. Ratios above 1 mean more cost than Rust on the
+  same host, including for throughput, where the ratio is Rust/wrapper.
+- Rust Criterion mean ns per iteration, with the measured confidence interval.
+  The `net-cpu` source and the moq-quic targets time thread CPU; the others
+  time elapsed time. The Rust bench targets job measures them with a short warmup
+  and Criterion's quick mode. Per-group benchmark settings can override the default warmup. Quick estimates
+  trade precision for nightly runtime; use full paired runs to assess a change.
+- JS benchmark CSV time columns (`ns`, `us`, `ms`, and their per-operation
+  variants), retaining every workload dimension in each series name.
+
+Hosted runners vary across nights. Raw trends are informational, and binding
+ratios compare languages within one run. A failed suite contributes no point;
+malformed, empty, nonfinite, or duplicate measurements fail validation. The
+nightly artifacts retain the reports, raw JS output, and each Criterion run's
+metadata and estimates for 45 days. Relay load, audio quality, and the Rust
+benchmark tests job's nextest output do not enter these charts.
+
+Reproduce the chart inputs locally:
+
+```bash
+nix develop --command just bench-ffi
+nix develop --command just gh nightly rust-bench
+nix develop --command just gh nightly js
+```
+
+The latter two write `target/bench/nightly/{rust-bench,js}/report.json`. For a focused
+measured Rust run using the same evidence lifecycle:
+
+```bash
+nix develop --command env MOQ_BENCH_LABEL=pattern MOQ_BENCH_RUNS=/tmp/criterion-runs.txt just gh criterion -p moq-pattern --bench matching
+nix develop --command just gh benchmark-report rust /tmp /tmp/rust-report.json
+```
+
+A new source uploads a `bench-*` artifact containing `report.json`, a nonempty
+JSON array of `{ "name": "case/dimensions", "unit": "ns", "value": 123 }`,
+and `outcome.json` containing `{ "status": "success" }`. Names must be unique
+within a source; dimensions belong in the name. Supported units are `ns`, `us`,
+`µs`, `ms`, `s`, `ms/MB`, `x`, `bytes`, `B`, `MiB`, `MB/s`, `MiB/s`, `B/s`,
+`ops/s`, and `frames/s`. Optional `range` and `extra` fields retain uncertainty
+and context. Validate and split sources into cost and throughput charts with
+`just gh benchmark-report prepare INPUT_DIRECTORY OUTPUT_DIRECTORY`; extending
+the site requires no new chart renderer or data branch.

bench/metadata.py

@@ -27,15 +27,16 @@ def command(*args):
  "rustc": command("rustc", "-vV"), "cargo": command("cargo", "-V"),
  "os": platform.platform(), "machine": platform.machine(), "processor": platform.processor(),
  "affinity": sorted(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else None,
- "environment": {key: value for key, value in os.environ.items() if key.startswith(("MOQ_BENCH_", "CARGO_PROFILE_")) or key in ("RUSTFLAGS", "CARGO_BUILD_TARGET", "CARGO_BUILD_RUSTFLAGS", "TOKIO_WORKER_THREADS", "LD_PRELOAD", "MALLOC_CONF", "GLIBC_TUNABLES", "RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER")},
+ "environment": {key: value for key, value in os.environ.items() if key.startswith(("MOQ_BENCH_", "CARGO_PROFILE_")) or key in ("RUSTFLAGS", "CARGO_BUILD_TARGET", "CARGO_BUILD_RUSTFLAGS", "TOKIO_WORKER_THREADS", "LD_PRELOAD", "MALLOC_CONF", "GLIBC_TUNABLES", "RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER", "SESSION_CPU")},
  "profile": "profiling" if mode == "--profile" else "release",
- "features": {"--runtime": "default + moq-relay/io-uring,moq-bench/uring", "--profile": "default; heap adds moq-relay/jemalloc"}.get(mode, "default"),
+ "features": {"--runtime": "default + moq-relay/io-uring,moq-bench/uring", "--profile": "default; heap adds moq-relay/jemalloc",
+              "--criterion": "see criterion-command.json"}.get(mode, "default"),
  "mode": mode, "runtime_workers": sys.argv[4] if len(sys.argv) > 4 else None,
  "rounds": int(sys.argv[5]) if len(sys.argv) > 5 else None,
  "allocator": "system by build; preload and allocator tunables are recorded in environment"
  + ("; heap runs jemalloc, profiled per the MALLOC_CONF in harness/relay.sh" if mode == "--profile" else ""),
  "profilers": {tool: command(tool, "--version").splitlines()[0] for tool in ("perf", "jeprof") if shutil.which(tool)} if mode == "--profile" else None,
- "workload": {"startup": "2s", "duration": "10s", "report": "500ms"}}
+ "workload": None if mode == "--criterion" else {"startup": "2s", "duration": "10s", "report": "500ms"}}
 (out / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n")
 subprocess.run(["cargo", "tree", "--locked", "-p", "moq-relay", "-e", "features"], stdout=(out / "features.txt").open("w"), check=True)
 if Path("/proc/cpuinfo").exists():

bench/run.sh

@@ -47,6 +47,21 @@ usable_cpus() {
     env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc
 }
 case $MODE in
+    --criterion)
+        BASE=
+        CRITERION_CARGO_ARGS=()
+        CRITERION_FILTER_ARGS=()
+        parsing_filters=false
+        for argument in "${@:2}"; do
+            if [[ $argument == -- && $parsing_filters == false ]]; then
+                parsing_filters=true
+            elif [[ $parsing_filters == true ]]; then
+                CRITERION_FILTER_ARGS+=("$argument")
+            else
+                CRITERION_CARGO_ARGS+=("$argument")
+            fi
+        done
+        ;;
     --audio)
         BASE=
         ;;
@@ -710,7 +725,17 @@ run_profile() {
 
 cd "$ROOT"
 
-if [[ $MODE == --audio ]]; then
+if [[ $MODE == --criterion ]]; then
+    # Each invocation owns fresh results; previous runs cannot become new points.
+    python3 -c 'import json,sys; from pathlib import Path; Path(sys.argv[1]).write_text(json.dumps(sys.argv[2:]))' \
+        "$RUN/criterion-command.json" "${@:2}"
+    if [[ -n ${MOQ_BENCH_RUNS:-} ]]; then
+        printf '%s\n' "$RUN" >>"$MOQ_BENCH_RUNS"
+    fi
+    CARGO_TARGET_DIR=$CURRENT_TARGET cargo bench --locked ${CRITERION_CARGO_ARGS[@]+"${CRITERION_CARGO_ARGS[@]}"} -- \
+        --quick --warm-up-time 0.05 --measurement-time 0.2 --noplot ${CRITERION_FILTER_ARGS[@]+"${CRITERION_FILTER_ARGS[@]}"}
+    exit 0
+elif [[ $MODE == --audio ]]; then
     run_audio_comparison
     exit 0
 elif [[ $MODE == --runtime ]]; then

js/net/bench/frames.ts

@@ -13,6 +13,9 @@ import * as Path from "../src/path.ts";
 import { Reader, Stream } from "../src/stream.ts";
 import * as Varint from "../src/varint.ts";
 
+// Keep protocol diagnostics out of the measured CSV on stdout.
+console.debug = console.error;
+
 const frameSizes = [16, 100, 1000];
 const chunkSizes = [1200, 16 * 1024, 64 * 1024];
 const framesPerGroup = 50;

js/net/bench/requests.ts

@@ -16,6 +16,9 @@ import * as Path from "../src/path";
 import { Stream } from "../src/stream";
 import { wireOf } from "../src/wire";
 
+// Keep protocol diagnostics out of the measured CSV on stdout.
+console.debug = console.error;
+
 const settle = () => new Promise<void>((resolve) => setImmediate(resolve));
 console.log("touched,routes,followers,update_us");
 for (const touched of ["watched", "unrelated"])

js/publish/bench/catalog.ts

@@ -1,26 +1,13 @@
 /** Sweep tracks x viewers for 1000 estimate rises inside one second. */
 
-import { Milli } from "@moq/clock";
+import { Manual, Milli, Nano } from "@moq/clock";
 import * as Catalog from "@moq/hang/catalog";
 import * as Json from "@moq/json";
 import { Track } from "@moq/net";
 import { Effect } from "@moq/signals";
 import { CatalogProducer } from "../src/catalog";
 
-const realNow = performance.now.bind(performance);
-const realTimer = Effect.prototype.timer;
-let now = 0;
-let timers = new Set<{ at: number; run: () => void }>();
-performance.now = () => now;
-Effect.prototype.timer = function (fn, ms) {
-	const timer = { at: now + ms, run: fn };
-	timers.add(timer);
-	this.cleanup(() => timers.delete(timer));
-};
-
-async function measure(tracks: number, viewers: number) {
-	now = 0;
-	timers = new Set();
+async function measure(tracks: number, viewers: number, clock: Manual) {
 	const scope = new Effect();
 	const catalog = new CatalogProducer();
 	const media = Array.from({ length: tracks }, (_, index) => new Track.Producer(`v${index}`));
@@ -68,19 +55,18 @@ async function measure(tracks: number, viewers: number) {
 			}
 		}
 	};
-	const start = realNow();
+	const start = performance.now();
 	for (let step = 0; step < 1000; step++) {
-		now = step;
+		if (step > 0) clock.advance(Nano.fromMilli(Milli(1)));
 		catalog.mutate((root) => {
 			if (!root.video) throw new Error("video missing");
 			for (const config of Object.values(root.video.renditions)) config.jitter = Catalog.u53(step + 1);
 		});
 		await receive();
 	}
-	now = 1000;
-	for (const timer of [...timers]) if (timer.at <= now) timer.run();
+	await clock.advanceAsync(Nano.fromMilli(Milli(1)));
 	await receive();
-	const elapsed = realNow() - start;
+	const elapsed = performance.now() - start;
 	if (publishes !== 2 || updates !== 2 * tracks * viewers) {
 		throw new Error(`estimate churn regression: ${publishes} publishes, ${updates} subscribe updates`);
 	}
@@ -94,10 +80,15 @@ async function measure(tracks: number, viewers: number) {
 	for (const track of media) track.close();
 }
 
-try {
-	console.log("tracks,viewers,catalog_publishes,subscribe_updates,unlimited_subscribe_updates,elapsed_ms");
-	for (const tracks of [1, 4, 16]) for (const viewers of [1, 4, 16]) await measure(tracks, viewers);
-} finally {
-	performance.now = realNow;
-	Effect.prototype.timer = realTimer;
+console.log("tracks,viewers,catalog_publishes,subscribe_updates,unlimited_subscribe_updates,elapsed_ms");
+for (const tracks of [1, 4, 16]) {
+	for (const viewers of [1, 4, 16]) {
+		const clock = new Manual();
+		const restore = clock.install();
+		try {
+			await measure(tracks, viewers, clock);
+		} finally {
+			restore();
+		}
+	}
 }

quest/m2/bench-site.md

@@ -10,6 +10,12 @@ site.
 
 ## Plan
 
+The pipeline is in review. Remaining: the maintainer enables repository Pages
+with **Source: GitHub Actions** (until then the Nightly's Benchmark site job
+fails at deployment), then verifies the first Nightly deployment at
+<https://moq-dev.github.io/moq/bench/>. The "Rust benchmark tests" job is not
+charted yet: its nextest benchmarks print no structured measurements.
+
 Decided with the maintainer on 2026-10-15:
 
 - **GitHub Pages via github-action-benchmark**: the nightly pushes each