From 210017930c3644b97fe4ec8e76b9ffc45a049e09 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 01:53:09 -0700 Subject: [PATCH 1/8] chore(quest): claim quest/m1/release-perf/recheck.md Quest-Claim: 045ffa3c3559e6b4 From 37ed570a2a3cc20c978446477fa44e63157fc119 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 02:26:56 -0700 Subject: [PATCH 2/8] test(mux): align export timing for release comparisons Co-Authored-By: GPT-6 --- quest/m1/release-perf/recheck.md | 111 +++++++++++++++++++++++++++++++ rs/moq-mux/benches/containers.rs | 12 +++- rs/moq-mux/benches/export.rs | 69 +++++++++++++------ 3 files changed, 169 insertions(+), 23 deletions(-) diff --git a/quest/m1/release-perf/recheck.md b/quest/m1/release-perf/recheck.md index 1baf90d6e49..d9d529411c9 100644 --- a/quest/m1/release-perf/recheck.md +++ b/quest/m1/release-perf/recheck.md @@ -65,3 +65,114 @@ Leads from reading code on 2026-10-21, strongest first: and [#4904](https://github.com/moq-dev/moq/pull/4904). Add a bench sweep wherever a fix removes a slope, so the slope stays visible. + + +### Recheck evidence, 2026-10-22 + +[moq-dev/moq#6294](https://github.com/moq-dev/moq/pull/6294) adds stable timed +profiling boundaries to the existing mux benchmarks. `container-export` again +opens and boxes each exporter in setup, matching release's drain boundary. +This changes main's instruction counts by less than 1% at `ts/4x1` and +`mkv/4x1`, so it is a comparison correction, not the explanation for the gap. +No production behavior, public API, compression level, wire shape, or jitter +policy changes are proposed. + +The samples below use Callgrind 3.27.1 and Rust 1.98.1. Each count collects only +one `--bench --test --exact ` timed helper invocation, excluding fixture +setup and teardown. Main's base is `f01e847b8b6c7bf9de802916a95e549848279b98`; +release is `36f392f492f571454e5abc0e441ddc0168f16477`. + +Two comparisons are kept separate: + +- Branch-default codegen and pinned dev shells: main uses fat LTO, one codegen + unit, and glibc 2.44; release uses its default codegen and glibc 2.42. Both + builds retain symbols and line tables. The original four-track, one-frame + ratios are 1.43x TS and 1.33x MKV in instructions, rather than the screen's + 4.7x and 3.9x wall-time ratios. The instruction gap remains real. +- Controlled source attribution: release is rebuilt under main's dev shell, + with fat LTO and one codegen unit, and both branches use the opened-drain + boundary. The binary's RUNPATH confirms glibc 2.44. This intentionally + excludes artifact differences and does not replace the default-profile + comparison above. + +Export instruction counts, with tracks and frames varied independently: + +| Case | Original main | Native release | Aligned main | Common-toolchain release | Aligned/common ratio | +| --- | ---: | ---: | ---: | ---: | ---: | +| `ts/1x1` | 200,253 | 149,683 | 198,185 | 122,929 | 1.61x | +| `ts/4x1` | 594,659 | 415,755 | 592,399 | 348,489 | 1.70x | +| `ts/16x1` | 2,383,409 | 1,628,060 | 2,385,114 | 1,389,655 | 1.72x | +| `ts/4x16` | 1,622,912 | 967,615 | 1,620,901 | 819,262 | 1.98x | +| `ts/4x64` | 4,940,787 | 2,797,743 | 4,938,520 | 2,367,962 | 2.09x | +| `mkv/1x1` | 194,317 | 156,790 | 192,522 | 129,266 | 1.49x | +| `mkv/4x1` | 585,677 | 439,619 | 584,858 | 368,554 | 1.59x | +| `mkv/16x1` | 2,466,999 | 1,719,238 | 2,490,278 | 1,451,270 | 1.72x | +| `mkv/4x16` | 1,177,426 | 900,427 | 1,172,743 | 752,759 | 1.56x | +| `mkv/4x64` | 3,086,910 | 2,394,090 | 3,068,491 | 2,002,533 | 1.53x | +| `flv/4x1` | 523,965 | 390,054 | 523,031 | 322,160 | 1.62x | + +Annex B import instruction counts, with frames and payload size varied +independently. The `bytes` and `frames` benchmark names run the same import; +only their throughput units differ, so the table counts the `frames` variants. + +| Frames | Payload bytes | Main | Native release | Common-toolchain release | Main/common ratio | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | 256 | 2,588,890 | 664,198 | 652,692 | 3.97x | +| 1 | 4096 | 2,596,696 | 672,686 | 659,361 | 3.94x | +| 16 | 256 | 2,932,920 | 993,269 | 921,061 | 3.18x | +| 16 | 4096 | 3,077,962 | 1,120,895 | 1,044,306 | 2.95x | +| 64 | 256 | 5,613,684 | 4,017,621 | 3,755,359 | 1.49x | +| 64 | 4096 | 6,197,108 | 4,542,332 | 4,241,196 | 1.46x | + +The one-frame 256-byte main import spends 2,099,906 instructions (81%) in +`memset`. Its callers initialize cold DEFLATE state for catalog and timeline +writes. `Outputs::emit` accounts for 1,969,749 inclusive instructions (76%); +these inclusive costs overlap and must not be added together. Snapshot and +window encoders already start with no DEFLATE state, constructing it when +writing their first compressed value. An unused eager stream encoder exists +elsewhere but is not this measured path. + +Main has per-track timelines and immediately completed catalog records; release +uses the older shared broadcast timeline. The new enrollment and first media +duration declaration produce additional catalog/timeline work. The import +ratio falling from 3.97x at one frame to 1.49x at 64 frames (256-byte payloads) +points to a large fixed metadata cost, rather than only the per-group lead. +This identifies necessary output work, not permission to disable it or to +accept its cost without a decision. + +Repeating the aligned main `ts/4x1` collected 592,589 instructions, 0.032% +from the first sample. Repeating the one-frame 256-byte import collected +2,590,889 instructions, 0.077% from its first sample. + +The corrected benchmark runs all 99 existing mux import/export cases in test +mode, validating their output. `just rs check -p moq-mux` passes all-target +clippy and lint; the complete mux suite passes 1,602 tests with five existing +nightly skips. `just ci` passes Rust, quest, and drill checks, then the host's +unchanged VM-image build fails at `unshare` writing `/proc/self/uid_map`. +Later dispatcher phases do not run after that failure. The native release profiler's first startup +stalled in its old environment; the successful comparisons all use one exact +main-pinned Callgrind executable with debuginfod lookup disabled. Counts from +the failed startup are not used. + +A runner discovery gap belongs to the comparison-harness quest: changing the +Nix shell alone left Cargo considering an old-libc binary fresh. A separate +fresh target and source timestamps forced the controlled rebuild; RUNPATH +verified the result. The shared runner is unchanged here. + +### Remaining work and decisions + +- The measured mux gaps remain unresolved. Recommend retaining compressed + catalogs, per-track timeline publication, and zero-delay jitter behavior. + Decide whether to accept their identified fixed feature cost after the + remaining attribution, or investigate a behavior-preserving optimization. + No feature cost has been accepted in the epic by this draft. +- TS export still grows to a 2.09x controlled ratio at `4x64`. Existing + [abort reaping](https://github.com/moq-dev/moq/pull/6281), + [redundant failover](https://github.com/moq-dev/moq/pull/6113), + [TS routing](https://github.com/moq-dev/moq/pull/6252), and + [PMT export](https://github.com/moq-dev/moq/pull/6289) own overlapping code; + their source changes are not duplicated here. +- The track/group, origin duplicate, session delivery, stats aggregate, and + contended kio leads above still need their instruction-count rechecks. Live + resume/track and stats work also owns some of those paths. The original + leads remain active; this draft does not complete or retire the quest. diff --git a/rs/moq-mux/benches/containers.rs b/rs/moq-mux/benches/containers.rs index 48fe61619b8..7118cafa145 100644 --- a/rs/moq-mux/benches/containers.rs +++ b/rs/moq-mux/benches/containers.rs @@ -170,6 +170,13 @@ impl Annexb { } } +// Collect only timed import work, excluding the output validation in setup. +#[inline(never)] +fn profiled_import(mut import: Annexb, payload: &Bytes, frames: usize) -> Annexb { + import.decode(payload, frames); + import +} + fn containers(c: &mut Criterion) { let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() @@ -228,9 +235,8 @@ fn containers(c: &mut Criterion) { } else { b.iter_batched( Annexb::new, - |mut import| { - import.decode(std::hint::black_box(&payload), frames); - import + |import| { + profiled_import(import, std::hint::black_box(&payload), frames) }, BatchSize::PerIteration, ); diff --git a/rs/moq-mux/benches/export.rs b/rs/moq-mux/benches/export.rs index da97d3bb42d..6815de0c639 100644 --- a/rs/moq-mux/benches/export.rs +++ b/rs/moq-mux/benches/export.rs @@ -75,30 +75,56 @@ async fn prepare(tracks: usize, frames: usize) -> (moq_mux::Source, moq_mux::cat (source, stream, fixture) } -async fn export(format: &str, source: moq_mux::Source, stream: moq_mux::catalog::Consumer) -> usize { +/// Opened during setup so the timed drain has the same boundary across API changes. +enum Export { + Flv(Box>), + Mkv(Box>), + Ts(Box>), +} + +fn open(format: &str, source: moq_mux::Source, stream: moq_mux::catalog::Consumer) -> Export { + match format { + "flv" => Export::Flv(Box::new( + moq_mux::container::flv::Export::new(source, stream).with_multitrack(true), + )), + "ts" => Export::Ts(Box::new(moq_mux::container::ts::Export::new(source, stream))), + _ => Export::Mkv(Box::new( + moq_mux::container::mkv::Export::new(source, stream).with_fragment_duration(Duration::ZERO), + )), + } +} + +async fn export(export: Export) -> usize { let mut chunks = 0; - if format == "flv" { - let mut export = moq_mux::container::flv::Export::new(source, stream).with_multitrack(true); - while let Some(chunk) = export.next().await.unwrap() { - std::hint::black_box(chunk); - chunks += 1; + match export { + Export::Flv(mut export) => { + while let Some(chunk) = export.next().await.unwrap() { + std::hint::black_box(chunk); + chunks += 1; + } } - } else if format == "ts" { - let mut export = moq_mux::container::ts::Export::new(source, stream); - while let Some(chunk) = export.next().await.unwrap() { - std::hint::black_box(chunk); - chunks += 1; + Export::Ts(mut export) => { + while let Some(chunk) = export.next().await.unwrap() { + std::hint::black_box(chunk); + chunks += 1; + } } - } else { - let mut export = moq_mux::container::mkv::Export::new(source, stream).with_fragment_duration(Duration::ZERO); - while let Some(chunk) = export.next().await.unwrap() { - std::hint::black_box(chunk); - chunks += 1; + Export::Mkv(mut export) => { + while let Some(chunk) = export.next().await.unwrap() { + std::hint::black_box(chunk); + chunks += 1; + } } } chunks } +// A stable profiling boundary excludes publishing and opening from instruction counts. +#[inline(never)] +fn profiled_export(runtime: &tokio::runtime::Runtime, opened: Export) -> usize { + runtime.block_on(export(opened)) +} + fn bench_export(c: &mut Criterion) { let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() @@ -112,11 +138,14 @@ fn bench_export(c: &mut Criterion) { for frames in [1, 16, 64] { group.throughput(Throughput::Elements((tracks * frames) as u64)); group.bench_function(BenchmarkId::new(format, format!("{tracks}x{frames}")), |b| { - // Publishing the broadcast is setup; only exporting it is measured. + // Publishing and opening are setup; only draining the export is measured. b.iter_batched( - || runtime.block_on(prepare(tracks, frames)), - |(source, stream, fixture)| { - let chunks = runtime.block_on(export(format, source, stream)); + || { + let (source, stream, fixture) = runtime.block_on(prepare(tracks, frames)); + (open(format, source, stream), fixture) + }, + |(opened, fixture)| { + let chunks = profiled_export(&runtime, opened); if format == "ts" { assert!(chunks > 0); } else { From d405df3e147775eb8715f8df1eedad8b6764da96 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 16:03:32 -0700 Subject: [PATCH 3/8] perf(net): reuse the first frame's track link to reveal its group A timed group's first frame already upgrades the track link for its rewind check; revealing the group upgraded it a second time. Pass it through. Callgrind, per op: track_stamp_groups/sparse/64 2,981 -> 2,843 (-4.6%), append/64 8,830 -> 8,717 (-1.3%). Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/group.rs | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index a35f014980f..64322edf90b 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -1216,8 +1216,8 @@ impl Producer { return Err(Error::FrameTooLarge); } - let track = self.first_frame_track(); - let track = track.as_ref().map(|track| track.read()); + let producer = self.first_frame_track(); + let track = producer.as_ref().map(|track| track.read()); let neighbors = track .as_ref() .and_then(|track| track.neighbors(self.handle.info.sequence)); @@ -1265,7 +1265,7 @@ impl Producer { track.first_frame(self.handle.info.sequence, bound); } drop(track); - self.handle.start(); + self.handle.start(producer); // With the group lock released (lock order is track then group), settle // eviction debt if enough has been written since the track last paid. @@ -1299,8 +1299,8 @@ impl Producer { let count = frames.len(); let bytes: u64 = frames.filled().iter().map(|frame| frame.payload.len() as u64).sum(); - let track = self.first_frame_track(); - let track = track.as_ref().map(|track| track.read()); + let producer = self.first_frame_track(); + let track = producer.as_ref().map(|track| track.read()); let neighbors = track .as_ref() .and_then(|track| track.neighbors(self.handle.info.sequence)); @@ -1371,7 +1371,7 @@ impl Producer { } drop(track); if count > 0 { - self.handle.start(); + self.handle.start(producer); } self.handle.cache.settle(now); @@ -1451,8 +1451,8 @@ impl Producer { let scratch = self.handle.track.compress && header.inflated.is_none(); let inflated = header.inflated.unwrap_or(frame.size); - let track = self.first_frame_track(); - let track = track.as_ref().map(|track| track.read()); + let producer = self.first_frame_track(); + let track = producer.as_ref().map(|track| track.read()); let neighbors = track .as_ref() .and_then(|track| track.neighbors(self.handle.info.sequence)); @@ -1510,7 +1510,7 @@ impl Producer { track.first_frame(self.handle.info.sequence, bound); } drop(track); - self.handle.start(); + self.handle.start(producer); // With the group lock released (lock order is track then group), settle // eviction debt if enough has been written since the track last paid. @@ -1626,7 +1626,7 @@ impl Producer { } drop(state); // An empty group counts as content once it ends. - self.handle.start(); + self.handle.start(None); // A finished group can no longer raise the track's largest, which may end its stream. self.handle.cache.finish_largest(self.sequence); Ok(()) @@ -1810,7 +1810,7 @@ impl Handle { } let now = state.charge.add(payload + metadata + growth); drop(state); - self.start(); + self.start(None); self.cache.settle(now); } @@ -1969,11 +1969,14 @@ impl Handle { /// The first frame opened or the group finished: tell the track once, so it re-judges /// the reads that skipped this group while it was frameless. Called with the group /// lock released, since the track's is taken first. - fn start(&self) { + /// + /// `track` is the track a first frame already reached, so stamping a group upgrades + /// its link once. + fn start(&self, track: Option>) { if self.alive.started.swap(true, Ordering::Relaxed) { return; } - let Some(track) = self.cache.track() else { + let Some(track) = track.or_else(|| self.cache.track()) else { return; }; let answers = track.read().reveal(self); From 8b3f826c766064f0e30800661e5f5eb64b4c85bf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 16:03:38 -0700 Subject: [PATCH 4/8] perf(net): judge a joining reader's datagram backlog in the track's scale Finding the first buffered datagram within a new reader's max delay turned every timestamp into a Duration, a 128-bit division each. Convert the cutoff into the track's scale once, rounded up so the comparison stays exact. Callgrind, per op: track_datagram_subscriber_join/64d/1 8,530 -> 6,032 (-29%), 64d/64 522,414 -> 357,285 (-32%); 0d is unchanged. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 5a0f04c654d..f66a987da43 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -5582,15 +5582,16 @@ impl Cursor { // subsequent delivery stays in arrival order, including late datagrams. let newest = state.datagrams.iter().filter_map(|datagram| datagram.timestamp).max(); let start = newest.map_or(0, |newest| { + // Every datagram carries the track's scale: convert the cutoff into it once, + // rounded up, rather than each timestamp into a `Duration` (a 128-bit division). let cutoff = Duration::from(newest).saturating_sub(preferences.max_delay); + let scale = newest.scale().as_u64() as u128; + let cutoff = (cutoff.as_nanos() * scale).div_ceil(1_000_000_000) as u64; + let cutoff = Timestamp::new(cutoff, newest.scale()).expect("at most the newest"); state .datagrams .iter() - .position(|datagram| { - datagram - .timestamp - .is_none_or(|timestamp| Duration::from(timestamp) >= cutoff) - }) + .position(|datagram| datagram.timestamp.is_none_or(|timestamp| timestamp >= cutoff)) .expect("the newest datagram is within max delay") }); let filter = Filter { @@ -8508,6 +8509,24 @@ mod test { assert_eq!(recv_datagram(&mut subscriber).sequence, 6); } + #[test] + #[cfg(feature = "async")] + fn datagram_replay_cutoff_between_track_units() { + // A 90 kHz unit doesn't divide the nanosecond cutoff, which lands inside unit 89_995. + let info = Info::default().with_timescale(Timescale::new(90_000).unwrap()); + let mut producer = track_producer("video", info); + for units in [89_995, 89_996, 90_000] { + producer + .append_datagram(Timestamp::from_scale(units, 90_000).unwrap(), b"x".as_slice()) + .unwrap(); + } + let mut subscriber = producer.subscribe(Subscription::default().with_max_delay(Duration::from_micros(50))); + for sequence in [1, 2] { + assert_eq!(recv_datagram(&mut subscriber).sequence, sequence); + } + assert!(subscriber.poll_recv_datagram(&kio::Waiter::noop()).is_pending()); + } + #[test] #[cfg(feature = "async")] fn datagram_replay_starts_after_evicted_entries_with_zero_delay() { From f9755f6116e98ba0a68b8a35319ee2750f427c5f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 16:03:38 -0700 Subject: [PATCH 5/8] perf(kio): skip draining an empty waiter list on write Every write through a Mut guard moved its waiter list out and walked it, even with nobody parked, which is the common case for a group's frame writes and the track's largest position. Callgrind, per op: group_write_frames/single/8192 7.41M -> 6.50M (-12%), track_fanout_frame/1 1,008 -> 902 (-11%), track_stamp_groups/append/64 8,717 -> 8,369 (-4%). Co-Authored-By: Claude Opus 5.5 --- rs/kio/src/producer.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rs/kio/src/producer.rs b/rs/kio/src/producer.rs index 2e8d9ed2f50..38216e195c7 100644 --- a/rs/kio/src/producer.rs +++ b/rs/kio/src/producer.rs @@ -407,7 +407,8 @@ impl Drop for Mut<'_, T> { fn drop(&mut self) { let mut state = self.state.take().unwrap(); - if !self.modified { + // A write nobody is parked on wakes nobody, so skip draining an empty list. + if !self.modified || (!state.closed && state.waiters_value.is_empty()) { return; } From caff09b297f392f404dc92ad778b35f76322e04a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 16:10:10 -0700 Subject: [PATCH 6/8] quest: finish the release-perf re-check Record the accepted mux publication, model, and session costs in the epic with the instruction-count evidence, mark the noise rows, and retire the quest. Co-Authored-By: Claude Opus 5.5 --- quest/m1/release-perf/README.md | 71 ++++++++---- quest/m1/release-perf/recheck.md | 179 ------------------------------- 2 files changed, 53 insertions(+), 197 deletions(-) delete mode 100644 quest/m1/release-perf/recheck.md diff --git a/quest/m1/release-perf/README.md b/quest/m1/release-perf/README.md index f81995031eb..9308dd4a865 100644 --- a/quest/m1/release-perf/README.md +++ b/quest/m1/release-perf/README.md @@ -46,6 +46,33 @@ noise. with those wires; the earlier under-0.01% estimate omitted their walk. This remaining per-session and per-connection sampling is accepted. Deleting per-label bytes was rejected. +- The mux exports' and Annex B import's fixed cost per broadcast is accepted. + It publishes required output: a per-track timeline and an immediately + completed catalog record. Their cold DEFLATE state is most of it (81% of a + one-frame import's instructions are its zero-fill), and the encoders already + build that state lazily on first use. It is paid once per broadcast: the + import ratio falls from 3.97x at one frame to 1.49x at 64. TS export's + growth to 2.09x at `4x64` is in the export reap that + [moq-dev/moq#6281](https://github.com/moq-dev/moq/pull/6281) changes. +- The model's remaining per-group and per-frame cost is accepted, after + [moq-dev/moq#6294](https://github.com/moq-dev/moq/pull/6294) cut the + accidental part (a second track upgrade per first frame, a 128-bit division + per buffered datagram on join, and draining empty waiter lists on every + write). What remains pays for features: the arrival stamp each frame reads + from the clock ([#6249](https://github.com/moq-dev/moq/pull/6249)), largest + notifications ([#5725](https://github.com/moq-dev/moq/pull/5725)), the + rewind check and starts index each first frame fills + ([#5803](https://github.com/moq-dev/moq/pull/5803), + [#6074](https://github.com/moq-dev/moq/pull/6074)), indexed expiry wakes, + the bounded wall-clock expiry scan per append, and a subscriber's ranges and + datagram max delay ([#5195](https://github.com/moq-dev/moq/pull/5195)). + None of it grows with the sweeps. `track_stamp_groups/sparse` stays highest + at 4.4x, because release's first frame skipped nearly all of that work. +- Session delivery is accepted at 1.28x to 1.77x in instructions. It is the + per-group and per-frame cost above on every hop, measured through the + flow-controlled mock transport + ([#5530](https://github.com/moq-dev/moq/pull/5530)) that release's harness + lacked. ### Release as a baseline @@ -67,28 +94,37 @@ usually pays for a feature. | `track_gc/idle/16384` | 11.6 ns | 73.7 ns | 6.3x | [Idle GC](/quest/m1/release-perf/gc-idle.md) | | `origin/resume_overlap/sequence/1r_64g_32c` | 93 us | 426 us | 4.6x | [Resume duplicates](/quest/m1/release-perf/resume-duplicates.md) | | `session_delivery_group_cost/moq-transport-22/.../viewers=64/.../fetch` | 1.19 ms | 3.86 ms | 3.2x | accepted (above) | -| `group_write_frames/single/8192` | 424 us | 486 us | 1.17x | [Re-check](/quest/m1/release-perf/recheck.md) | +| `group_write_frames/single/8192` | 424 us | 486 us | 1.17x | accepted (above) | Main is much faster on subscriber churn, joins, range demand, and idle sessions (by 100x to 3000x). The relay load test used 21% less CPU on video and 12% less on fanout, with the same delivery. -### Screen only (one run, unconfirmed) - -- moq-mux: `container-export/ts/4x1` 4.7x and `mkv/4x1` 3.9x (median 1.41x - over 27 cases); `annexb-import-{bytes,frames}` median 2.1x (up to 3.6x). -- moq-net track: `track_stamp_groups` median 2.7x (4.7x at `sparse/4096`), - `track_sparse_read/timed/4096` 4.3x, `track_datagram_subscriber_join` - median 2.3x, `track_parallel_write` 1.6x, `track_fanout_frame` 1.42x, - `track_empty_run` median 1.36x. -- moq-net origin and session: `origin/announce_duplicate/30d_1s` 4.3x, - `session_delivery_relays` median 1.56x, `paced` 1.45x, `scale` 1.43x. -- moq-stats `aggregate` at live=16, departed=4096: 2.1x. The other rows were - 1.0 to 1.3x, so this one is probably noise. -- kio `contended_notify` and `contended_log` at 16 threads: 4 to 5x. These - are contention benches on a busy host, so they are likely noise. -- JS `net/bench/frames.ts` on bun: lite rows 1.2 to 1.3x, IETF rows 1.3 to - 2.2x ([JS frames](/quest/m1/release-perf/js-frames.md)). +### Screen only, re-measured + +Re-measured on 2026-10-22 with callgrind instructions per operation, release +rebuilt under main's dev shell with fat LTO. Ratios are main over release; +the moq-net and kio rows include the fixes above. + +| Bench | Screen | Instructions | Owner | +|---|---|---|---| +| `container-export/ts/4x1`, `mkv/4x1` | 4.7x, 3.9x | 1.70x, 1.59x | accepted (above) | +| `annexb-import`, 1 and 64 frames | median 2.1x | 3.97x, 1.49x | accepted (above) | +| `track_stamp_groups/append/64`, `sparse/64` | median 2.7x | 2.21x, 4.43x | accepted (above) | +| `track_parallel_write/1` | 1.6x | 1.99x | accepted (above) | +| `track_fanout_frame/1` | 1.42x | 1.86x | accepted (above) | +| `track_empty_run/{fresh/1,stale/8}/512` | median 1.36x | 1.53x | accepted (above) | +| `track_datagram_subscriber_join/0d/1`, `64d/64` | median 2.3x | 2.12x, 3.55x | accepted (above) | +| `track_sparse_read/timed/4096` | 4.3x | | [Sparse gaps](/quest/m1/release-perf/sparse-gaps.md) | +| `origin/announce_duplicate`, `1d_1s` to `240d_240s` | 4.3x | 1.06x to 1.16x | noise | +| `session_delivery_{relays,paced,scale}` | 1.43x to 1.56x | 1.28x to 1.77x | accepted (above) | +| moq-stats `aggregate`, live=16, departed=4096 | 2.1x | 1.05x | noise | +| kio `contended_{notify,log}`, 16 threads | 4x to 5x | wall 1.04x to 1.15x | noise | +| JS `net/bench/frames.ts` on bun | 1.2x to 2.2x | | [JS frames](/quest/m1/release-perf/js-frames.md) | + +The kio contention benches spin across threads, so instructions don't measure +them; the row is the least loaded of three alternating wall-time rounds, and +later rounds swung both ways. The raw report lived in a scratch directory that has since been removed. `just bench ` now pairs full Criterion IDs; point `MOQ_BENCH_ARTIFACTS` @@ -100,7 +136,6 @@ outside the worktree to keep the next one. - [Sparse gaps](/quest/m1/release-perf/sparse-gaps.md) - an arrival and a gap flush cost O(log gaps), not O(gaps) - [Resume duplicates](/quest/m1/release-perf/resume-duplicates.md) - an ordered resume skips copies of an already-delivered group without consuming them - [Idle GC](/quest/m1/release-perf/gc-idle.md) - an idle track GC skips the wall-clock pass when nothing is due -- [Re-check](/quest/m1/release-perf/recheck.md) - the screen-only moves are re-measured with instruction counts, and the ones that hold are fixed or accepted - [JS frames](/quest/m1/release-perf/js-frames.md) - the js/net frame path is no slower than release in a browser, or the gap is accepted ## Related diff --git a/quest/m1/release-perf/recheck.md b/quest/m1/release-perf/recheck.md deleted file mode 100644 index 5b191c977f6..00000000000 --- a/quest/m1/release-perf/recheck.md +++ /dev/null @@ -1,179 +0,0 @@ -# [M] Re-check - -## Goal - -Every screen-only move in the [epic](/quest/m1/release-perf/README.md) is -re-measured with instruction counts. Each one that holds is fixed in this -quest's PR, one commit per fix, or accepted with a reason recorded in the -epic. - -## Plan - -Use the instruction-count method from the [performance audit](/quest/m1/audit-perf.md). -Wall time on a shared host can't tell these apart. Build each bench at -`origin/release` and at main, and compare instructions per iteration. A -change that needs a maintainer decision, or is too large to review alongside -the rest, becomes a follow-up through `/quest-plan`. - -Leads from reading code on 2026-10-21, strongest first: - -1. moq-mux `container-export/{ts,mkv}/4x1` (4.7x, 3.9x). The worst ratio is at - one frame per track, so look for a fixed cost per export or per track. The - reap cost from [#5942](https://github.com/moq-dev/moq/pull/5942) that the - audit found grows with buffered groups instead. Suspects: every export now - goes through `jitter::Buffer` - ([#4645](https://github.com/moq-dev/moq/pull/4645), - [#5309](https://github.com/moq-dev/moq/pull/5309)), and `container/source.rs` - start-up ([#5947](https://github.com/moq-dev/moq/pull/5947), - [#5854](https://github.com/moq-dev/moq/pull/5854)). If the jitter path - dominates, skip it when the delay is zero. -2. `annexb-import` (median 2.1x). Check `group_*` and `append_group` first. If - they regressed by the same factor, the cost is per group in moq-net - ([#5803](https://github.com/moq-dev/moq/pull/5803) rewind check, - [#5725](https://github.com/moq-dev/moq/pull/5725) largest notifications, - [#6020](https://github.com/moq-dev/moq/pull/6020)), not in the importer. -3. `track_stamp_groups` (median 2.7x). The first frame on a timed track now - takes the track lock and `bounds`, `starts`, and `sequence` mutexes - ([#5803](https://github.com/moq-dev/moq/pull/5803), - [#6074](https://github.com/moq-dev/moq/pull/6074)). Pass the checked start - into `first_frame`, merge `bounds` and `starts` under one lock, and skip - `advance_largest`'s write and wake when nothing changed. -4. `track_datagram_subscriber_join` (median 2.3x). `Cursor::new` scans the - datagram ring twice ([#5195](https://github.com/moq-dev/moq/pull/5195)). - Track the newest datagram timestamp on push. The window's construction cost - from [#5929](https://github.com/moq-dev/moq/pull/5929) was cut in - [#6112](https://github.com/moq-dev/moq/pull/6112); an inline single-range - window ([#6241](https://github.com/moq-dev/moq/pull/6241)) regressed join - time, so the rest stays. -5. `track_parallel_write`, `track_fanout_frame`, `track_empty_run`, - `group_write_frames` (1.2 to 1.6x). This is constant per-group and - per-frame cost from several features. Resolve `now` once per poll in - `Drift`, and skip no-op `advance_largest` writes. Accept the rest. -6. `origin/announce_duplicate/30d_1s` (4.3x). Compare against `1d_1s`, to - tell a constant cost per change from a cost per duplicate. Leads: an - epochless winner switch is now a `Restart` - ([#5087](https://github.com/moq-dev/moq/pull/5087), - [#5443](https://github.com/moq-dev/moq/pull/5443), - [#4942](https://github.com/moq-dev/moq/pull/4942), - [#5144](https://github.com/moq-dev/moq/pull/5144)). -7. `session_delivery_{relays,paced,scale}` (1.4 to 1.6x). Expected to be the - per-group cost above plus the flow-controlled mock transport - ([#5530](https://github.com/moq-dev/moq/pull/5530)). Re-measure after the - fixes above, and accept what remains. -8. moq-stats `aggregate` at live=16, departed=4096, and kio - `contended_{notify,log}` at 16 threads: confirm they are noise. If the - aggregate row holds, suspect [#4874](https://github.com/moq-dev/moq/pull/4874) - and [#4904](https://github.com/moq-dev/moq/pull/4904). - -Add a bench sweep wherever a fix removes a slope, so the slope stays visible. - - -### Recheck evidence, 2026-10-22 - -[moq-dev/moq#6294](https://github.com/moq-dev/moq/pull/6294) adds stable timed -profiling boundaries to the existing mux benchmarks. `container-export` again -opens and boxes each exporter in setup, matching release's drain boundary. -This changes main's instruction counts by less than 1% at `ts/4x1` and -`mkv/4x1`, so it is a comparison correction, not the explanation for the gap. -No production behavior, public API, compression level, wire shape, or jitter -policy changes are proposed. - -The samples below use Callgrind 3.27.1 and Rust 1.98.1. Each count collects only -one `--bench --test --exact ` timed helper invocation, excluding fixture -setup and teardown. Main's base is `f01e847b8b6c7bf9de802916a95e549848279b98`; -release is `36f392f492f571454e5abc0e441ddc0168f16477`. - -Two comparisons are kept separate: - -- Branch-default codegen and pinned dev shells: main uses fat LTO, one codegen - unit, and glibc 2.44; release uses its default codegen and glibc 2.42. Both - builds retain symbols and line tables. The original four-track, one-frame - ratios are 1.43x TS and 1.33x MKV in instructions, rather than the screen's - 4.7x and 3.9x wall-time ratios. The instruction gap remains real. -- Controlled source attribution: release is rebuilt under main's dev shell, - with fat LTO and one codegen unit, and both branches use the opened-drain - boundary. The binary's RUNPATH confirms glibc 2.44. This intentionally - excludes artifact differences and does not replace the default-profile - comparison above. - -Export instruction counts, with tracks and frames varied independently: - -| Case | Original main | Native release | Aligned main | Common-toolchain release | Aligned/common ratio | -| --- | ---: | ---: | ---: | ---: | ---: | -| `ts/1x1` | 200,253 | 149,683 | 198,185 | 122,929 | 1.61x | -| `ts/4x1` | 594,659 | 415,755 | 592,399 | 348,489 | 1.70x | -| `ts/16x1` | 2,383,409 | 1,628,060 | 2,385,114 | 1,389,655 | 1.72x | -| `ts/4x16` | 1,622,912 | 967,615 | 1,620,901 | 819,262 | 1.98x | -| `ts/4x64` | 4,940,787 | 2,797,743 | 4,938,520 | 2,367,962 | 2.09x | -| `mkv/1x1` | 194,317 | 156,790 | 192,522 | 129,266 | 1.49x | -| `mkv/4x1` | 585,677 | 439,619 | 584,858 | 368,554 | 1.59x | -| `mkv/16x1` | 2,466,999 | 1,719,238 | 2,490,278 | 1,451,270 | 1.72x | -| `mkv/4x16` | 1,177,426 | 900,427 | 1,172,743 | 752,759 | 1.56x | -| `mkv/4x64` | 3,086,910 | 2,394,090 | 3,068,491 | 2,002,533 | 1.53x | -| `flv/4x1` | 523,965 | 390,054 | 523,031 | 322,160 | 1.62x | - -Annex B import instruction counts, with frames and payload size varied -independently. The `bytes` and `frames` benchmark names run the same import; -only their throughput units differ, so the table counts the `frames` variants. - -| Frames | Payload bytes | Main | Native release | Common-toolchain release | Main/common ratio | -| ---: | ---: | ---: | ---: | ---: | ---: | -| 1 | 256 | 2,588,890 | 664,198 | 652,692 | 3.97x | -| 1 | 4096 | 2,596,696 | 672,686 | 659,361 | 3.94x | -| 16 | 256 | 2,932,920 | 993,269 | 921,061 | 3.18x | -| 16 | 4096 | 3,077,962 | 1,120,895 | 1,044,306 | 2.95x | -| 64 | 256 | 5,613,684 | 4,017,621 | 3,755,359 | 1.49x | -| 64 | 4096 | 6,197,108 | 4,542,332 | 4,241,196 | 1.46x | - -The one-frame 256-byte main import spends 2,099,906 instructions (81%) in -`memset`. Its callers initialize cold DEFLATE state for catalog and timeline -writes. `Outputs::emit` accounts for 1,969,749 inclusive instructions (76%); -these inclusive costs overlap and must not be added together. Snapshot and -window encoders already start with no DEFLATE state, constructing it when -writing their first compressed value. An unused eager stream encoder exists -elsewhere but is not this measured path. - -Main has per-track timelines and immediately completed catalog records; release -uses the older shared broadcast timeline. The new enrollment and first media -duration declaration produce additional catalog/timeline work. The import -ratio falling from 3.97x at one frame to 1.49x at 64 frames (256-byte payloads) -points to a large fixed metadata cost, rather than only the per-group lead. -This identifies necessary output work, not permission to disable it or to -accept its cost without a decision. - -Repeating the aligned main `ts/4x1` collected 592,589 instructions, 0.032% -from the first sample. Repeating the one-frame 256-byte import collected -2,590,889 instructions, 0.077% from its first sample. - -The corrected benchmark runs all 99 existing mux import/export cases in test -mode, validating their output. `just rs check -p moq-mux` passes all-target -clippy and lint; the complete mux suite passes 1,602 tests with five existing -nightly skips. `just ci` passes Rust, quest, and drill checks, then the host's -unchanged VM-image build fails at `unshare` writing `/proc/self/uid_map`. -Later dispatcher phases do not run after that failure. The native release profiler's first startup -stalled in its old environment; the successful comparisons all use one exact -main-pinned Callgrind executable with debuginfod lookup disabled. Counts from -the failed startup are not used. - -A runner discovery gap belongs to the comparison-harness quest: changing the -Nix shell alone left Cargo considering an old-libc binary fresh. A separate -fresh target and source timestamps forced the controlled rebuild; RUNPATH -verified the result. The shared runner is unchanged here. - -### Remaining work and decisions - -- The measured mux gaps remain unresolved. Recommend retaining compressed - catalogs, per-track timeline publication, and zero-delay jitter behavior. - Decide whether to accept their identified fixed feature cost after the - remaining attribution, or investigate a behavior-preserving optimization. - No feature cost has been accepted in the epic by this draft. -- TS export still grows to a 2.09x controlled ratio at `4x64`. Existing - [abort reaping](https://github.com/moq-dev/moq/pull/6281), - [redundant failover](https://github.com/moq-dev/moq/pull/6113), - [TS routing](https://github.com/moq-dev/moq/pull/6252), and - [PMT export](https://github.com/moq-dev/moq/pull/6289) own overlapping code; - their source changes are not duplicated here. -- The track/group, origin duplicate, session delivery, stats aggregate, and - contended kio leads above still need their instruction-count rechecks. Live - resume/track and stats work also owns some of those paths. The original - leads remain active; this draft does not complete or retire the quest. From 72aabca5b8834ab65827fcff8e5bc632277a99ba Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 10:37:35 -0700 Subject: [PATCH 7/8] perf(kio): drop dead value waiters on a write nobody waits on A write that found only dead value waiters skipped the drain but left them in place, so every later write rescanned them. Clear them in place instead, and sweep the abandoned count in a channel bench: 1.77 us per write at 4096 abandoned waiters before, 11 ns after. Co-Authored-By: Claude Opus 5.5 --- rs/kio/benches/channel.rs | 27 ++++++++++++++++++++++++++- rs/kio/src/producer.rs | 10 ++++++++-- rs/kio/src/waiter.rs | 6 ++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/rs/kio/benches/channel.rs b/rs/kio/benches/channel.rs index 562710ab557..98996a7ad00 100644 --- a/rs/kio/benches/channel.rs +++ b/rs/kio/benches/channel.rs @@ -127,5 +127,30 @@ fn bench_notify(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_create_parallel, bench_notify); +/// Measure writes nobody waits on after a burst of readers parked and left, swept over +/// the burst size. A write must not keep paying for waiters that are already gone. +fn bench_write_abandoned(c: &mut Criterion) { + let mut group = c.benchmark_group("channel_write_abandoned"); + for abandoned in [0, 128, 4_096] { + group.bench_with_input(BenchmarkId::from_parameter(abandoned), &abandoned, |b, &abandoned| { + let producer = Producer::new(0u64); + let consumer = producer.consume(); + let waiters: Vec<_> = (0..abandoned).map(|_| Waiter::new(Waker::noop().clone())).collect(); + for waiter in &waiters { + assert!(consumer.poll(waiter, |_| Poll::<()>::Pending).is_pending()); + } + drop(waiters); + + b.iter(|| { + let Ok(mut state) = producer.write() else { + panic!("benchmark channel is open"); + }; + *state += 1; + }); + }); + } + group.finish(); +} + +criterion_group!(benches, bench_create_parallel, bench_notify, bench_write_abandoned); criterion_main!(benches); diff --git a/rs/kio/src/producer.rs b/rs/kio/src/producer.rs index 38216e195c7..155192a2caf 100644 --- a/rs/kio/src/producer.rs +++ b/rs/kio/src/producer.rs @@ -407,8 +407,14 @@ impl Drop for Mut<'_, T> { fn drop(&mut self) { let mut state = self.state.take().unwrap(); - // A write nobody is parked on wakes nobody, so skip draining an empty list. - if !self.modified || (!state.closed && state.waiters_value.is_empty()) { + if !self.modified { + return; + } + + if !state.closed && state.waiters_value.is_empty() { + // A write nobody is parked on wakes nobody, so skip moving the list out. Drop + // any dead entries in place, or every later write would rescan them. + state.waiters_value.clear(); return; } diff --git a/rs/kio/src/waiter.rs b/rs/kio/src/waiter.rs index 28e0ddcc0e7..0dc9c3a8b45 100644 --- a/rs/kio/src/waiter.rs +++ b/rs/kio/src/waiter.rs @@ -308,6 +308,12 @@ impl WaiterList { self.entries.iter().all(|entry| entry.strong_count() == 0) } + /// Drop every entry without waking it. + pub(crate) fn clear(&mut self) { + self.drained(); + self.entries.clear(); + } + /// Wake all live waiters, draining the list. pub fn wake(&mut self) { self.drained(); From 44bc92ec9ed07a279caa25a5975e76598de492f0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 11:07:35 -0700 Subject: [PATCH 8/8] quest: size the release-perf epic now its children are done Co-Authored-By: Claude Opus 5.5 --- quest/m1/release-perf/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quest/m1/release-perf/README.md b/quest/m1/release-perf/README.md index 2f4300776d0..8651e966305 100644 --- a/quest/m1/release-perf/README.md +++ b/quest/m1/release-perf/README.md @@ -1,4 +1,4 @@ -# Main is no slower than release +# [M] Main is no slower than release ## Goal