From a3c30792b9cdc177ef0e3b62967cbef25e25706b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 16:45:33 -0700 Subject: [PATCH 01/40] quest(m1): start T-STD compliant TS export questline Co-Authored-By: Claude Opus 5.5 From 02126bec61786bf24d6bfb9390b2ae2d07395c6f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 16:46:35 -0700 Subject: [PATCH 02/40] quest(m1/tstd): claim fixed-delay release Co-Authored-By: Claude Opus 5.5 From 8edda5591d023ec44679dec663c195cf750a8efc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 17:31:35 -0700 Subject: [PATCH 03/40] feat(moq-mux)!: release TS export frames on a fixed delay ts::Export::with_delay replaces with_max_age: every frame is muxed a fixed delay after its decode time, anchored at the first frame's arrival, in (DTS, PID) order across tracks. A frame that arrives past its deadline is dropped and counted. The stall/hold interleave is deleted. moq export ts takes --delay in place of --max-age. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 13 +- doc/bin/srt.md | 5 +- quest/m1/flv-export-delay.md | 10 +- quest/m1/mkv-export-delay.md | 10 +- quest/m1/plan-max-delay.md | 5 +- quest/m1/tstd/README.md | 1 - quest/m1/tstd/byte-schedule.md | 6 +- quest/m1/tstd/check.md | 2 +- quest/m1/tstd/delay.md | 74 ---- rs/moq-cli/src/args.rs | 107 ++++-- rs/moq-cli/src/publish.rs | 8 +- rs/moq-cli/src/subscribe.rs | 9 +- rs/moq-mux/src/container/ts/export.rs | 377 +++++++++--------- rs/moq-mux/src/container/ts/export_test.rs | 428 ++++++++------------- rs/moq-mux/src/container/ts/import.rs | 12 +- rs/moq-mux/src/container/ts/import_test.rs | 7 +- rs/moq-mux/src/lib.rs | 1 + rs/moq-mux/src/pace.rs | 2 +- rs/moq-mux/src/release.rs | 417 ++++++++++++++++++++ rs/moq-srt/src/ts.rs | 16 +- test/ts/README.md | 2 +- test/ts/pcr-timing.py | 2 +- 22 files changed, 886 insertions(+), 628 deletions(-) delete mode 100644 quest/m1/tstd/delay.md create mode 100644 rs/moq-mux/src/release.rs diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 2e3f831dd85..af5e0b114a4 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -323,11 +323,14 @@ See [Authentication](/bin/relay/auth). groups fetchable, which the [HLS gateway](/bin/hls) depends on. `export --max-age` (default 500 ms) is how long *this* consumer waits for a stalled group before skipping. Raising the first never delays playback. -For `export ts`, `--max-age` also bounds how long the muxer holds a leading -track for a lagging one. Frames go out in media-time order across all tracks, -not arrival order, so two exporters of one broadcast emit them in one order. A -track quiet for longer is muxed around until it catches up; a sparse track -(SCTE-35) costs that wait once per cue. `--max-age 0` keeps arrival order. +`export ts` takes `--delay` (default 500 ms) instead, and works like an SRT +receiver's latency. Every frame is written that long after its decode time, +with the clock started when the first frame arrived, so the output keeps the +source's pace and interleaves all tracks in decode order whatever their arrival +skew: two exporters of one broadcast emit them in one order. A frame that +arrives later than its deadline is dropped, and video then resumes at its next +keyframe. The delay is also how stale a group may get before it is skipped. +`--delay 0` holds nothing and drops nothing, writing frames in arrival order. A stdout export ends with the broadcast. `export ts --linger 10s` waits that long for the broadcast to come back instead: a publisher that restarts within diff --git a/doc/bin/srt.md b/doc/bin/srt.md index a2abd327ab5..ae305c7bf31 100644 --- a/doc/bin/srt.md +++ b/doc/bin/srt.md @@ -34,8 +34,9 @@ publishes each program as its own broadcast (`event.hang` becomes moq --connect https://relay.example.com/anon --broadcast event.hang import srt --listen '[::]:9000' --program all ``` -`--latency` sets the SRT receive buffer and doubles as the skip threshold on -export. Export paces each SRT payload on the media clock, and re-anchors that +`--latency` sets the SRT receive buffer and doubles as the export's release +delay, as `export ts --delay` in the [CLI](/bin/cli): each frame is muxed that +long after its decode time, and one arriving later is dropped. Export paces each SRT payload on the media clock, and re-anchors that pacing on a declared marker, so a restarted timeline plays out from the live edge instead of stalling until it catches up. A `--connect` URL needs a `streamid` query or a path; a listener bridges one `--broadcast` and ignores the stream id it is offered. The diff --git a/quest/m1/flv-export-delay.md b/quest/m1/flv-export-delay.md index 4496cb2e952..fbed3af14f5 100644 --- a/quest/m1/flv-export-delay.md +++ b/quest/m1/flv-export-delay.md @@ -10,13 +10,9 @@ of arrival, as the TS export's is. Today `pick_next_track` (`rs/moq-mux/src/container/flv/export.rs`) takes the smallest *pending* timestamp, so which track goes first depends on which frame -has arrived. Replace it with the release stage from -[fixed-delay release](/quest/m1/tstd/delay.md), with the same `--delay` flag -and the same late-frame drop. Update `doc/bin/cli.md`. +has arrived. Replace it with the release stage the TS export uses +(`rs/moq-mux/src/release.rs`), with the same `--delay` flag and the same +late-frame drop. Update `doc/bin/cli.md`. Public API: the exporter gains a delay setting; breaking only if it replaces an existing one. Wire: none. - -## Required - -- [Fixed-delay release](/quest/m1/tstd/delay.md) - builds the shared release stage diff --git a/quest/m1/mkv-export-delay.md b/quest/m1/mkv-export-delay.md index 4fb5d5fcf67..6024d9d4245 100644 --- a/quest/m1/mkv-export-delay.md +++ b/quest/m1/mkv-export-delay.md @@ -10,13 +10,9 @@ of arrival, as the TS export's is. Today `pick_next_track` (`rs/moq-mux/src/container/mkv/export.rs`) takes the smallest *pending* timestamp, so which track goes first depends on which frame -has arrived. Replace it with the release stage from -[fixed-delay release](/quest/m1/tstd/delay.md), with the same `--delay` flag -and the same late-frame drop. Update `doc/bin/cli.md`. +has arrived. Replace it with the release stage the TS export uses +(`rs/moq-mux/src/release.rs`), with the same `--delay` flag and the same +late-frame drop. Update `doc/bin/cli.md`. Public API: the exporter gains a delay setting; breaking only if it replaces an existing one. Wire: none. - -## Required - -- [Fixed-delay release](/quest/m1/tstd/delay.md) - builds the shared release stage diff --git a/quest/m1/plan-max-delay.md b/quest/m1/plan-max-delay.md index a8c700c497f..2c37b866aa0 100644 --- a/quest/m1/plan-max-delay.md +++ b/quest/m1/plan-max-delay.md @@ -11,9 +11,8 @@ delete it. ## Plan The maintainer finds "delay" clearer than "age" for how far a group may fall -behind before it's skipped. `moq play --delay` and `moq export ts --delay` -(planned in [fixed-delay release](/quest/m1/tstd/delay.md)) use one delay -knob for presentation and staleness; only `moq play`'s exists today. +behind before it's skipped. `moq play --delay` and `moq export ts --delay` use one delay knob for +presentation and staleness. To weigh: diff --git a/quest/m1/tstd/README.md b/quest/m1/tstd/README.md index ec8f9dc2cff..29b1fbc055f 100644 --- a/quest/m1/tstd/README.md +++ b/quest/m1/tstd/README.md @@ -22,6 +22,5 @@ nightly. ## Required -- [Fixed-delay release](/quest/m1/tstd/delay.md) - frames go out at media time plus a fixed `--delay`, in one order under loss - [T-STD check](/quest/m1/tstd/check.md) - the harness grades the full buffer model instead of the transport buffer alone - [TS byte schedule](/quest/m1/tstd/byte-schedule.md) - PCRs sit on the byte grid the mux rate implies, paced against the fixed delay diff --git a/quest/m1/tstd/byte-schedule.md b/quest/m1/tstd/byte-schedule.md index 5ff8a268bd6..eebe67bf8fe 100644 --- a/quest/m1/tstd/byte-schedule.md +++ b/quest/m1/tstd/byte-schedule.md @@ -18,7 +18,7 @@ UDP sink is out of scope; delivery stays with an external tool. spread over the slots before its DTS instead of landing between two PCRs. - That makes the mux run ahead of decode by a buffer delay. Decided (2026-09-30): the delay is the fixed `--delay` from - [fixed-delay release](/quest/m1/tstd/delay.md), not one that grows + release (`moq export ts --delay`, `ts::Export::with_delay`), not one that grows to fit bursts. Say what happens when a burst does not fit the delay (fail loud, or drop to VBR with a warning) and record the choice here. - The CLI `Delivery` pacer releases one slice per PCR interval; confirm it @@ -29,7 +29,3 @@ UDP sink is out of scope; delivery stays with an external tool. a PCR slot; the generated clip mostly pads and passes already. - `doc/bin/cli.md`: say that export pads to `mpegts.muxRate` on a constant-rate schedule and what latency that adds. - -## Required - -- [Fixed-delay release](/quest/m1/tstd/delay.md) - the delay this schedule paces against diff --git a/quest/m1/tstd/check.md b/quest/m1/tstd/check.md index a7c42f511f8..dbf3b1fdfe3 100644 --- a/quest/m1/tstd/check.md +++ b/quest/m1/tstd/check.md @@ -21,5 +21,5 @@ a deliberately bursty one fails. Where TSDuck or another maintained tool already implements T-STD, prefer it over hand-rolled math. Gate: strict `tstd` in the TS recipe for export output. Until -[fixed-delay release](/quest/m1/tstd/delay.md) lands, the check will likely +fixed-delay release (`moq export ts --delay`) lands, the check will likely fail, so land it report-only and flip the gate in that PR. diff --git a/quest/m1/tstd/delay.md b/quest/m1/tstd/delay.md deleted file mode 100644 index e07d26616e6..00000000000 --- a/quest/m1/tstd/delay.md +++ /dev/null @@ -1,74 +0,0 @@ -# [M] Muxing exports release frames on a fixed delay - -## Goal - -`moq export ts --delay ` works like an SRT receiver's TSBPD, through a -release stage in moq-mux that the FLV and MKV exports adopt next. Each frame -goes out at its media time plus a fixed delay, measured from when the first -frame arrived, and in `(pts, pid)` order across tracks. A frame that misses its -deadline is dropped and counted. Output order is then a function of the media -alone, even under sustained loss, and the output runs at the source's pace -instead of bursting after a stall. - -## Plan - -Decided (2026-09-30): - -- Why: TS assumes a constant end-to-end delay. Its tracks share one PCR clock, - and a receiver's T-STD buffers are sized for access units that arrive a fixed - time before their DTS, which is what SRT and UDP sinks deliver. -- The release stage is shared, not TS-specific: it takes each track's - frames, releases them in `(pts, track)` order at anchor + pts + delay, and - drops and counts late ones. TS adopts it here; FLV and MKV follow in their - own quests. fMP4 stays out: it deliberately skips cross-track ordering, - because its demuxers index by track (`fmp4/export.rs`). The fixed delay is - needed only at the final receiver; relays stay hop-by-hop. - Open for the PR: the stage's name and module. Ask the maintainer. -- One mode. Fixed-delay release replaces the stall/hold interleave in - `rs/moq-mux/src/container/ts/export.rs` (`stall`, `hold`, and the wait in - `pick_next_track`), which is deleted. #4618 is the interim fix for #4613 - until this lands. The hold can't stay deterministic under loss, because a - rewind clears every track's `timeline`, so the mux falls back to arrival - order until a quiet track shows. -- The release clock is anchored at the first frame's local arrival. A rewind - (a new program generation) re-anchors it. Publisher clock drift slowly - grows or shrinks the buffer; correct it only if measured. -- The flag is `--delay`, like `moq play`'s: one knob sets both the release - delay and the sources' staleness budget, and it replaces `--max-age` on - `moq export ts` (`moq play` already dropped `--max-age` for the same reason). - moq-srt egress passes its SRT latency (`rs/moq-srt/src/ts.rs`, which feeds - it to the muxer as the skip budget today). -- Deadlines are on decode time, not PTS. Reordered video reaches the muxer in - decode order (PTS 0, 120, 40, 80 ms), so a PTS deadline would either strand - a B-frame behind its reference or send it first. Each frame's deadline is - anchor + its authored DTS + delay, each track keeps its decode order, and - tracks interleave by `(DTS, pid)`, which is also the order T-STD removes - access units in. -- Late frames are dropped, as SRT's too-late drop does. A dropped video frame - leaves its track waiting for the next keyframe. - -Open: a backlogged or finished source (a file sink, an archive replay) would -come out at real-time pace from the first frame's arrival. Decide whether -release only gates drops and ordering, with pacing left to the CLI's -`Delivery` pacer, or whether file sinks skip it. - -Test with mocked time: two exporters fed the same frames with different -arrival skew, every frame inside its deadline in both, emit identical packet -order. Separately, a frame arriving after its deadline is dropped and counted, -and the rest still go out in `(DTS, pid)` order. A B-frame fixture (decode -order PTS 0, 120, 40, 80) loses nothing on a clean path. It must pass before -the hold is deleted. Rerun the #4613 netem rig (10% loss, 120 s) and -compare with #4618's numbers. - -Update `doc/bin/cli.md` and the `moq export ts` examples. - -Public API: `ts::Export` takes the delay in place of its max age and loses the -hold; breaking, on `dev`. Wire: -none. - -## Related - -- [FLV export delay](/quest/m1/flv-export-delay.md) - adopts the release stage -- [MKV export delay](/quest/m1/mkv-export-delay.md) - adopts the release stage -- [TS byte schedule](/quest/m1/tstd/byte-schedule.md) - uses this delay as its mux-ahead buffer delay -- [Plan: max-delay](/quest/m1/plan-max-delay.md) - whether `max_age` becomes `max_delay` everywhere else diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 01678e32b9e..a0d33540f6b 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -282,17 +282,6 @@ impl Invocation { /// Called before anything binds a port or dials out, so a refused invocation has /// no side effects to unwind. pub fn validate(&self) -> anyhow::Result<()> { - for command in &self.stages { - if let Command::Export(export) = command - && let Some(stdout) = export.sink.stdout() - { - anyhow::ensure!( - stdout.linger.is_zero() || matches!(stdout.format, SubscribeFormat::Ts), - "--linger needs an output that can mark a restart, and only `export ts` can" - ); - } - } - // One stage is what the CLI has always run, so nothing below can bite. if self.stages.len() == 1 { return Ok(()); @@ -854,7 +843,7 @@ impl Export { } match &self.sink { ExportSink::Fmp4(args) | ExportSink::Mkv(args) => found.extend(args.container.deprecated()), - ExportSink::Ts(args) => found.extend(args.container.deprecated()), + ExportSink::Ts(args) => found.extend(args.deprecated()), ExportSink::Flv(args) | ExportSink::H264(args) | ExportSink::H265(args) => found.extend(args.deprecated()), ExportSink::Hls(hls) => found.extend(hls.tls.deprecated()), ExportSink::Rtmp(rtmp) if rtmp.latency_max.is_some() => { @@ -904,7 +893,7 @@ impl ExportSink { let container = |format, container: &Container| Stdout { format, max_age: container.max_age.into_std(), - linger: container.linger.into_std(), + linger: Duration::ZERO, fragment_duration: None, mux_rate: None, }; @@ -918,8 +907,11 @@ impl ExportSink { ..container(SubscribeFormat::Mkv, &args.container) }, Self::Ts(args) => Stdout { + format: SubscribeFormat::Ts, + max_age: args.delay.into_std(), + linger: args.linger.into_std(), + fragment_duration: None, mux_rate: args.mux_rate, - ..container(SubscribeFormat::Ts, &args.container) }, Self::Flv(args) => container(SubscribeFormat::Flv, args), Self::H264(args) => container(SubscribeFormat::H264, args), @@ -932,6 +924,7 @@ impl ExportSink { /// A stdout sink's format and the options that apply to it. pub struct Stdout { pub format: SubscribeFormat, + /// The staleness budget, which `ts` also holds every frame for (`--delay`). pub max_age: Duration, pub linger: Duration, pub fragment_duration: Option, @@ -946,11 +939,6 @@ pub struct Container { #[usage(long, default = "500ms")] pub max_age: crate::duration::Duration, - /// How long to wait for the broadcast to come back once it ends (e.g. `10s`). - /// `ts` only; the output stops while it is gone and resumes flagged as a break. - #[usage(long, default = "0s")] - pub linger: crate::duration::Duration, - /// The released spelling of [`Self::max_age`]. #[usage(long = "latency-max", hide = true)] latency_max: Option, @@ -966,18 +954,48 @@ impl Container { } } -/// The MPEG-TS stdout container: [`Container`] plus null padding. +/// The MPEG-TS stdout container. #[derive(usage::Args, Clone)] #[usage(unknown_flags = "error", args_override_self = false)] pub struct Transport { - #[usage(flatten)] - pub container: Container, + /// How long after its decode time each frame is written (e.g. `500ms`), like an SRT + /// receiver's latency. It is also how stale a group may get before it is skipped, + /// and a frame that arrives later than this is dropped. + #[usage(long, default = "500ms")] + pub delay: crate::duration::Duration, + + /// How long to wait for the broadcast to come back once it ends (e.g. `10s`). + /// The output stops while it is gone and resumes flagged as a break. + #[usage(long, default = "0s")] + pub linger: crate::duration::Duration, /// Pad the output with null packets to this constant rate, in bits per second. /// Defaults to the multiplex rate the catalog recorded from a constant-rate /// source (`mpegts.muxRate`); without either the output is unpadded. #[usage(long)] pub mux_rate: Option, + + /// The released spelling of [`Self::delay`]. + #[usage(long = "max-age", hide = true)] + max_age: Option, + + /// The released spelling of [`Self::delay`], before `--max-age`. + #[usage(long = "latency-max", hide = true)] + latency_max: Option, +} + +impl Transport { + fn deprecated(&self) -> moq_tokio::cli::Deprecated { + const HOLDS: &str = "it also holds every frame that long after its decode time"; + let mut found = moq_tokio::cli::Deprecated::default(); + if self.max_age.is_some() { + found.changed("--max-age", None, "--delay", HOLDS); + } + if self.latency_max.is_some() { + found.changed("--latency-max", None, "--delay", HOLDS); + } + found + } } /// The fmp4 / mkv stdout containers: [`Container`] plus a fragment cap. @@ -1024,18 +1042,41 @@ mod tests { /// Only TS can mark where a returned broadcast restarts, so only `export ts` may linger. #[test] fn linger_is_ts_only() { - let parse = |format: &str, linger: &str| { - Invocation::try_parse_from(["moq", "--connect", "http://relay", "export", format, "--linger", linger]) - .unwrap() + let parse = |format: &str| { + Invocation::try_parse_from(["moq", "--connect", "http://relay", "export", format, "--linger", "10s"]) }; - assert!(parse("ts", "10s").validate().is_ok()); + assert!(parse("ts").unwrap().validate().is_ok()); for format in ["fmp4", "mkv", "flv", "h264", "h265"] { - let err = parse(format, "10s").validate().unwrap_err().to_string(); - assert!(err.contains("--linger"), "{format}: {err}"); - assert!( - parse(format, "0s").validate().is_ok(), - "{format}: no linger is always fine" - ); + let Err(err) = parse(format) else { + panic!("{format} took --linger"); + }; + assert!(err.to_string().contains("--linger"), "{format}: {err}"); + } + } + + /// `export ts` holds every frame for its staleness budget, so the budget is spelled + /// `--delay` there, and the released spellings stop a run with a migration. + #[test] + fn export_ts_takes_a_delay() { + let parse = |args: &[&str]| { + let mut argv = vec!["moq", "export", "--broadcast", "b", "ts"]; + argv.extend_from_slice(args); + Invocation::try_parse_from(argv) + }; + let stdout = |args: &[&str]| { + let cli = parse(args).unwrap(); + let Command::Export(export) = &cli.stages[0] else { + panic!("not an export"); + }; + export.sink.stdout().unwrap().max_age + }; + assert_eq!(stdout(&[]), Duration::from_millis(500)); + assert_eq!(stdout(&["--delay", "2s"]), Duration::from_secs(2)); + for old in ["--max-age", "--latency-max"] { + let Err(err) = parse(&[old, "1s"]) else { + panic!("{old} must not start a run"); + }; + assert!(err.to_string().contains(&format!("{old} -> --delay")), "{err}"); } } @@ -1548,7 +1589,7 @@ mod tests { }; assert!(err.to_string().contains("--latency-max -> --max-age"), "{}", err); - let Err(err) = Invocation::try_parse_from(["moq", "export", "--broadcast", "b", "ts", "--latency-max", "1s"]) + let Err(err) = Invocation::try_parse_from(["moq", "export", "--broadcast", "b", "mkv", "--latency-max", "1s"]) else { panic!("--latency-max must not start a run"); }; diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 3dc1adf9632..f5dea993e9a 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -632,7 +632,9 @@ mod tests { /// `next()` blocks, surfaced here as a timeout once the buffered frames are gone. async fn drain(mut exporter: Export) -> Vec { let mut out = Vec::new(); - while let Ok(res) = tokio::time::timeout(Duration::from_millis(500), exporter.next()).await { + // Past the export's release delay, so the first frame goes out before it ends. + let wait = RECORDING_MAX_AGE + Duration::from_secs(1); + while let Ok(res) = tokio::time::timeout(wait, exporter.next()).await { match res.expect("exporter error") { Some(frame) => out.extend_from_slice(&frame.payload), None => break, @@ -729,7 +731,7 @@ mod tests { Export::with_ts(moq_mux::Source::new(origin.consume(), "cli"), CatalogFormat::Hang) .await .unwrap() - .with_max_age(RECORDING_MAX_AGE), + .with_delay(RECORDING_MAX_AGE), ) .await } @@ -783,7 +785,7 @@ mod tests { Export::with_ts(moq_mux::Source::new(origin.consume(), "cli"), format) .await .unwrap() - .with_max_age(RECORDING_MAX_AGE), + .with_delay(RECORDING_MAX_AGE), ) .await; diff --git a/rs/moq-cli/src/subscribe.rs b/rs/moq-cli/src/subscribe.rs index 2d1d8f67c58..aede13db84f 100644 --- a/rs/moq-cli/src/subscribe.rs +++ b/rs/moq-cli/src/subscribe.rs @@ -136,7 +136,8 @@ pub struct SubscribeArgs { /// The format to write to stdout. pub format: SubscribeFormat, - /// How far playback may drift from the live edge before skipping groups. + /// How far playback may drift from the live edge before skipping groups. TS also + /// holds every frame this long after its decode time (`--delay`). pub max_age: Duration, /// How long to wait for the broadcast to come back after it ends (TS only). @@ -307,7 +308,7 @@ impl Subscribe { let mut broadcast = source.broadcast().await?; let mut ts = moq_mux::container::ts::Export::with_ts(self.source, self.catalog) .await? - .with_max_age(self.args.max_age); + .with_delay(self.args.max_age); if let Some(mux_rate) = self.args.mux_rate { ts = ts.with_mux_rate(mux_rate); } @@ -439,7 +440,7 @@ async fn resume_within( /// gone out any earlier. What that gives up is a producer running faster than real /// time indefinitely: the sink keeps pace with it and falls further behind live /// without the budget noticing. Bounding *that* is the export's own -/// `--max-age`, which sheds media rather than compressing the clock. +/// `--delay`, which sheds media rather than compressing the clock. /// /// The budget is the lead plus whatever standing lag the pacer has absorbed /// ([`Pacer::slack`](moq_mux::Pacer::slack)), which is a distance it is holding on @@ -583,7 +584,7 @@ mod tests { /// media rate instead of being shed, because it is indistinguishable from the /// TS export's own mux buffer from here: both hand over a frame that is ready /// and ahead of the schedule. The size of such a backlog is bounded by the - /// export's `--max-age`, which is where it belongs. + /// export's `--delay`, which is where it belongs. #[tokio::test(start_paused = true)] async fn a_sink_that_cannot_keep_up_sheds_the_lag() { let mut delivery = Delivery::new(Duration::from_millis(500)); diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index e493c9a3afc..8cd9a7fec6a 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -9,6 +9,10 @@ //! position agrees with the values, and a pacing caller releases each slot at //! the instant it asserts. Video is carried as Annex-B, audio as ADTS AAC. //! +//! Every frame is muxed a fixed delay after its decode time ([`Export::with_delay`]), +//! so tracks interleave in `(DTS, PID)` order whatever the arrival skew between them, +//! and the output keeps the source's pace. +//! //! Video flows through [`ExportSource`], which normalizes every H.264/H.265 //! source to length-prefixed NALU plus a resolved avcC/hvcC (parsing in-band //! avc3/hev1 parameter sets out of the bitstream, or taking the catalog @@ -17,7 +21,6 @@ //! inline NALs on every keyframe. CMAF tracks are rejected with a clear error. use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; -use std::pin::Pin; use std::task::Poll; use std::time::Duration; @@ -40,6 +43,7 @@ use crate::catalog::{CatalogFormat, Stream}; use crate::codec::video::Reorder; use crate::codec::{aac, annexb}; use crate::container::{ExportSource, Frame}; +use crate::release::{Arrival, Push, Release, Released}; use super::adts; use super::catalog; @@ -117,7 +121,14 @@ pub struct Export { /// Tracks [`Self::resume`] left on the ended broadcast, resubscribed as the /// returned catalog lists them. stale: HashSet, - max_age: Duration, + /// How long after its decode time each frame goes out, and each source's staleness budget. + delay: Duration, + /// Holds every track's frames until `delay` past their decode time, keyed by PID. + release: Release, + /// A released frame waiting for the tail of the generation before it to go out. + held: Option>, + /// Release generation of the program being muxed; a newer one rewinds it. + generation: u64, tracks: HashMap, /// Continuity counter per PID (PAT, PMT, and each elementary stream). @@ -173,12 +184,6 @@ pub struct Export { /// step backwards all the time, so a span closes on a timestamp passing this /// high-water mark rather than on every frame. watermark: Option, - /// When the interleave started waiting on a lagging track: the arrival of the - /// first leading frame it held. Cleared once every track has caught up, or by - /// [`Self::resume`], but not by a rewind ([`Self::pick_next_track`]). - stall: Option, - /// Wakes [`Self::pick_next_track`] once the stall has lasted `max_age`. - hold: Option>>, /// The rate to pad the output to with null packets, in bits per second: the /// builder override when set, else the catalog's recorded multiplex rate, else /// none and the output is unpadded ([`Self::stuff`]). @@ -199,21 +204,28 @@ pub struct Export { video_start: Option, } +/// A frame read before the program tables are built. struct Pending { frame: Frame, discontinuity: u64, - /// When the frame was pulled from its source: the start of a stall it leads - /// ([`Export::pick_next_track`]). arrived: web_async::time::Instant, } +/// A frame waiting in the [`Release`] stage, with what it needs from the moment it was read. +struct Queued { + frame: Frame, + /// Authored decode timestamp, as [`PesUnit::dts`]. + dts: Option, + /// The avcC/hvcC the frame was read under, for video. + description: Option, +} + struct Track { source: ExportSource, + /// The first frame, held until the program tables are built. pending: Option, - /// Last consumed boundary count from this source. Never compared with peers. + /// The source's discontinuity counter the decode clock (`last_dts`, `timeline`) runs under. discontinuity: u64, - /// Program generation this rendition has joined. Older generations are discarded. - epoch: u64, finished: bool, pid: u16, kind: Kind, @@ -223,8 +235,8 @@ struct Track { /// Last decode timestamp (continuous 90 kHz ticks) authored for this track, keeping the /// decode clock monotonic across reordered (B-frame) video. Only video uses it. last_dts: Option, - /// High-water mark of the timestamps muxed within this rendition, independent of - /// cross-track skew. Bounds where its next frame can land ([`Track::shown`]). + /// High-water mark of the timestamps read within this rendition, which sizes the + /// video [`Reserve`] from how far a reordered frame lands below it. timeline: Option, /// Decode-clock reserve: how far ahead of its PTS each frame decodes. Only video sizes it; /// every other kind holds [`DEFAULT_DTS_RESERVE`]. @@ -232,28 +244,61 @@ struct Track { } impl Track { - /// Admit any frame that belongs to the current program generation. - fn admit(&mut self, pending: Pending, epoch: u64) -> Option { - (self.epoch == epoch).then_some(pending) - } - - /// Whether this track's next frame is known to sort after `(timestamp, pid)`: it - /// holds one, it has finished, or the frames it already muxed bound the next one - /// from below. Video is emitted in decode order, so a B-frame can land up to the - /// reorder depth (the decode-clock reserve) below the high-water mark. - fn shown(&self, timestamp: Timestamp, pid: u16) -> bool { - if self.pending.is_some() || self.finished { - return true; - } - let Some(timeline) = self.timeline else { - return false; + /// Author the frame's decode time and queue it for release. + /// + /// The decode clock runs here, as frames are read, because the release deadline is + /// on decode time: reordered video arrives in decode order with PTS 0, 120, 40, 80, + /// so a PTS deadline would strand a B-frame behind its reference or send it first. + fn queue(&mut self, name: &str, pending: Pending, release: &mut Release) { + let Pending { + frame, + discontinuity, + arrived, + } = pending; + if discontinuity != self.discontinuity { + // The source may have restarted its timeline, so the decode clock restarts too. + self.discontinuity = discontinuity; + self.last_dts = None; + self.timeline = None; + } + + let (dts, sync, description) = match self.kind { + Kind::Video(stream_type) => { + if frame.keyframe { + self.reserve.describe(stream_type, self.source.description(), name); + } + if let Some(timeline) = self.timeline + && frame.timestamp < timeline + { + let gap = to_ticks(timeline) - to_ticks(frame.timestamp); + self.reserve.observe(gap, name); + } else { + self.reserve.peak(); + } + let dts = author_dts(to_ticks(frame.timestamp), self.reserve.ticks, &mut self.last_dts); + (dts, frame.keyframe, self.source.description().cloned()) + } + _ => (None, true, None), }; - let reorder = match self.kind { - Kind::Video(_) => u128::from(self.reserve.ticks) * 1_000_000_000 / 90_000, - _ => 0, + self.timeline = Some(self.timeline.map_or(frame.timestamp, |last| last.max(frame.timestamp))); + + let decode = dts + .and_then(|ticks| Timestamp::from_scale(ticks, 90_000).ok()) + .unwrap_or(frame.timestamp); + let arrival = Arrival { + arrived, + decode, + discontinuity, + sync, + item: Queued { + frame, + dts, + description, + }, }; - let floor = timeline.as_nanos().saturating_sub(reorder); - (floor, self.pid) > (timestamp.as_nanos(), pid) + if release.push(self.pid, arrival) == Push::Late { + tracing::warn!(track = %name, dropped = release.dropped(), "frame missed its release deadline; dropped"); + } } } @@ -669,7 +714,10 @@ impl Export { catalog: Some(catalog), catalog_format, stale: HashSet::new(), - max_age: Duration::ZERO, + delay: Duration::ZERO, + release: Release::new(Duration::ZERO), + held: None, + generation: 0, tracks: HashMap::new(), counters: HashMap::new(), span_counters: None, @@ -692,8 +740,6 @@ impl Export { clock: None, low: None, watermark: None, - stall: None, - hold: None, video_start: None, mux_rate: None, mux_rate_override: None, @@ -716,15 +762,17 @@ impl Export { self } - /// Set the max age for each per-track source, which also bounds how long the - /// interleave holds a leading track for a lagging one. + /// Mux each frame this long after its decode time, like an SRT receiver's TSBPD. /// - /// See [`Consumer`](crate::container::Consumer) for the per-track skip behavior. - /// Frames are muxed in media-time order across every track, waiting up to this - /// long for a track that has not yet shown where its next frame lands. Defaults - /// to [`Duration::ZERO`] (skip aggressively, and mux in arrival order). - pub fn with_max_age(mut self, max_age: Duration) -> Self { - self.max_age = max_age; + /// The clock starts at the first frame's arrival, so the output keeps the source's + /// pace and interleaves every track in `(DTS, PID)` order whatever the arrival skew + /// between them. A frame that arrives after its deadline is dropped, and a video + /// track that dropped one resumes at its next keyframe. The delay is also each + /// source's staleness budget (see [`Consumer`](crate::container::Consumer)). + /// Defaults to [`Duration::ZERO`]. + pub fn with_delay(mut self, delay: Duration) -> Self { + self.delay = delay; + self.release = Release::new(delay); self } @@ -769,7 +817,8 @@ impl Export { si.poll(waiter); } - // 2. Pull a frame into every idle track. + // 2. Read every frame the sources have: into the release stage, or before the + // program tables are built, one per track. self.fill(waiter)?; // 3. Build the program tables once the layout is resolved and every @@ -807,12 +856,24 @@ impl Export { track.pending = None; } } - // Show where the dropped tracks resume, or the interleave waits on them. - self.fill(waiter)?; } + // Hand the held frames to the release stage in arrival order, so the + // earliest anchors its clock, then read on. + let mut held: Vec<(web_async::time::Instant, String)> = self + .tracks + .iter() + .filter_map(|(name, t)| t.pending.as_ref().map(|p| (p.arrived, name.clone()))) + .collect(); + held.sort(); + for (_, name) in held { + let track = self.tracks.get_mut(&name).unwrap(); + let pending = track.pending.take().unwrap(); + track.queue(&name, pending, &mut self.release); + } + self.fill(waiter)?; } - // 4. Mux the smallest-timestamp pending frame into the open span (the first + // 4. Mux each frame the release stage lets go into the open span (the first // one carries the buffered PAT/PMT). Nothing goes out until a later // timestamp measures that span: only then is it known how many bytes it // carried, which is what puts the clock packets at the byte position their @@ -823,58 +884,44 @@ impl Export { self.emitted_epoch = self.epoch; return Poll::Ready(Ok(Some(out))); } - let Some(name) = self.pick_next_track(waiter) else { - break; + let released = match self.held.take() { + Some(released) => released, + None => match self.release.poll_next(waiter) { + Poll::Ready(released) => released, + Poll::Pending => break, + }, }; - let pending = self.tracks.get_mut(&name).unwrap().pending.take().unwrap(); - let changed = pending.discontinuity != self.tracks[&name].discontinuity; - if changed { - let joined = self.tracks[&name].epoch == self.epoch; - if joined { - if !self.pending.is_empty() { - // A boundary ends valid media rather than reneging it. - // Return that tail under the old generation before adopting the new one. - self.emit(None)?; - self.tracks.get_mut(&name).unwrap().pending = Some(pending); - continue; - } - self.rewind(); - } - let track = self.tracks.get_mut(&name).unwrap(); - track.discontinuity = pending.discontinuity; - track.epoch = self.epoch; - track.last_dts = None; - track.timeline = None; - } - let frame = pending.frame; - let track = self.tracks.get_mut(&name).unwrap(); - if let Kind::Video(stream_type) = track.kind { - if frame.keyframe { - track.reserve.describe(stream_type, track.source.description(), &name); - } - if let Some(timeline) = track.timeline - && frame.timestamp < timeline - { - let gap = to_ticks(timeline) - to_ticks(frame.timestamp); - track.reserve.observe(gap, &name); - } else { - track.reserve.peak(); + if released.generation > self.generation { + if !self.pending.is_empty() { + // A boundary ends valid media rather than reneging it. + // Return that tail under the old generation before adopting the new one. + self.emit(None)?; + self.held = Some(released); + continue; } + self.rewind(); + self.generation = released.generation; } - track.timeline = Some(track.timeline.map_or(frame.timestamp, |last| last.max(frame.timestamp))); - self.last_timestamp = Some(frame.timestamp); - self.advance(frame.timestamp)?; - self.mux(&name, frame)?; - // Refill the track we just drained: the next span is measured by its - // successor, and without the refill nothing would be polling for it. - self.fill(waiter)?; + let name = self + .tracks + .iter() + .find(|(_, t)| t.pid == released.track) + .map(|(name, _)| name.clone()) + .context("released frame for an unknown PID")?; + let timestamp = released.item.frame.timestamp; + self.last_timestamp = Some(timestamp); + self.advance(timestamp)?; + self.mux(&name, released.item)?; } // 5. Once every track has drained, no later timestamp is coming to measure // the open span, so its bytes go out whole. That's independent of the // catalog: a retained track finishes while the broadcast stays live, and // holding its tail until the catalog closed would strand it indefinitely. - let drained = !self.tracks.is_empty() && self.tracks.values().all(|t| t.finished); + let drained = !self.tracks.is_empty() + && self.tracks.values().all(|t| t.finished) + && self.release.is_empty() + && self.held.is_none(); if drained { self.emit(None)?; if let Some(out) = self.queue.pop_front() { @@ -930,17 +977,17 @@ impl Export { })) } - /// Pull a frame into every idle track. + /// Read every frame the sources have ready. /// - /// [`ExportSource`] has already transformed Annex-B avc3/hev1 into - /// length-prefixed form and resolved the avcC/hvcC. Before the program tables - /// are written, drop slices that arrive before their codec config resolves: a - /// receiver joining mid-GOP can't use them, and parking them would stop us - /// polling for the keyframe that carries the parameter sets. + /// Before the program tables are built, each track holds its first frame instead, + /// and slices that arrive before their codec config resolves are dropped: a receiver + /// joining mid-GOP can't use them, and parking them would stop us polling for the + /// keyframe that carries the parameter sets. [`ExportSource`] has already transformed + /// Annex-B avc3/hev1 into length-prefixed form and resolved the avcC/hvcC. fn fill(&mut self, waiter: &kio::Waiter) -> crate::Result<()> { let waiting_for_header = self.psi.is_none(); let video_start = self.video_start; - for track in self.tracks.values_mut() { + for (name, track) in self.tracks.iter_mut() { if track.pending.is_some() || track.finished { continue; } @@ -956,19 +1003,18 @@ impl Export { discontinuity: track.source.discontinuity(), arrived: web_async::time::Instant::now(), }; - let Some(pending) = track.admit(pending, self.epoch) else { - continue; - }; - let changed = pending.discontinuity != track.discontinuity; // A new timeline must reach the reset before tune-in alignment can drop it. if let Some(start) = video_start - && !is_video && !changed + && !is_video && pending.discontinuity == track.discontinuity && pending.frame.timestamp < start { continue; } - track.pending = Some(pending); - break; + if waiting_for_header { + track.pending = Some(pending); + break; + } + track.queue(name, pending, &mut self.release); } Poll::Ready(None) => { track.finished = true; @@ -980,6 +1026,7 @@ impl Export { } Ok(()) } + fn update_catalog(&mut self, mut catalog: Catalog) -> anyhow::Result<()> { self.source.retain_valid(&mut catalog); @@ -1020,7 +1067,7 @@ impl Export { // staying attached would repeat its stale sections forever. The last // snapshot carries across so emission never goes dark mid-swap. Some(existing) if existing.track != entry.track => { - let mut replacement = SiTrack::new(&self.source, entry, self.max_age); + let mut replacement = SiTrack::new(&self.source, entry, self.delay); // An inline entry already holds its snapshot; only a track entry // has nothing to emit until its first group lands. if replacement.active.is_empty() { @@ -1046,7 +1093,7 @@ impl Export { } None => { self.si - .insert((*pid, *table_id), SiTrack::new(&self.source, entry, self.max_age)); + .insert((*pid, *table_id), SiTrack::new(&self.source, entry, self.delay)); } } } @@ -1139,7 +1186,7 @@ impl Export { self.tracks.insert(name.clone(), track); } None => { - let Some(source) = ExportSource::for_video(&self.source, name, config, self.max_age)? else { + let Some(source) = ExportSource::for_video(&self.source, name, config, self.delay)? else { continue; }; let mut reserve = Reserve::default(); @@ -1160,7 +1207,7 @@ impl Export { self.tracks.insert(name.clone(), track); } None => { - let Some(source) = ExportSource::for_audio(&self.source, name, config, self.max_age)? else { + let Some(source) = ExportSource::for_audio(&self.source, name, config, self.delay)? else { continue; }; self.insert_track(name, source, pid, kind, descriptors, Reserve::default()); @@ -1186,7 +1233,7 @@ impl Export { self.tracks.insert(name.clone(), existing); } None => { - let source = ExportSource::for_stream(&self.source, name, self.max_age)?; + let source = ExportSource::for_stream(&self.source, name, self.delay)?; self.insert_track(name, source, pid, kind, descriptors, Reserve::default()); } } @@ -1204,11 +1251,11 @@ impl Export { continue; } let source = if let Some(config) = catalog.video.renditions.get(name) { - ExportSource::for_video(&self.source, name, config, self.max_age)? + ExportSource::for_video(&self.source, name, config, self.delay)? } else if let Some(config) = catalog.audio.renditions.get(name) { - ExportSource::for_audio(&self.source, name, config, self.max_age)? + ExportSource::for_audio(&self.source, name, config, self.delay)? } else if mpegts.tracks.get(name).is_some_and(|t| t.verbatim.is_some()) { - Some(ExportSource::for_stream(&self.source, name, self.max_age)?) + Some(ExportSource::for_stream(&self.source, name, self.delay)?) } else { None }; @@ -1238,7 +1285,6 @@ impl Export { source, pending: None, discontinuity: 0, - epoch: self.epoch, finished: false, pid, kind, @@ -1250,6 +1296,18 @@ impl Export { ); } + /// How many frames were dropped for missing their release deadline. + #[cfg(test)] + pub(super) fn dropped(&self) -> u64 { + self.release.dropped() + } + + /// When the next queued frame is due for release. + #[cfg(test)] + pub(super) fn next_release(&self) -> Option { + self.release.next_deadline() + } + /// The discontinuity counter of the most recently returned output frame. /// Compare across reads and re-anchor pacing when it changes. Renditions have /// independent source counters; this counter describes the emitted program. @@ -1282,18 +1340,21 @@ impl Export { track.finished = true; track.pending = None; track.discontinuity = 0; + track.last_dts = None; + track.timeline = None; self.stale.insert(name.clone()); } - // A replacement broadcast gets its own budget: only a rewind within one broadcast - // carries a stall across. - self.stall = None; - self.hold = None; + // The replacement's clock starts afresh at its own first frame. + self.release.clear(); + self.held = None; + self.generation = 0; self.rewind(); Ok(()) } /// Discard uncommitted bytes and restart the program clock. Every rendition - /// joins the new generation: no track is fenced across a declared marker. + /// joins the new generation: no track is fenced across a declared marker, since a + /// latency skip on one track says nothing about another's timeline. fn rewind(&mut self) { self.epoch += 1; if let Some(counters) = self.span_counters.take() { @@ -1303,8 +1364,6 @@ impl Export { self.keyframes.clear(); self.queue.clear(); self.watermark = None; - // The stall carries across with its budget, and so does a held frame's - // `arrived` ([`Self::pick_next_track`]). self.clock = None; self.low = None; self.last_pcr = None; @@ -1315,17 +1374,6 @@ impl Export { self.video_start = None; self.pcr_discontinuity = true; self.stuffing = Stuffing::default(); - for track in self.tracks.values_mut() { - track.last_dts = None; - track.timeline = None; - track.epoch = self.epoch; - if let Some(pending) = track.pending.as_ref() { - track.discontinuity = pending.discontinuity; - } - if let Some(pending) = track.pending.take() { - track.pending = track.admit(pending, self.epoch); - } - } } /// Header is ready when every track's [`ExportSource`] has resolved its @@ -1508,53 +1556,18 @@ impl Export { Ok(()) } - /// The track whose pending frame goes next, in `(timestamp, pid)` order across - /// every track rather than only those whose frame has arrived. - /// - /// The earliest pending frame waits until every other track has shown it cannot - /// be preceded ([`Track::shown`]), so the interleave is a function of the media - /// and two exporters of one broadcast render it in one order whatever the arrival - /// skew between tracks. Once that wait has lasted `max_age`, the same budget the - /// sources give a stalled group, output goes around the lagging track until it - /// catches up; zero keeps arrival order. The stall is timed from its first held - /// frame rather than per frame: a frame's successor is only pulled once it goes - /// out, so a per-frame wait would release one frame per `max_age`. A rewind does - /// not restart it either: a source's latency skip is a rewind, and a hold renewed - /// at each one delays every source by the budget they skip on, so under loss the - /// feed collapses into alternating holds and skips. [`Self::resume`] does, since a - /// replacement broadcast owes nothing to the one it replaced. No track is fenced, - /// so a boundary does not jump the queue. - fn pick_next_track(&mut self, waiter: &kio::Waiter) -> Option { - let (timestamp, pid, name, arrived) = self - .tracks - .iter() - .filter_map(|(n, t)| t.pending.as_ref().map(|p| (p.frame.timestamp, t.pid, n, p.arrived))) - .min_by_key(|(timestamp, pid, name, _)| (*timestamp, *pid, *name))?; - let name = name.clone(); - if self.max_age.is_zero() { - return Some(name); - } - if self.tracks.values().all(|t| t.shown(timestamp, pid)) { - self.stall = None; - return Some(name); - } - let deadline = *self.stall.get_or_insert(arrived) + self.max_age; - let hold = self - .hold - .get_or_insert_with(|| Box::pin(web_async::time::sleep_until(deadline))); - if hold.deadline() != deadline { - hold.as_mut().reset(deadline); - } - waiter.poll_future(hold.as_mut()).is_ready().then_some(name) - } - /// Packetize one media frame into the open span, re-emitting PAT/PMT before /// video keyframes (and periodically) so receivers can tune in mid-stream. /// /// The bytes are buffered rather than returned: which grid slots they belong /// to isn't known until a later timestamp measures the span (see /// [`Self::advance`]). - fn mux(&mut self, name: &str, frame: Frame) -> anyhow::Result<()> { + fn mux(&mut self, name: &str, queued: Queued) -> anyhow::Result<()> { + let Queued { + frame, + dts, + description, + } = queued; if self.span_counters.is_none() { self.span_counters = Some(self.counters.clone()); } @@ -1572,7 +1585,7 @@ impl Export { // resolved avcC/hvcC to rewrite length-prefixed NALs as Annex-B. Section-framed // verbatim streams carry no PES payload; the section is written separately below. let es_payload = match &kind { - Kind::Video(stream_type) => Some(video_es_payload(*stream_type, track.source.description(), &frame)?), + Kind::Video(stream_type) => Some(video_es_payload(*stream_type, description.as_ref(), &frame)?), Kind::Aac(aac) => { let pce = aac.program_config.as_deref().unwrap_or_default(); let raw_len = pce.len() + frame.payload.len(); @@ -1598,16 +1611,6 @@ impl Export { } => None, }; - // Author a monotonic decode timeline for reordered video (B-frames). Other kinds - // never reorder, so DTS == PTS and the PES stays PTS-only. - let dts = if is_video { - let pts = to_ticks(frame.timestamp); - let track = self.tracks.get_mut(name).context("missing track")?; - author_dts(pts, track.reserve.ticks, &mut track.last_dts) - } else { - None - }; - let mut out = Vec::with_capacity(TsPacket::SIZE); // Refresh PSI at keyframes or after the interval lapses. @@ -1761,7 +1764,7 @@ impl Export { // // The clock only ever moves forward. A track skewed far enough behind that it // decodes before the clock already reached it can't be placed ahead of its own - // decode time, and `with_latency` owns that skew, so its bytes go out at the + // decode time, and [`Self::with_delay`] owns that skew, so its bytes go out at the // clock rather than dragging it backwards. let start = match self.clock { Some(clock) => clock.as_nanos(), diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index f51b49f1952..38b0969feb3 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -69,6 +69,9 @@ fn length_prefixed(nals: &[&[u8]]) -> Bytes { /// exporter's default [`Duration::ZERO`] collapses to the /// live edge: completeness has to be asked for, exactly as a real recorder does. const RECORDING_MAX_AGE: Duration = Duration::from_secs(30); +/// How long a drain waits for the next frame: past the recording delay, so the first +/// frame goes out, and then until output stops. +const DRAIN: std::time::Duration = RECORDING_MAX_AGE.saturating_add(std::time::Duration::from_secs(1)); async fn drain(consumer: moq_net::broadcast::Consumer) -> BytesMut { drain_with(Export::new(crate::source::announced(&consumer)).await.unwrap()).await @@ -76,10 +79,10 @@ async fn drain(consumer: moq_net::broadcast::Consumer) -> BytesMut { /// `drain` for an exporter built with an explicit catalog extension. async fn drain_with(exporter: Export) -> BytesMut { - let mut exporter = exporter.with_max_age(RECORDING_MAX_AGE); + let mut exporter = exporter.with_delay(RECORDING_MAX_AGE); let mut out = BytesMut::new(); // `while let Ok` stops on the first timeout (`Pending`: no more output). - while let Ok(res) = tokio::time::timeout(Duration::from_secs(1), exporter.next()).await { + while let Ok(res) = tokio::time::timeout(DRAIN, exporter.next()).await { let Some(frame) = res.expect("exporter error") else { break; }; @@ -778,7 +781,7 @@ async fn export_pcr_wraps_below_the_reserve_at_start() { let mut exporter = Export::new(crate::source::announced(&consumer)) .await .unwrap() - .with_max_age(RECORDING_MAX_AGE); + .with_delay(RECORDING_MAX_AGE); let frames = drain_frames(&mut exporter).await; // A slot's clock packets lead the frame carrying that slot's bytes, and the @@ -1254,7 +1257,7 @@ async fn scte35_without_video_export_is_rejected() { .await .unwrap(); let err = loop { - match tokio::time::timeout(Duration::from_secs(1), exporter.next()).await { + match tokio::time::timeout(DRAIN, exporter.next()).await { Ok(Ok(Some(_))) => continue, Ok(Ok(None)) => panic!("export completed; a cue program without video must be rejected"), Ok(Err(e)) => break e, @@ -1558,7 +1561,7 @@ async fn aac_export_refuses_what_adts_cannot_label() { let mut exporter = Export::new(crate::source::announced(&consumer)).await.unwrap(); let err = loop { - match tokio::time::timeout(Duration::from_secs(1), exporter.next()).await { + match tokio::time::timeout(DRAIN, exporter.next()).await { Ok(Ok(Some(_))) => continue, Ok(Ok(None)) => panic!("export completed; expected a refusal naming {refusal}"), Ok(Err(e)) => break e, @@ -2430,7 +2433,7 @@ async fn export_rejects_si_pid_on_the_pmt() { let mut exporter = Export::with_ts(crate::source::announced(&consumer), crate::catalog::CatalogFormat::Hang) .await .unwrap() - .with_max_age(RECORDING_MAX_AGE); + .with_delay(RECORDING_MAX_AGE); let err = exporter .next() .await @@ -2479,7 +2482,7 @@ async fn export_rejects_si_pid_on_an_elementary_stream() { let mut exporter = Export::with_ts(crate::source::announced(&consumer), crate::catalog::CatalogFormat::Hang) .await .unwrap() - .with_max_age(RECORDING_MAX_AGE); + .with_delay(RECORDING_MAX_AGE); let err = exporter .next() .await @@ -2495,7 +2498,7 @@ async fn export_of(consumer: &moq_net::broadcast::Consumer) -> Export (Vec, Vec) { /// whole (this compares them one by one, not as one byte stream). async fn drain_frames(export: &mut Export) -> Vec { let mut out = Vec::new(); - while let Ok(res) = tokio::time::timeout(Duration::from_secs(1), export.next()).await { - match res.expect("exporter error") { - Some(frame) => out.push(frame), - None => break, + loop { + out.extend(poll_frames(export)); + // Run the paused clock only as far as the next release, so a test that writes + // more afterwards is still on time for it. + if let Some(deadline) = export.next_release() { + tokio::time::sleep_until(deadline).await; + continue; + } + match tokio::time::timeout(Duration::from_millis(10), export.next()).await { + Ok(Ok(Some(frame))) => out.push(frame), + Ok(Ok(None)) => break, + Err(_) if export.next_release().is_none() => break, + Err(_) => {} + Ok(Err(err)) => panic!("exporter error: {err}"), } } out @@ -3646,9 +3656,9 @@ async fn late_join_matches_a_running_exporter_without_video() { assert_only_continuity_differs(&a, &b, from); } -// The interleave is a function of the media, not of arrival (moq-dev/moq#2829): the -// earliest pending frame waits for every track to show it cannot be preceded, bounded -// by the exporter's `max_age`. +// The interleave is a function of the media, not of arrival (moq-dev/moq#2829): each frame +// goes out a fixed delay after its decode time, in `(DTS, PID)` order, and one that arrives +// past that deadline is dropped. /// A video and an audio rendition written frame by frame, on the late-join fixture's /// cadence, so a test decides exactly what each exporter has seen when it polls. @@ -3660,6 +3670,8 @@ struct Interleave { source: crate::Source, /// The next audio frame to write. audio_index: u64, + /// The paused clock's instant at media time zero, set by the first [`Self::at`]. + start: Option, } impl Interleave { @@ -3714,16 +3726,23 @@ impl Interleave { video, audio, audio_index: 0, + start: None, } } /// An exporter with its subscriptions resolved, so later polls need no runtime. - async fn export(&self, max_age: Duration) -> Export { - let mut export = Export::new(self.source.clone()).await.unwrap().with_max_age(max_age); + async fn export(&self, delay: Duration) -> Export { + let mut export = Export::new(self.source.clone()).await.unwrap().with_delay(delay); assert!(drain_frames(&mut export).await.is_empty()); export } + /// Run the paused clock to `micros` of media time, as a live source would. + async fn at(&mut self, micros: u64) { + let start = *self.start.get_or_insert_with(tokio::time::Instant::now); + tokio::time::sleep_until(start + Duration::from_micros(micros)).await; + } + fn video(&mut self, tick: u64) { let keyframe = tick.is_multiple_of(GOP); let slice = if keyframe { @@ -3765,8 +3784,7 @@ impl Interleave { } } -/// Pull every frame an exporter can render without letting time pass, so a `max_age` -/// wait lapses only where a test advances the clock itself. +/// Pull every frame an exporter can render without letting time pass. fn poll_frames(export: &mut Export) -> Vec { let waiter = kio::Waiter::noop(); let mut out = Vec::new(); @@ -3779,8 +3797,9 @@ fn poll_frames(export: &mut Export) -> Vec { out } -/// Every PES start's PTS in the order the stream carries them, across PIDs. -fn pes_pts_in_order(frames: &[Frame]) -> Vec { +/// Every PES start's decode time (its DTS, else its PTS) in the order the stream carries +/// them, across PIDs. +fn pes_decode_in_order(frames: &[Frame]) -> Vec { let bytes: Vec = frames.iter().flat_map(|f| f.payload.iter().copied()).collect(); let mut reader = TsPacketReader::new(Cursor::new(bytes)); let mut out = Vec::new(); @@ -3788,55 +3807,61 @@ fn pes_pts_in_order(frames: &[Frame]) -> Vec { if let Some(TsPayload::PesStart(pes)) = packet.payload && let Some(pts) = pes.header.pts { - out.push(pts.as_u64()); + out.push(pes.header.dts.unwrap_or(pts).as_u64()); } } out } -/// Audio lagging video by a frame: each video frame lands before the audio that -/// precedes it. Returns the PTS order the exporter emitted. -async fn lagging_audio(max_age: Duration) -> Vec { +/// Audio lagging video by up to a frame, on a live clock: each video frame lands before +/// the audio that precedes it. Returns the decode order the exporter emitted. +async fn lagging_audio(delay: Duration) -> Vec { let mut rig = Interleave::new(); - let mut export = rig.export(max_age).await; + let mut export = rig.export(delay).await; let mut out = Vec::new(); for tick in 0..TICKS / 2 { + rig.at(tick * VIDEO_US).await; rig.video(tick); out.extend(poll_frames(&mut export)); rig.audio_until(tick * VIDEO_US, &mut export, &mut out); } rig.finish(); out.extend(drain_frames(&mut export).await); - pes_pts_in_order(&out) + assert_eq!(export.dropped(), 0, "every frame arrived inside its deadline"); + pes_decode_in_order(&out) } #[tokio::test(start_paused = true)] async fn late_audio_still_leads_the_interleave() { - let pts = lagging_audio(Duration::from_millis(500)).await; - assert!(pts.len() > 100, "too little output to judge: {}", pts.len()); - assert!(pts.is_sorted(), "a frame went out ahead of an earlier one: {pts:?}"); + let decode = lagging_audio(Duration::from_millis(500)).await; + assert!(decode.len() > 100, "too little output to judge: {}", decode.len()); + assert!( + decode.is_sorted(), + "a frame went out ahead of an earlier one: {decode:?}" + ); } -/// Zero `max_age` waits for nothing, so the interleave stays in arrival order. +/// A zero delay holds nothing, so the interleave stays in arrival order. #[tokio::test(start_paused = true)] -async fn zero_max_age_keeps_arrival_order() { - let pts = lagging_audio(Duration::ZERO).await; +async fn zero_delay_keeps_arrival_order() { + let decode = lagging_audio(Duration::ZERO).await; assert!( - !pts.is_sorted(), + !decode.is_sorted(), "video should have led the audio that arrived after it" ); } -/// Two exporters that see the same frames arrive in different orders render the same -/// bytes: one polls after every frame (video ahead of its audio), the other only once -/// each tick's frames have all landed. +/// Two exporters that see the same frames arrive with different skew, every frame inside +/// its deadline in both, render the same bytes: one polls after every frame (video ahead +/// of its audio), the other only once each tick's frames have all landed. #[tokio::test(start_paused = true)] async fn arrival_order_does_not_change_the_output() { let mut rig = Interleave::new(); - let max_age = Duration::from_millis(500); - let (mut eager, mut batched) = (rig.export(max_age).await, rig.export(max_age).await); + let delay = Duration::from_millis(500); + let (mut eager, mut batched) = (rig.export(delay).await, rig.export(delay).await); let (mut out_eager, mut out_batched) = (Vec::new(), Vec::new()); for tick in 0..TICKS / 2 { + rig.at(tick * VIDEO_US).await; rig.video(tick); out_eager.extend(poll_frames(&mut eager)); rig.audio_until(tick * VIDEO_US, &mut eager, &mut out_eager); @@ -3852,183 +3877,56 @@ async fn arrival_order_does_not_change_the_output() { assert!(eager == batched, "arrival order changed the rendering"); } -/// A track quiet past `max_age` is emitted around, and once it catches up the -/// interleave waits for it again. +/// Audio that arrives past its deadline is dropped and counted, and everything else still +/// goes out in decode order. #[tokio::test(start_paused = true)] -async fn quiet_track_is_emitted_around_then_rejoins() { - let max_age = Duration::from_millis(500); +async fn a_late_frame_is_dropped_and_the_rest_keep_their_order() { + let delay = Duration::from_millis(100); let mut rig = Interleave::new(); - let mut export = rig.export(max_age).await; + let mut export = rig.export(delay).await; let mut out = Vec::new(); - - rig.video(0); - rig.audio_until(1, &mut export, &mut out); - - // Audio goes quiet while video runs on. Video waits for it, then goes around it. - for tick in 1..=10 { - rig.video(tick); - } - assert!(poll_frames(&mut export).is_empty(), "video waits for the quiet audio"); - tokio::time::advance(max_age).await; - out.extend(poll_frames(&mut export)); - assert!( - pes_pts_in_order(&out) - .iter() - .any(|&pts| pts >= 9 * VIDEO_US * 90 / 1_000), - "video went out around the quiet audio" - ); - - // Audio catches up with its backlog, then keeps lagging by a frame: the order - // resumes without the clock moving. - rig.audio_until(10 * VIDEO_US, &mut export, &mut out); - for tick in 11..TICKS / 2 { + for tick in 0..TICKS / 2 { + rig.at(tick * VIDEO_US).await; rig.video(tick); out.extend(poll_frames(&mut export)); - rig.audio_until(tick * VIDEO_US, &mut export, &mut out); + // Audio stalls for 400ms, then its backlog lands at once. + if !(20..30).contains(&tick) { + rig.audio_until(tick * VIDEO_US, &mut export, &mut out); + } } rig.finish(); out.extend(drain_frames(&mut export).await); - // Everything from the first frame written after the catch-up on is in order. - let mut pts = pes_pts_in_order(&out); - let resumed = pts.iter().position(|&pts| pts >= 11 * VIDEO_US * 90 / 1_000).unwrap(); - let pts = pts.split_off(resumed); - assert!(pts.len() > 100, "too little output to judge: {}", pts.len()); - assert!(pts.is_sorted(), "the interleave did not resume: {pts:?}"); + let written = rig.audio_index + TICKS / 2; + let decode = pes_decode_in_order(&out); + assert!(export.dropped() > 0, "the stalled audio missed its deadline"); + assert_eq!( + decode.len() as u64, + written - export.dropped(), + "only the late frames were dropped" + ); + assert!( + decode.is_sorted(), + "a frame went out ahead of an earlier one: {decode:?}" + ); } -/// Audio dropped for leading the first keyframe does not stall the interleave: the -/// audio already cached past the keyframe shows where the track resumes. +/// Audio dropped for leading the first keyframe does not stall the interleave. #[tokio::test(start_paused = true)] async fn tune_in_does_not_wait_on_dropped_audio() { + let delay = Duration::from_millis(500); let mut rig = Interleave::new(); - let mut export = rig.export(Duration::from_millis(500)).await; + let mut export = rig.export(delay).await; let mut out = Vec::new(); rig.audio_until(3 * GOP * VIDEO_US, &mut export, &mut out); for tick in GOP..2 * GOP { rig.video(tick); } out.extend(poll_frames(&mut export)); - assert!(!out.is_empty(), "the tune-in waited on audio it had dropped"); -} - -/// A rewind taken while going around a quiet track keeps going around it: the -/// expired hold carries into the new generation rather than starting afresh. -#[tokio::test(start_paused = true)] -async fn rewind_keeps_an_expired_stall() { - let max_age = Duration::from_millis(500); - let mut rig = Interleave::new(); - let mut export = rig.export(max_age).await; - let mut out = Vec::new(); - - rig.video(0); - rig.audio_until(1, &mut export, &mut out); - for tick in 1..=5 { - rig.video(tick); - } - assert!(poll_frames(&mut export).is_empty(), "video waits for the quiet audio"); - tokio::time::advance(max_age).await; - out.extend(poll_frames(&mut export)); - assert!(!out.is_empty(), "video went out around the quiet audio"); - - rig.video.discontinuity().unwrap(); - for tick in 0..=5 { - rig.video(GOP + tick); - } + // Past the keyframe's deadline, and far enough on that a later frame closes its span. + tokio::time::advance(delay + Duration::from_micros(4 * VIDEO_US)).await; out.extend(poll_frames(&mut export)); - let after = pes_pts_in_order(&out) - .into_iter() - .filter(|&pts| pts >= GOP * VIDEO_US * 90 / 1_000) - .count(); - assert!(after > 0, "the new generation waited on the audio again"); - assert_eq!(export.discontinuity(), 1); -} - -/// A frame held across a rewind keeps the wait it has already served. -#[tokio::test(start_paused = true)] -async fn rewind_keeps_a_held_frame_waiting_time() { - let max_age = Duration::from_millis(500); - let mut rig = Interleave::new(); - let mut export = rig.export(max_age).await; - let mut out = Vec::new(); - - rig.video(0); - rig.audio_until(1, &mut export, &mut out); - for tick in 1..=5 { - rig.video(tick); - } - // Audio jumps past the coming break, so it is held waiting on the video. - rig.audio_index = (GOP + 3) * VIDEO_US / AUDIO_US; - rig.audio_until(rig.audio_index * AUDIO_US + 1, &mut export, &mut out); - tokio::time::advance(max_age).await; - - rig.video.discontinuity().unwrap(); - rig.video(GOP); - out.extend(poll_frames(&mut export)); - assert_eq!(export.discontinuity(), 1, "the held audio waited a second budget"); -} - -/// A rewind part way through a hold neither ends it nor renews it: the new generation -/// waits out only what is left of the budget. -#[tokio::test(start_paused = true)] -async fn rewind_keeps_a_partial_stall() { - let max_age = Duration::from_millis(500); - let mut rig = Interleave::new(); - let mut export = rig.export(max_age).await; - let mut out = Vec::new(); - - rig.video(0); - rig.audio_until(1, &mut export, &mut out); - for tick in 1..=5 { - rig.video(tick); - } - assert!(poll_frames(&mut export).is_empty(), "video waits for the quiet audio"); - tokio::time::advance(max_age / 2).await; - assert!(poll_frames(&mut export).is_empty(), "the hold lapsed early"); - - rig.video.discontinuity().unwrap(); - for tick in 0..=5 { - rig.video(GOP + tick); - } - assert!(poll_frames(&mut export).is_empty(), "the rewind ended the hold"); - - tokio::time::advance(max_age / 2).await; - assert!(!poll_frames(&mut export).is_empty(), "the rewind renewed the hold"); - assert_eq!(export.discontinuity(), 1); -} - -/// Under loss every source skip is a rewind. However many arrive while a sparse -/// track stays quiet, the interleave holds for one budget, not one per rewind: -/// renewing it at each would delay every source by the budget they skip on, -/// and the feed would collapse into alternating holds and skips. -#[tokio::test(start_paused = true)] -async fn repeated_rewinds_hold_once() { - let max_age = Duration::from_millis(500); - let mut rig = Interleave::new(); - let mut export = rig.export(max_age).await; - let mut out = Vec::new(); - - rig.video(0); - rig.audio_until(1, &mut export, &mut out); - for tick in 1..=5 { - rig.video(tick); - } - assert!(poll_frames(&mut export).is_empty(), "video waits for the quiet audio"); - tokio::time::advance(max_age).await; - out.extend(poll_frames(&mut export)); - - for generation in 1..=8 { - rig.video.discontinuity().unwrap(); - for tick in 0..=5 { - rig.video(generation * GOP + tick); - } - out.extend(poll_frames(&mut export)); - assert_eq!( - export.discontinuity(), - generation, - "rewind {generation} held the feed for a fresh budget" - ); - } + assert!(!out.is_empty(), "the tune-in waited on audio it had dropped"); } /// A section lost before the cycle wraps commits an observed subset; the next @@ -4137,7 +4035,7 @@ async fn repointed_si_entry_resubscribes() { let mut exporter = Export::with_ts(crate::source::announced(&consumer), crate::catalog::CatalogFormat::Hang) .await .unwrap() - .with_max_age(RECORDING_MAX_AGE); + .with_delay(RECORDING_MAX_AGE); let mut before = BytesMut::new(); write_key(&mut producer, 0); write_key(&mut producer, 1); @@ -4168,7 +4066,7 @@ async fn repointed_si_entry_resubscribes() { // that should carry the new sections, or the exporter drains them all first. let mut after = BytesMut::new(); write_key(&mut producer, 2); - let frame = tokio::time::timeout(Duration::from_secs(1), exporter.next()) + let frame = tokio::time::timeout(DRAIN, exporter.next()) .await .expect("a frame after the switch") .unwrap() @@ -4180,7 +4078,7 @@ async fn repointed_si_entry_resubscribes() { write_key(&mut producer, 3); write_key(&mut producer, 4); producer.finish().unwrap(); - while let Ok(res) = tokio::time::timeout(Duration::from_secs(1), exporter.next()).await { + while let Ok(res) = tokio::time::timeout(DRAIN, exporter.next()).await { let Some(frame) = res.unwrap() else { break }; after.extend_from_slice(&frame.payload); } @@ -4264,7 +4162,7 @@ async fn si_revision_after_final_media_frame_is_flushed() { .await .unwrap(); let first = loop { - let frame = tokio::time::timeout(Duration::from_secs(1), exporter.next()) + let frame = tokio::time::timeout(DRAIN, exporter.next()) .await .expect("the only media frame") .unwrap() @@ -4283,7 +4181,7 @@ async fn si_revision_after_final_media_frame_is_flushed() { .unwrap(); catalog.finish().unwrap(); - let tail = tokio::time::timeout(Duration::from_secs(1), exporter.next()) + let tail = tokio::time::timeout(DRAIN, exporter.next()) .await .expect("a trailing SI frame rather than an immediate end") .unwrap() @@ -4293,7 +4191,7 @@ async fn si_revision_after_final_media_frame_is_flushed() { contains(&tail.payload, &sdt_v2), "the trailing flush carries the revision" ); - let end = tokio::time::timeout(Duration::from_secs(1), exporter.next()) + let end = tokio::time::timeout(DRAIN, exporter.next()) .await .expect("the stream ends after the flush") .unwrap(); @@ -4770,7 +4668,7 @@ async fn export_cbr_video() -> Vec { let mut exporter = Export::new(crate::source::announced(&consumer)) .await .unwrap() - .with_max_age(RECORDING_MAX_AGE); + .with_delay(RECORDING_MAX_AGE); drain_frames(&mut exporter).await } @@ -4852,7 +4750,7 @@ async fn pcr_stays_among_the_bytes_across_reordered_tracks() { let mut exporter = Export::new(crate::source::announced(&consumer)) .await .unwrap() - .with_max_age(RECORDING_MAX_AGE); + .with_delay(RECORDING_MAX_AGE); let frames = drain_frames(&mut exporter).await; let pcrs = collect_pcrs(&frames); assert!(pcrs.len() > 50, "expected the full feed, got {} PCRs", pcrs.len()); @@ -5232,7 +5130,7 @@ impl Reordered { /// An exporter with its subscriptions resolved, so later polls need no runtime. async fn export(&self, max_age: Duration) -> Export { - let mut export = Export::new(self.source.clone()).await.unwrap().with_max_age(max_age); + let mut export = Export::new(self.source.clone()).await.unwrap().with_delay(max_age); assert!(drain_frames(&mut export).await.is_empty()); export } @@ -5437,6 +5335,45 @@ async fn declared_reorder_sizes_the_decode_clock_from_the_first_frame() { assert_decodes_after_the_clock(&out_late); } +/// Reordered video (each pyramid in decode order: PTS 0, 160, 80, 40, 120 ms) loses nothing on +/// a clean path. The release deadline is on decode time, so no B-frame is stranded behind its +/// reference or sent ahead of it, and every track interleaves in decode order. +#[tokio::test(start_paused = true)] +async fn reordered_video_loses_nothing_on_a_clean_path() { + let delay = Duration::from_millis(500); + let mut rig = Reordered::new(SPS, PPS, &[0, 1, 2, 3, 4]); + let mut export = rig.export(delay).await; + let start = tokio::time::Instant::now(); + let mut out = Vec::new(); + for tick in 0..TICKS / 2 { + tokio::time::sleep_until(start + Duration::from_micros(tick * VIDEO_US)).await; + rig.video(tick); + while rig.audio_before(tick + 1) {} + out.extend(poll_frames(&mut export)); + } + rig.finish(); + out.extend(drain_frames(&mut export).await); + + assert_eq!(export.dropped(), 0, "a clean path drops nothing"); + assert_eq!( + video_timing(&out, ..).len() as u64, + TICKS / 2, + "every video frame went out" + ); + // The SPS declares no reorder depth, but the first pyramid is read a delay ahead of + // the first PCR, so the reserve it grows never steps the clock back. + assert_eq!( + count_discontinuity(&out), + 0, + "the reserve settled before the clock started" + ); + let decode = pes_decode_in_order(&out); + assert!( + decode.is_sorted(), + "a frame went out ahead of an earlier one: {decode:?}" + ); +} + /// With nothing declared, the reserve grows to the deepest reordering muxed so far. This is the /// one path where it depends on when an exporter joined: a joiner that has muxed only shallow /// groups runs a shallower clock than a runner that saw a deep one before the join, until the @@ -5564,7 +5501,7 @@ async fn a_track_leaving_the_catalog_is_read_to_its_end() { let mut export = Export::new(crate::source::announced(&consumer)) .await .unwrap() - .with_max_age(RECORDING_MAX_AGE); + .with_delay(RECORDING_MAX_AGE); for ms in (0..200).step_by(20) { write_aac(&mut kept, ms); write_aac(&mut leaving, ms); @@ -5606,10 +5543,7 @@ async fn resume_after(finish: bool) { let (mut broadcast, mut catalog) = publish(); let mut track = aac_rendition(&mut broadcast, &mut catalog, "a.aac"); let ended = source.broadcast().await.unwrap(); - let mut export = Export::new(source.clone()) - .await - .unwrap() - .with_max_age(RECORDING_MAX_AGE); + let mut export = Export::new(source.clone()).await.unwrap().with_delay(RECORDING_MAX_AGE); for ms in (0..200).step_by(20) { write_aac(&mut track, ms); } @@ -5649,66 +5583,6 @@ async fn resume_after(finish: bool) { assert_eq!(pes_count(&after), 10, "the returned broadcast's frames all went out"); } -/// A replacement broadcast gets a fresh interleave budget: a hold that expired on the -/// broadcast before it doesn't let the new one go around a track still within its own. -#[tokio::test(start_paused = true)] -async fn resume_does_not_inherit_an_expired_stall() { - let max_age = Duration::from_secs(2); - let origin = crate::source::produce_origin(); - let source = crate::Source::new(origin.consume(), "live"); - let publish = || { - let mut broadcast = origin.publish("live", Default::default()).unwrap(); - let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - (broadcast, catalog) - }; - // Output that needs no more than a tenth of the budget to pass. - async fn quick(export: &mut Export, max_age: Duration) -> Vec { - let mut out = Vec::new(); - while let Ok(frame) = tokio::time::timeout(max_age / 10, export.next()).await { - out.extend(frame.expect("exporter error")); - } - out - } - - let (mut broadcast, mut catalog) = publish(); - let mut leading = aac_rendition(&mut broadcast, &mut catalog, "a.aac"); - let mut quiet = aac_rendition(&mut broadcast, &mut catalog, "b.aac"); - let ended = source.broadcast().await.unwrap(); - let mut export = Export::new(source.clone()).await.unwrap().with_max_age(max_age); - write_aac(&mut quiet, 0); - for ms in (0..200).step_by(20) { - write_aac(&mut leading, ms); - } - quick(&mut export, max_age).await; - tokio::time::advance(max_age).await; - assert!( - !quick(&mut export, max_age).await.is_empty(), - "the leading track went around the quiet one" - ); - drop((broadcast, catalog, leading, quiet)); - let (_, end) = drain_to_end(&mut export).await; - assert!(end.is_err(), "a drop fails the export"); - - let (mut broadcast, mut catalog) = publish(); - let mut leading = aac_rendition(&mut broadcast, &mut catalog, "a.aac"); - let mut quiet = aac_rendition(&mut broadcast, &mut catalog, "b.aac"); - source.returned(&ended).await.unwrap(); - export.resume().await.unwrap(); - for ms in (0..200).step_by(20) { - write_aac(&mut leading, ms); - } - assert_eq!( - pes_count(&quick(&mut export, max_age).await), - 0, - "the replacement went around its quiet track on the old broadcast's budget" - ); - write_aac(&mut quiet, 200); - assert!( - !quick(&mut export, max_age).await.is_empty(), - "the replacement went out once both tracks showed" - ); -} - #[tokio::test(start_paused = true)] async fn resume_after_a_finish() { resume_after(true).await; diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 6bbb02801a4..c79d6f9237a 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -6293,16 +6293,16 @@ pub(super) mod test { import.finish().unwrap(); // `import` and `catalog` stay alive so the exporter can subscribe to the finished, - // retained tracks. A generous max age, since the default of zero would shed the - // earlier groups and the boundary with them. + // retained tracks. A generous delay, since the default of zero would shed the + // earlier groups and the boundary with them. The paused clock runs out the + // delay and every gap in the media without a real wait. + let delay = std::time::Duration::from_secs(3600); let mut exporter = crate::container::ts::Export::new(crate::source::announced(&consumer)) .await .unwrap() - .with_max_age(std::time::Duration::from_secs(3600)); + .with_delay(delay); let mut flagged = 0; - while let Ok(Ok(Some(frame))) = - tokio::time::timeout(std::time::Duration::from_millis(100), exporter.next()).await - { + while let Ok(Ok(Some(frame))) = tokio::time::timeout(2 * delay, exporter.next()).await { flagged += frame .payload .as_chunks::<188>() diff --git a/rs/moq-mux/src/container/ts/import_test.rs b/rs/moq-mux/src/container/ts/import_test.rs index f06ed12c92e..f9dc05ccfa2 100644 --- a/rs/moq-mux/src/container/ts/import_test.rs +++ b/rs/moq-mux/src/container/ts/import_test.rs @@ -12,6 +12,9 @@ use bytes::BytesMut; /// [`std::time::Duration::ZERO`](std::time::Duration::ZERO) budget collapses to the live edge: /// completeness has to be asked for. const RECORDING_MAX_AGE: std::time::Duration = std::time::Duration::from_secs(30); +/// How long a drain waits for the next frame: past the recording delay, so the first +/// frame goes out, and then until output stops. +const DRAIN: std::time::Duration = RECORDING_MAX_AGE.saturating_add(std::time::Duration::from_secs(1)); /// Decode a whole TS buffer into a fresh broadcast and return the catalog. fn import_ts(data: &[u8]) -> crate::catalog::hang::Catalog { @@ -441,9 +444,9 @@ async fn import_export_import_roundtrip() { let mut exporter = crate::container::ts::Export::new(crate::source::announced(&consumer)) .await .unwrap() - .with_max_age(RECORDING_MAX_AGE); + .with_delay(RECORDING_MAX_AGE); let mut out = BytesMut::new(); - while let Ok(res) = tokio::time::timeout(std::time::Duration::from_secs(1), exporter.next()).await { + while let Ok(res) = tokio::time::timeout(DRAIN, exporter.next()).await { match res.expect("exporter error") { Some(frame) => out.extend_from_slice(&frame.payload), None => break, diff --git a/rs/moq-mux/src/lib.rs b/rs/moq-mux/src/lib.rs index b128bf22ded..087eb6f93e9 100644 --- a/rs/moq-mux/src/lib.rs +++ b/rs/moq-mux/src/lib.rs @@ -36,6 +36,7 @@ pub mod import; pub mod json; mod pace; pub mod rate; +mod release; pub mod select; mod source; pub mod timeline; diff --git a/rs/moq-mux/src/pace.rs b/rs/moq-mux/src/pace.rs index 919b30560e4..83bae1786dd 100644 --- a/rs/moq-mux/src/pace.rs +++ b/rs/moq-mux/src/pace.rs @@ -261,7 +261,7 @@ mod tests { } /// Regression: the lead comparison must not construct `now + lead`, which - /// panics on a large but valid `Duration` (`--max-age` is unbounded). + /// panics on a large but valid `Duration` (`--delay` is unbounded). #[test] fn huge_lead_does_not_overflow() { let start = Instant::now(); diff --git a/rs/moq-mux/src/release.rs b/rs/moq-mux/src/release.rs new file mode 100644 index 00000000000..041d0834c7b --- /dev/null +++ b/rs/moq-mux/src/release.rs @@ -0,0 +1,417 @@ +//! Fixed-delay release of several tracks' frames in one decode order. + +use std::collections::{BTreeMap, VecDeque}; +use std::pin::Pin; +use std::task::Poll; +use std::time::Duration; + +use moq_net::Timestamp; +use web_async::time::{Instant, Sleep}; + +/// A frame handed to [`Release::push`]. +pub(crate) struct Arrival { + /// When the frame was read from its source. + pub arrived: Instant, + /// When the frame decodes, nondecreasing within a track between discontinuities. + pub decode: Timestamp, + /// The source's discontinuity counter when the frame was read. + pub discontinuity: u64, + /// Whether the frame decodes without the track's earlier frames. + pub sync: bool, + pub item: T, +} + +/// A frame [`Release::poll_next`] let go. +pub(crate) struct Released { + pub track: K, + /// Counts up each time a track's discontinuity started a new anchor. Tracks that + /// cross the same discontinuity share one. + pub generation: u64, + pub item: T, +} + +/// What [`Release::push`] did with a frame. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Push { + Queued, + /// It missed its deadline and was dropped; the track resumes at its next sync frame. + Late, + /// It was dropped because the track is waiting for a sync frame. + Waiting, +} + +/// Releases each track's frames at a fixed delay past their decode time, like an SRT +/// receiver's TSBPD, in `(deadline, track)` order across tracks. +/// +/// The clock is anchored at the first frame's arrival. Within one anchor the deadline +/// order is the decode order, so two releases that saw the same frames arrive with +/// different skew, each within its deadline, emit them in the same order. A frame that +/// arrives past its deadline would break that order, so it is dropped and counted. +/// +/// A track whose source reports a discontinuity may have restarted its timeline, so it +/// moves to a new generation with its own anchor: the newest one when another track got +/// there first and the frame lands on its clock, otherwise one anchored at this frame, +/// though never ahead of a deadline already given out. A backlog read in one go still +/// releases each generation after the one before it, and a live stream keeps its delay. +/// +/// A zero delay holds nothing and drops nothing: each frame goes out as soon as it is +/// read, ordered only among the frames read together. +pub(crate) struct Release { + delay: Duration, + /// Each generation's anchor: the first frame's arrival and decode time. + anchors: BTreeMap, + tracks: BTreeMap>, + /// The latest deadline given out. + horizon: Option, + timer: Option>>, + dropped: u64, +} + +struct Track { + /// Frames in arrival order, each with its deadline and generation. + queue: VecDeque<(Instant, u64, T)>, + generation: u64, + discontinuity: u64, + /// A frame was dropped, so later ones are too until the next sync frame. + waiting: bool, +} + +impl Release { + pub fn new(delay: Duration) -> Self { + Self { + delay, + anchors: BTreeMap::new(), + tracks: BTreeMap::new(), + horizon: None, + timer: None, + dropped: 0, + } + } + + /// Queue a frame for release, or drop it if it cannot make its deadline. + pub fn push(&mut self, key: K, arrival: Arrival) -> Push { + // A new track counts from the start, as if it had always been there. + let (mut generation, discontinuity) = self + .tracks + .get(&key) + .map_or((0, 0), |track| (track.generation, track.discontinuity)); + if discontinuity != arrival.discontinuity { + generation = self.rejoin(generation, &arrival); + } + let anchor = *self.anchors.entry(generation).or_insert_with(|| { + let after = self.horizon.and_then(|horizon| horizon.checked_sub(self.delay)); + ( + after.map_or(arrival.arrived, |after| after.max(arrival.arrived)), + arrival.decode, + ) + }); + let deadline = self.deadline(anchor, arrival.decode); + + let track = self.tracks.entry(key).or_insert_with(|| Track { + queue: VecDeque::new(), + generation, + discontinuity: 0, + waiting: false, + }); + track.generation = generation; + track.discontinuity = arrival.discontinuity; + let push = match deadline { + _ if track.waiting && !arrival.sync => Push::Waiting, + Some(deadline) if arrival.arrived <= deadline || self.delay.is_zero() => { + track.waiting = false; + track.queue.push_back((deadline, generation, arrival.item)); + self.horizon = self.horizon.max(Some(deadline)); + Push::Queued + } + // Past the deadline, or so far off the clock that no instant holds it. + _ => { + track.waiting = true; + Push::Late + } + }; + if push != Push::Queued { + self.dropped += 1; + } + + // Generation 0 stays, since a track that has yet to show up starts there. + let oldest = self.tracks.values().map(|track| track.generation).min(); + self.anchors + .retain(|generation, _| *generation == 0 || oldest.is_none_or(|oldest| *generation >= oldest)); + push + } + + /// The generation a track at `generation` moves to on a discontinuity: the newest, if + /// another track opened it and this frame lands on its clock, or else a new one. + /// + /// Landing on the clock means the frame is neither late nor held past both an on-time + /// frame and everything already queued, which a different timeline would be. + fn rejoin(&self, generation: u64, arrival: &Arrival) -> u64 { + let Some((&latest, &anchor)) = self.anchors.last_key_value() else { + return generation + 1; + }; + let lands = self.deadline(anchor, arrival.decode).is_some_and(|deadline| { + let bound = (arrival.arrived + self.delay).max(self.horizon.unwrap_or(arrival.arrived)); + arrival.arrived <= deadline && deadline <= bound + }); + match latest > generation && lands { + true => latest, + false => latest.max(generation) + 1, + } + } + + /// When a frame decoding at `decode` goes out under `anchor`, if any instant holds it. + fn deadline(&self, (anchor, base): (Instant, Timestamp), decode: Timestamp) -> Option { + // Signed, since a frame can decode before the one that anchored its generation. + let offset = decode.as_nanos() as i128 - base.as_nanos() as i128 + self.delay.as_nanos() as i128; + let nanos = u64::try_from(offset.unsigned_abs()).ok()?; + match offset >= 0 { + true => anchor.checked_add(Duration::from_nanos(nanos)), + false => anchor.checked_sub(Duration::from_nanos(nanos)), + } + } + + /// The next frame whose deadline has come, earliest deadline first, ties by track. + pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll> { + let Some((deadline, key)) = self + .tracks + .iter() + .filter_map(|(key, track)| track.queue.front().map(|(deadline, ..)| (*deadline, key))) + .min() + else { + self.timer = None; + return Poll::Pending; + }; + if !self.delay.is_zero() && Instant::now() < deadline { + let timer = self + .timer + .get_or_insert_with(|| Box::pin(web_async::time::sleep_until(deadline))); + if timer.deadline() != deadline { + timer.as_mut().reset(deadline); + } + if waiter.poll_future(timer.as_mut()).is_pending() { + return Poll::Pending; + } + } + let track = key.clone(); + let (_, generation, item) = self.tracks.get_mut(&track).and_then(|t| t.queue.pop_front()).unwrap(); + Poll::Ready(Released { + track, + generation, + item, + }) + } + + /// When the next queued frame is due. + #[cfg(test)] + pub fn next_deadline(&self) -> Option { + self.tracks + .values() + .filter_map(|track| track.queue.front().map(|(deadline, ..)| *deadline)) + .min() + } + + /// Whether no frame is waiting for release. + pub fn is_empty(&self) -> bool { + self.tracks.values().all(|track| track.queue.is_empty()) + } + + /// How many frames were dropped, late or waiting for a sync frame. + pub fn dropped(&self) -> u64 { + self.dropped + } + + /// Drop every queued frame and anchor, so the next frame starts the clock afresh. + pub fn clear(&mut self) { + self.anchors.clear(); + self.tracks.clear(); + self.horizon = None; + self.timer = None; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const DELAY: Duration = Duration::from_millis(100); + + fn ms(ms: u64) -> Timestamp { + Timestamp::from_millis(ms).unwrap() + } + + fn arrival(arrived: Instant, decode: u64, item: &'static str) -> Arrival<&'static str> { + Arrival { + arrived, + decode: ms(decode), + discontinuity: 0, + sync: true, + item, + } + } + + /// Every frame due by now, in release order. + fn due(release: &mut Release) -> Vec<&'static str> { + let waiter = kio::Waiter::noop(); + let mut out = Vec::new(); + while let Poll::Ready(released) = release.poll_next(&waiter) { + out.push(released.item); + } + out + } + + #[tokio::test(start_paused = true)] + async fn releases_at_the_delay_in_decode_order() { + let start = Instant::now(); + let mut release = Release::new(DELAY); + assert_eq!(release.push(1, arrival(start, 0, "v0")), Push::Queued); + assert_eq!(release.push(1, arrival(start, 40, "v40")), Push::Queued); + // Audio arrives later than the video, with an earlier decode time. + assert_eq!( + release.push(2, arrival(start + Duration::from_millis(30), 20, "a20")), + Push::Queued + ); + assert_eq!( + release.push(2, arrival(start + Duration::from_millis(30), 40, "a40")), + Push::Queued + ); + + assert!(due(&mut release).is_empty(), "nothing is due before the delay"); + tokio::time::advance(DELAY).await; + assert_eq!(due(&mut release), ["v0"]); + tokio::time::advance(Duration::from_millis(20)).await; + assert_eq!(due(&mut release), ["a20"]); + tokio::time::advance(Duration::from_millis(20)).await; + assert_eq!(due(&mut release), ["v40", "a40"], "a tie goes to the lower track"); + assert!(release.is_empty()); + } + + /// Two releases fed the same frames with different arrival skew, every frame inside + /// its deadline, emit the same order. + #[tokio::test(start_paused = true)] + async fn arrival_skew_does_not_change_the_order() { + let start = Instant::now(); + let (mut early, mut late) = (Release::new(DELAY), Release::new(DELAY)); + let frames = [ + (1, 0, "v0"), + (1, 40, "v40"), + (1, 80, "v80"), + (2, 0, "a0"), + (2, 20, "a20"), + (2, 60, "a60"), + ]; + for (key, decode, item) in frames { + // One sees audio lag by 90ms, the other sees every frame on time. + let skew = if key == 2 { 90 } else { 0 }; + assert_eq!( + early.push(key, arrival(start + Duration::from_millis(decode), decode, item)), + Push::Queued + ); + let arrived = start + Duration::from_millis(decode + skew); + assert_eq!(late.push(key, arrival(arrived, decode, item)), Push::Queued); + } + tokio::time::advance(Duration::from_secs(1)).await; + let order = due(&mut early); + assert_eq!(order, ["v0", "a0", "a20", "v40", "a60", "v80"]); + assert_eq!(due(&mut late), order); + } + + /// A late frame is dropped and counted, and the rest keep their order. A track that + /// dropped a frame waits for its next sync frame. + #[tokio::test(start_paused = true)] + async fn late_frames_are_dropped_until_a_sync_frame() { + let start = Instant::now(); + let mut release = Release::new(DELAY); + release.push(1, arrival(start, 0, "v0")); + release.push(2, arrival(start, 0, "a0")); + let late = start + Duration::from_millis(200); + let video = |decode, sync, item| Arrival { + sync, + ..arrival(late, decode, item) + }; + assert_eq!(release.push(1, video(40, false, "v40")), Push::Late); + assert_eq!(release.push(1, video(200, false, "v200")), Push::Waiting); + assert_eq!(release.push(1, video(240, true, "v240")), Push::Queued); + assert_eq!(release.push(2, arrival(late, 180, "a180")), Push::Queued); + assert_eq!(release.dropped(), 2); + + tokio::time::advance(Duration::from_secs(1)).await; + assert_eq!(due(&mut release), ["v0", "a0", "a180", "v240"]); + } + + /// A discontinuity re-anchors the track, and a track reaching the same discontinuity + /// later joins that anchor rather than making its own. + #[tokio::test(start_paused = true)] + async fn a_discontinuity_re_anchors() { + let start = Instant::now(); + let mut release = Release::new(DELAY); + release.push(1, arrival(start, 5_000, "v5000")); + release.push(2, arrival(start, 5_000, "a5000")); + + // The publisher restarts its timeline at zero, a second in. + let restart = start + Duration::from_secs(1); + let reset = |arrived, decode, item| Arrival { + discontinuity: 1, + ..arrival(arrived, decode, item) + }; + assert_eq!(release.push(1, reset(restart, 0, "v0")), Push::Queued); + // Audio still carries its old timeline for a moment, then restarts too. + assert_eq!(release.push(2, arrival(restart, 6_000, "a6000")), Push::Queued); + let later = restart + Duration::from_millis(10); + assert_eq!(release.push(2, reset(later, 0, "a0")), Push::Queued); + + tokio::time::advance(Duration::from_secs(1) + DELAY).await; + assert_eq!(due(&mut release), ["v5000", "a5000", "v0", "a6000", "a0"]); + } + + /// A backlog read in one go releases a new generation after the one before it. + #[tokio::test(start_paused = true)] + async fn a_backlog_releases_generations_in_turn() { + let start = Instant::now(); + let mut release = Release::new(DELAY); + release.push(1, arrival(start, 5_000, "v5000")); + release.push(1, arrival(start, 5_040, "v5040")); + let reset = Arrival { + discontinuity: 1, + ..arrival(start, 0, "v0") + }; + release.push(1, reset); + release.push(2, arrival(start, 5_020, "a5020")); + + tokio::time::advance(DELAY).await; + assert_eq!(due(&mut release), ["v5000"]); + tokio::time::advance(Duration::from_millis(40)).await; + assert_eq!(due(&mut release), ["a5020", "v5040", "v0"]); + } + + /// Zero holds nothing: frames read together go out at once in decode order, and a + /// frame behind the clock still goes out. + #[tokio::test(start_paused = true)] + async fn zero_delay_releases_on_arrival() { + let start = Instant::now(); + let mut release = Release::new(Duration::ZERO); + release.push(1, arrival(start, 0, "v0")); + release.push(1, arrival(start, 40, "v40")); + release.push(2, arrival(start, 20, "a20")); + assert_eq!(due(&mut release), ["v0", "a20", "v40"]); + + let late = start + Duration::from_secs(1); + assert_eq!(release.push(2, arrival(late, 40, "a40")), Push::Queued); + assert_eq!(due(&mut release), ["a40"]); + assert_eq!(release.dropped(), 0); + } + + #[tokio::test(start_paused = true)] + async fn clear_starts_a_fresh_clock() { + let start = Instant::now(); + let mut release = Release::new(DELAY); + release.push(1, arrival(start, 5_000, "old")); + release.clear(); + assert!(release.is_empty()); + + tokio::time::advance(Duration::from_secs(1)).await; + let now = Instant::now(); + assert_eq!(release.push(1, arrival(now, 0, "new")), Push::Queued); + tokio::time::advance(DELAY).await; + assert_eq!(due(&mut release), ["new"]); + } +} diff --git a/rs/moq-srt/src/ts.rs b/rs/moq-srt/src/ts.rs index b470181c23b..186656b3011 100644 --- a/rs/moq-srt/src/ts.rs +++ b/rs/moq-srt/src/ts.rs @@ -168,13 +168,13 @@ pub struct Subscriber { impl Subscriber { /// Resolve the broadcast at `path` in the origin and prepare to mux it to TS. /// - /// `latency` bounds how long the muxer waits for a stalled group before it - /// skips ahead to a newer one. We reuse the locally configured SRT receive - /// latency for it: SRT paces egress on the media clock, so the skip threshold - /// shares the same latency budget. It's the configured value, not the - /// handshake result (srt-tokio doesn't expose the negotiated latency), so a - /// peer that requests a higher receive latency gets a larger actual buffer - /// than this skip threshold. + /// `latency` is the muxer's release delay: each frame is muxed that long after + /// its decode time, a frame arriving later is dropped, and a stalled group is + /// skipped after it. We reuse the locally configured SRT receive latency for + /// it, the same budget an SRT hop gives a packet. It's the configured value, + /// not the handshake result (srt-tokio doesn't expose the negotiated latency), + /// so a peer that requests a higher receive latency gets a larger actual + /// buffer than this delay. /// /// Returns `Ok(None)` if the broadcast can never be served (path outside the /// consumer's scope, or the origin closed). Otherwise waits for the broadcast @@ -190,7 +190,7 @@ impl Subscriber { let source = moq_mux::Source::new(origin.consume(), path); let export = ts::Export::with_ts(source, moq_mux::catalog::CatalogFormat::Hang) .await? - .with_max_age(latency); + .with_delay(latency); Ok(Some(Self { export })) } diff --git a/test/ts/README.md b/test/ts/README.md index 4fb5950b9d8..7fe35e01623 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -156,7 +156,7 @@ the total and reports the rate over the tail of the sample beside it. A sender that buffers builds a standing lag once and then runs at the media rate: the lag is a constant offset no receiver can see, and it cannot grow past the latency budget the sender is allowed to hold, so set `--drift-ms` to that budget -(`export ts --max-age`, 500 ms by default). A pipe that is not running at the +(`export ts --delay`, 500 ms by default). A pipe that is not running at the media rate never stops accumulating and so breaches any fixed bound given a long enough sample. The tail rate is what tells the two apart, and it needs a sample longer than the lag takes to build: measured against the grid-sliced exporter, diff --git a/test/ts/pcr-timing.py b/test/ts/pcr-timing.py index b318fe39043..aed69a962c0 100755 --- a/test/ts/pcr-timing.py +++ b/test/ts/pcr-timing.py @@ -638,7 +638,7 @@ def main(): type=float, default=500.0, help="bound on accumulated release drift, being the standing lag the sender may " - "hold; set it to the sender's latency budget (moq export ts --max-age, " + "hold; set it to the sender's latency budget (moq export ts --delay, " "itself 500ms by default)", ) ap.add_argument( From bcb0741df3379b058b30f2c2facab3a09280be57 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 18:09:00 -0700 Subject: [PATCH 04/40] fix(moq-mux): close TS export spans on decode time The release stage hands frames over in decode order, so the mux measures its spans and the decode bound on the same clock. The PCR runs one slot behind its grid boundary instead of backing off by the largest DTS reserve, which removes the clock rewinds a growing reserve caused. A frame's arrival is the last instant its source was found empty, so a caller that polls late does not make it late. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export.rs | 102 ++++++++++----------- rs/moq-mux/src/container/ts/export_test.rs | 56 +++++------ 2 files changed, 74 insertions(+), 84 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 8cd9a7fec6a..33ba73efee8 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -161,9 +161,6 @@ pub struct Export { /// Generation of the last returned frame, updated only at the output boundary. emitted_epoch: u64, pcr_discontinuity: bool, - /// Back-off of the last PCR emitted ([`Self::pcr_at`]), so a reserve that grows past it - /// flags the next one as a new time base. - pcr_reserve: Option, /// TS packets muxed into the span that is still open. pending: Vec, /// Offsets into [`pending`](Self::pending) where a keyframe's packets begin, so @@ -177,12 +174,10 @@ pub struct Export { pcr_cc: Option, /// Media time the next span's bytes are transmitted from ([`Export::emit`]). clock: Option, - /// Earliest media timestamp muxed into the open span: the decode time its bytes - /// have to arrive before, so it bounds how far the clock may run. + /// Earliest decode time muxed into the open span: what its bytes have to arrive + /// before, so it bounds how far the clock may run. low: Option, - /// Media timestamp that opened the current span. Reordered (B-frame) timestamps - /// step backwards all the time, so a span closes on a timestamp passing this - /// high-water mark rather than on every frame. + /// Decode time that opened the current span, which closes on a later one. watermark: Option, /// The rate to pad the output to with null packets, in bits per second: the /// builder override when set, else the catalog's recorded multiplex rate, else @@ -204,16 +199,19 @@ pub struct Export { video_start: Option, } -/// A frame read before the program tables are built. +/// A frame read from its source. struct Pending { frame: Frame, discontinuity: u64, + /// The earliest the frame could have arrived: when its source was last found empty. arrived: web_async::time::Instant, } /// A frame waiting in the [`Release`] stage, with what it needs from the moment it was read. struct Queued { frame: Frame, + /// When the frame decodes: its DTS, else its PTS. + decode: Timestamp, /// Authored decode timestamp, as [`PesUnit::dts`]. dts: Option, /// The avcC/hvcC the frame was read under, for video. @@ -224,6 +222,10 @@ struct Track { source: ExportSource, /// The first frame, held until the program tables are built. pending: Option, + /// When the source was last found empty. A frame read since arrived no earlier, and + /// no later than it is read; the release judges it on the earlier bound, so a caller + /// that polls late (a sink sleeping to pace its writes) does not make it late. + empty: web_async::time::Instant, /// The source's discontinuity counter the decode clock (`last_dts`, `timeline`) runs under. discontinuity: u64, finished: bool, @@ -292,6 +294,7 @@ impl Track { sync, item: Queued { frame, + decode, dts, description, }, @@ -303,8 +306,7 @@ impl Track { } /// A video rendition's decode-clock reserve in 90 kHz ticks: how far each frame's DTS runs -/// behind its PTS. It must exceed the rendition's reordering for `DTS <= PTS` to hold, and -/// [`Export::pcr_at`] backs the clock off by the largest one. +/// behind its PTS. It must exceed the rendition's reordering for `DTS <= PTS` to hold. /// /// The catalog `jitter` (the importer's max PTS - DTS) sizes it when published. Otherwise the /// reorder depth the SPS declares sizes it, at the SPS's fixed picture rate or the catalog @@ -312,8 +314,7 @@ impl Track { /// [`DEFAULT_DTS_RESERVE`]. Reordering deeper than all of those raises it further /// ([`Self::observe`]). /// -/// It never shrinks: that would step the decode clock forward past frames already muxed and -/// the PCR ahead of DTS values already sent. +/// It never shrinks: that would step the decode clock forward past frames already read. struct Reserve { /// The catalog `jitter`, in ticks. jitter: Option, @@ -732,7 +733,6 @@ impl Export { epoch: 0, emitted_epoch: 0, pcr_discontinuity: false, - pcr_reserve: None, pending: Vec::new(), keyframes: Vec::new(), queue: VecDeque::new(), @@ -908,9 +908,8 @@ impl Export { .find(|(_, t)| t.pid == released.track) .map(|(name, _)| name.clone()) .context("released frame for an unknown PID")?; - let timestamp = released.item.frame.timestamp; - self.last_timestamp = Some(timestamp); - self.advance(timestamp)?; + self.last_timestamp = Some(released.item.frame.timestamp); + self.advance(released.item.decode)?; self.mux(&name, released.item)?; } @@ -1001,7 +1000,7 @@ impl Export { let pending = Pending { frame, discontinuity: track.source.discontinuity(), - arrived: web_async::time::Instant::now(), + arrived: track.empty, }; // A new timeline must reach the reset before tune-in alignment can drop it. if let Some(start) = video_start @@ -1020,7 +1019,10 @@ impl Export { track.finished = true; break; } - Poll::Pending => break, + Poll::Pending => { + track.empty = web_async::time::Instant::now(); + break; + } } } } @@ -1265,6 +1267,7 @@ impl Export { self.stale.remove(name); track.source = source; track.finished = false; + track.empty = web_async::time::Instant::now(); } Ok(()) } @@ -1284,6 +1287,7 @@ impl Export { Track { source, pending: None, + empty: web_async::time::Instant::now(), discontinuity: 0, finished: false, pid, @@ -1565,6 +1569,7 @@ impl Export { fn mux(&mut self, name: &str, queued: Queued) -> anyhow::Result<()> { let Queued { frame, + decode, dts, description, } = queued; @@ -1578,7 +1583,6 @@ impl Export { aac.program_config.take(); } let is_video = matches!(kind, Kind::Video(_)); - let timestamp = frame.timestamp; let keyframe = frame.keyframe; // Build the elementary-stream payload for this frame. Video needs the @@ -1696,17 +1700,18 @@ impl Export { self.keyframes.push(self.pending.len()); } self.pending.extend_from_slice(&out); - self.low = Some(self.low.map_or(timestamp, |low| low.min(timestamp))); + self.low = Some(self.low.map_or(decode, |low| low.min(decode))); Ok(()) } /// Close the open span if `ts` passes the watermark, laying its bytes out. /// - /// `ts` belongs to the frame about to be muxed. Passing the watermark means the - /// span that timestamp opened is done: everything buffered since is exactly the - /// bytes it carried, and the distance from it measures how long it ran. A - /// reordered (B-frame) timestamp that trails the watermark closes nothing, and - /// its bytes join the open span, which is where they are transmitted anyway. + /// `ts` is the decode time of the frame about to be muxed. Passing the watermark + /// means the span that decode time opened is done: everything buffered since is + /// exactly the bytes it carried, and the distance from it measures how long it ran. + /// The release stage hands frames over in decode order, so only a tie, or a frame + /// read out of order under a zero delay, trails the watermark; it closes nothing, + /// and its bytes join the open span. /// /// This is why nothing goes out on arrival, and why it can't. A span's bytes /// have to reach a receiver before the units in it decode, so they ride the @@ -1740,17 +1745,13 @@ impl Export { /// assert. Each frame is stamped at its own slot boundary, so a pacing caller /// releases the clock at the instant it asserts rather than when the media that /// revealed it arrived. And the interval ends at the span's earliest media - /// timestamp, so every byte precedes the decode time of the unit it belongs to. + /// decode time, so every byte precedes the decode time of the unit it belongs to. /// - /// The PES units cannot carry the clock themselves. Frames arrive in decode - /// order, so the authored DTS is a saw: a reference frame leaps a whole reorder - /// span ahead and each B-frame nudges one tick past it. A PCR sampled from it - /// freezes and jumps, and no downstream CBR stage can repair that, because a - /// groomer can only place the clock samples it receives. So the PCR asserts its - /// own uniform ramp instead: absolute grid slots on the media timeline (shared by - /// every exporter of the broadcast, like [`due`]), each backed off by the largest - /// decode-clock reserve of any track so every PES unit, whichever rendition it - /// belongs to, decodes at or after the clock that precedes it. + /// The PES units cannot carry the clock themselves: a PCR sampled from the DTS + /// would step with the frame cadence, and no downstream CBR stage can repair + /// that, because a groomer can only place the clock samples it receives. So the + /// PCR asserts its own uniform ramp instead: absolute grid slots on the media + /// timeline (shared by every exporter of the broadcast, like [`due`]). fn emit(&mut self, span: Option) -> anyhow::Result<()> { self.span_counters = None; let bytes = std::mem::take(&mut self.pending); @@ -1911,28 +1912,17 @@ impl Export { /// The clock packet for grid slot `index`, and record that the slot is served. /// - /// The value backs off by the largest reserve of any track, not just the PCR - /// track's: every rendition's PES must decode at or after the clock, and each - /// video track backs its DTS off by its own [`Reserve`]. A reserve that grows steps - /// the clock back with it, which only a new time base allows (ISO 13818-1 2.4.3.4), - /// so that PCR sets `discontinuity_indicator`. The DTS already sent stay ahead of - /// the clock values sent with them, and [`author_dts`] keeps every later DTS above - /// those. Back off through the 33-bit wrap rather than saturating: a timeline that - /// starts inside the reserve would otherwise clamp its first slots to zero and break - /// the uniform step. The wire field is a circular clock, so the masked wrapped value - /// is the correct mod-2^33 back-off. + /// The value runs one slot behind the boundary. A receiver times the bytes + /// between two clock packets by interpolating their values, and the bytes after + /// the last boundary in a span run on toward the next one, past the span's decode + /// time; the slot of slack keeps every byte ahead of the unit it belongs to. Back + /// off through the 33-bit wrap rather than saturating: a timeline that starts in + /// its first slot would otherwise clamp to zero and break the uniform step. The + /// wire field is a circular clock, so the masked wrapped value is the correct + /// mod-2^33 back-off. fn pcr_at(&mut self, index: u128, before: Option) -> anyhow::Result> { let pcr_pid = self.psi.as_ref().context("PSI not built")?.pcr_pid; - let reserve = self - .tracks - .values() - .map(|t| t.reserve.ticks) - .max() - .unwrap_or(DEFAULT_DTS_RESERVE); - let ticks = slot_ticks(index, PCR_INTERVAL).wrapping_sub(reserve); - if self.pcr_reserve.replace(reserve).is_some_and(|last| reserve > last) { - self.pcr_discontinuity = true; - } + let ticks = slot_ticks(index, PCR_INTERVAL).wrapping_sub(slot_ticks(1, PCR_INTERVAL)); // Nothing has gone out on this PID yet, so there is no counter to repeat and // any value starts a valid run; take the one before the next to be used. let cc = match before { diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 38b0969feb3..7037b6ca918 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -739,12 +739,11 @@ async fn export_pcr_is_a_uniform_ramp() { } } -/// A timeline that starts inside the decode-clock reserve backs the PCR off -/// through the 33-bit wrap instead of saturating at zero: the grid step stays -/// uniform from the very first slot (saturation would emit 0 then 2234, and a -/// large catalog jitter would freeze several leading PCRs at zero). +/// A timeline that starts in its first slot backs the PCR off through the 33-bit +/// wrap instead of saturating at zero: the grid step stays uniform from the very +/// first slot (saturation would emit 0 then 0 again). #[tokio::test(start_paused = true)] -async fn export_pcr_wraps_below_the_reserve_at_start() { +async fn export_pcr_wraps_below_zero_at_start() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); @@ -810,30 +809,29 @@ async fn export_pcr_wraps_below_the_reserve_at_start() { head = Some(base); } // The frame is paced at the slot its last leading clock asserts (plus the - // reserve), which is the newest slot that has begun by then. + // slot of slack), which is the newest slot that has begun by then. let Some(base) = head else { continue }; let slot_ticks = frame.timestamp.as_micros() / 25_000 * 25_000 * 90 / 1_000; assert_eq!( base, - (slot_ticks as u64).wrapping_sub(16) & WIRE, + (slot_ticks as u64).wrapping_sub(2250) & WIRE, "pacing off value, at {i}" ); } const WIRE: u64 = (1 << 33) - 1; assert!(pcrs.len() >= 2, "expected at least two grid slots, got {pcrs:?}"); - // Slot 0 minus the 16-tick default reserve, mod 2^33. - assert_eq!(pcrs[0], WIRE - 15, "slot 0 backs off through the wrap: {pcrs:?}"); + // Slot 0 minus one 2250-tick slot, mod 2^33. + assert_eq!(pcrs[0], WIRE - 2249, "slot 0 backs off through the wrap: {pcrs:?}"); // Every step is exactly one 25 ms slot (2250 ticks) in the circular clock. for (i, w) in pcrs.windows(2).enumerate() { assert_eq!(w[1].wrapping_sub(w[0]) & WIRE, 2250, "step off the grid at {i}: {w:?}"); } } -/// The clock backs off by the largest reserve of any track, not just the PCR -/// track's: a second rendition with a deeper reorder (catalog `jitter`) authors -/// its DTS further behind the PTS, and a clock respecting only the PCR track's -/// reserve would run ahead of those frames' decode times. +/// Every rendition decodes after the clock, not just the PCR track: a second +/// rendition with a deeper reorder (catalog `jitter`) authors its DTS further +/// behind the PTS, and the spans close on those decode times. #[tokio::test(start_paused = true)] async fn export_pcr_respects_every_renditions_reserve() { let mut broadcast = moq_net::broadcast::Info::new().produce(); @@ -4401,14 +4399,16 @@ async fn si_reordered_frames_earn_no_emission_credit() { assert_eq!(rig.media(2_500, 0x0011).await, 1, "the lead emission closes"); // The next revision arrives on a frame stepping back behind the 2.5s anchor, - // like a B-frame emitted in decode order: no elapsed time, no emission. + // like a B-frame emitted in decode order: no elapsed time, no emission. It still + // decodes after the frame before it, so it closes the span the prior revision + // rode, and its own span carries nothing. rig.si_track .write_frame(Timestamp::ZERO, Bytes::from(section(2))) .unwrap(); - assert_eq!(rig.media(2_400, 0x0011).await, 0, "a reordered frame earns no credit"); + assert_eq!(rig.media(2_400, 0x0011).await, 1, "only the prior revision closes"); // The floor measures from the 2.5s anchor: not due at 3.4s, due at 3.5s. - assert_eq!(rig.media(3_400, 0x0011).await, 1, "only the prior revision closes"); + assert_eq!(rig.media(3_400, 0x0011).await, 0, "a reordered frame earns no credit"); assert_eq!( rig.media(3_500, 0x0011).await, 0, @@ -4436,8 +4436,8 @@ async fn si_anchor_survives_a_zero_interval_reorder() { assert_eq!(rig.media(1_000, 0x0011).await, 0, "the first span stays buffered"); assert_eq!(rig.media(3_000, 0x0011).await, 1, "the anchor advances to 3s"); // A reordered frame steps back behind the anchor, into the previous 1s slot; - // zero interval still emits. - assert_eq!(rig.media(2_900, 0x0011).await, 0, "the reordered span stays open"); + // zero interval still emits. It decodes after the 3s frame, so it closes that span. + assert_eq!(rig.media(2_900, 0x0011).await, 1, "the 3s span closes"); // The catalog raises the interval to 1s. The grid slot must be the 3s anchor's, // not the reordered 2.9s emission's. @@ -4452,11 +4452,7 @@ async fn si_anchor_survives_a_zero_interval_reorder() { .get_mut(&0x42) .unwrap() .interval = Some(Duration::from_secs(1)); - assert_eq!( - rig.media(3_400, 0x0011).await, - 2, - "both zero-interval frames close together" - ); + assert_eq!(rig.media(3_400, 0x0011).await, 1, "the reordered span closes"); assert_eq!(rig.media(3_900, 0x0011).await, 0, "still in the anchor's slot"); assert_eq!(rig.media(4_000, 0x0011).await, 0, "the due table enters the open span"); assert_eq!(rig.media(4_100, 0x0011).await, 1, "the next span closes the due table"); @@ -5186,18 +5182,22 @@ impl Reordered { } /// Export a [`Reordered`] broadcast twice, like [`export_twice`]: the second exporter joins at -/// [`JOIN`], and both are drained after every tick. `jitter` is published at its tick when given. +/// [`JOIN`]. Ticks run on the paused clock as a live source's would, and both exporters release +/// on a fixed delay. `jitter` is published at its tick when given. async fn export_reordered( sps: &'static [u8], pps: &'static [u8], pyramid: &'static [u64], jitter: Option<(u64, Duration)>, ) -> (Vec, Vec) { + let delay = Duration::from_millis(500); let mut rig = Reordered::new(sps, pps, pyramid); - let mut a = Export::new(rig.source.clone()).await.unwrap(); + let mut a = rig.export(delay).await; + let start = tokio::time::Instant::now(); let mut b = None; let (mut out_a, mut out_b) = (Vec::new(), Vec::new()); for tick in 0..TICKS { + tokio::time::sleep_until(start + Duration::from_micros(tick * VIDEO_US)).await; if let Some((at, jitter)) = jitter && at == tick { @@ -5206,12 +5206,12 @@ async fn export_reordered( rig.video(tick); while rig.audio_before(tick + 1) {} - out_a.extend(drain_frames(&mut a).await); + out_a.extend(poll_frames(&mut a)); if let Some(b) = b.as_mut() { - out_b.extend(drain_frames(b).await); + out_b.extend(poll_frames(b)); } if tick + 1 == JOIN { - b = Some(Export::new(rig.source.clone()).await.unwrap()); + b = Some(Export::new(rig.source.clone()).await.unwrap().with_delay(delay)); } } rig.finish(); From 1cda12b02f898d7b2b65a549a04b9604d1611f61 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 18:36:06 -0700 Subject: [PATCH 05/40] refactor(moq-mux): rename the release stage to jitter::Buffer Co-Authored-By: Claude Opus 5.5 --- doc/bin/srt.md | 4 +- quest/m1/flv-export-delay.md | 6 +- quest/m1/mkv-export-delay.md | 6 +- rs/moq-mux/src/container/ts/export.rs | 78 ++++++------ rs/moq-mux/src/container/ts/export_test.rs | 12 +- rs/moq-mux/src/{release.rs => jitter.rs} | 133 +++++++++++---------- rs/moq-mux/src/lib.rs | 2 +- rs/moq-srt/src/ts.rs | 2 +- 8 files changed, 122 insertions(+), 121 deletions(-) rename rs/moq-mux/src/{release.rs => jitter.rs} (75%) diff --git a/doc/bin/srt.md b/doc/bin/srt.md index ae305c7bf31..08eea1d00f7 100644 --- a/doc/bin/srt.md +++ b/doc/bin/srt.md @@ -34,8 +34,8 @@ publishes each program as its own broadcast (`event.hang` becomes moq --connect https://relay.example.com/anon --broadcast event.hang import srt --listen '[::]:9000' --program all ``` -`--latency` sets the SRT receive buffer and doubles as the export's release -delay, as `export ts --delay` in the [CLI](/bin/cli): each frame is muxed that +`--latency` sets the SRT receive buffer and doubles as the export's jitter +buffer delay, as `export ts --delay` in the [CLI](/bin/cli): each frame is muxed that long after its decode time, and one arriving later is dropped. Export paces each SRT payload on the media clock, and re-anchors that pacing on a declared marker, so a restarted timeline plays out from the live edge instead of stalling until it catches up. A `--connect` URL needs a `streamid` query or a path; a listener diff --git a/quest/m1/flv-export-delay.md b/quest/m1/flv-export-delay.md index fbed3af14f5..044785b1039 100644 --- a/quest/m1/flv-export-delay.md +++ b/quest/m1/flv-export-delay.md @@ -3,15 +3,15 @@ ## Goal `moq export flv --delay ` interleaves tracks through the shared -fixed-delay release stage, so its tag order is a function of the media, not +jitter buffer, so its tag order is a function of the media, not of arrival, as the TS export's is. ## Plan Today `pick_next_track` (`rs/moq-mux/src/container/flv/export.rs`) takes the smallest *pending* timestamp, so which track goes first depends on which frame -has arrived. Replace it with the release stage the TS export uses -(`rs/moq-mux/src/release.rs`), with the same `--delay` flag and the same +has arrived. Replace it with the jitter buffer the TS export uses +(`rs/moq-mux/src/jitter.rs`), with the same `--delay` flag and the same late-frame drop. Update `doc/bin/cli.md`. Public API: the exporter gains a delay setting; breaking only if it replaces diff --git a/quest/m1/mkv-export-delay.md b/quest/m1/mkv-export-delay.md index 6024d9d4245..661a16e234b 100644 --- a/quest/m1/mkv-export-delay.md +++ b/quest/m1/mkv-export-delay.md @@ -3,15 +3,15 @@ ## Goal `moq export mkv --delay ` interleaves tracks through the shared -fixed-delay release stage, so its tag order is a function of the media, not +jitter buffer, so its tag order is a function of the media, not of arrival, as the TS export's is. ## Plan Today `pick_next_track` (`rs/moq-mux/src/container/mkv/export.rs`) takes the smallest *pending* timestamp, so which track goes first depends on which frame -has arrived. Replace it with the release stage the TS export uses -(`rs/moq-mux/src/release.rs`), with the same `--delay` flag and the same +has arrived. Replace it with the jitter buffer the TS export uses +(`rs/moq-mux/src/jitter.rs`), with the same `--delay` flag and the same late-frame drop. Update `doc/bin/cli.md`. Public API: the exporter gains a delay setting; breaking only if it replaces diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 33ba73efee8..90763765b63 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -43,7 +43,7 @@ use crate::catalog::{CatalogFormat, Stream}; use crate::codec::video::Reorder; use crate::codec::{aac, annexb}; use crate::container::{ExportSource, Frame}; -use crate::release::{Arrival, Push, Release, Released}; +use crate::jitter::{self, Arrival, Push}; use super::adts; use super::catalog; @@ -124,10 +124,10 @@ pub struct Export { /// How long after its decode time each frame goes out, and each source's staleness budget. delay: Duration, /// Holds every track's frames until `delay` past their decode time, keyed by PID. - release: Release, - /// A released frame waiting for the tail of the generation before it to go out. - held: Option>, - /// Release generation of the program being muxed; a newer one rewinds it. + jitter: jitter::Buffer, + /// A frame the jitter buffer let go, waiting for the tail of the generation before it to go out. + held: Option>, + /// Jitter-buffer generation of the program being muxed; a newer one rewinds it. generation: u64, tracks: HashMap, @@ -207,7 +207,7 @@ struct Pending { arrived: web_async::time::Instant, } -/// A frame waiting in the [`Release`] stage, with what it needs from the moment it was read. +/// A frame waiting in the jitter buffer, with what it needs from the moment it was read. struct Queued { frame: Frame, /// When the frame decodes: its DTS, else its PTS. @@ -223,7 +223,7 @@ struct Track { /// The first frame, held until the program tables are built. pending: Option, /// When the source was last found empty. A frame read since arrived no earlier, and - /// no later than it is read; the release judges it on the earlier bound, so a caller + /// no later than it is read; the jitter buffer judges it on the earlier bound, so a caller /// that polls late (a sink sleeping to pace its writes) does not make it late. empty: web_async::time::Instant, /// The source's discontinuity counter the decode clock (`last_dts`, `timeline`) runs under. @@ -246,12 +246,12 @@ struct Track { } impl Track { - /// Author the frame's decode time and queue it for release. + /// Author the frame's decode time and queue it in the jitter buffer. /// - /// The decode clock runs here, as frames are read, because the release deadline is + /// The decode clock runs here, as frames are read, because the deadline is /// on decode time: reordered video arrives in decode order with PTS 0, 120, 40, 80, /// so a PTS deadline would strand a B-frame behind its reference or send it first. - fn queue(&mut self, name: &str, pending: Pending, release: &mut Release) { + fn queue(&mut self, name: &str, pending: Pending, jitter: &mut jitter::Buffer) { let Pending { frame, discontinuity, @@ -299,8 +299,8 @@ impl Track { description, }, }; - if release.push(self.pid, arrival) == Push::Late { - tracing::warn!(track = %name, dropped = release.dropped(), "frame missed its release deadline; dropped"); + if jitter.push(self.pid, arrival) == Push::Late { + tracing::warn!(track = %name, dropped = jitter.dropped(), "frame missed its deadline; dropped"); } } } @@ -716,7 +716,7 @@ impl Export { catalog_format, stale: HashSet::new(), delay: Duration::ZERO, - release: Release::new(Duration::ZERO), + jitter: jitter::Buffer::new(Duration::ZERO), held: None, generation: 0, tracks: HashMap::new(), @@ -772,7 +772,7 @@ impl Export { /// Defaults to [`Duration::ZERO`]. pub fn with_delay(mut self, delay: Duration) -> Self { self.delay = delay; - self.release = Release::new(delay); + self.jitter = jitter::Buffer::new(delay); self } @@ -817,7 +817,7 @@ impl Export { si.poll(waiter); } - // 2. Read every frame the sources have: into the release stage, or before the + // 2. Read every frame the sources have: into the jitter buffer, or before the // program tables are built, one per track. self.fill(waiter)?; @@ -857,7 +857,7 @@ impl Export { } } } - // Hand the held frames to the release stage in arrival order, so the + // Hand the held frames to the jitter buffer in arrival order, so the // earliest anchors its clock, then read on. let mut held: Vec<(web_async::time::Instant, String)> = self .tracks @@ -868,12 +868,12 @@ impl Export { for (_, name) in held { let track = self.tracks.get_mut(&name).unwrap(); let pending = track.pending.take().unwrap(); - track.queue(&name, pending, &mut self.release); + track.queue(&name, pending, &mut self.jitter); } self.fill(waiter)?; } - // 4. Mux each frame the release stage lets go into the open span (the first + // 4. Mux each frame the jitter buffer lets go into the open span (the first // one carries the buffered PAT/PMT). Nothing goes out until a later // timestamp measures that span: only then is it known how many bytes it // carried, which is what puts the clock packets at the byte position their @@ -884,33 +884,33 @@ impl Export { self.emitted_epoch = self.epoch; return Poll::Ready(Ok(Some(out))); } - let released = match self.held.take() { - Some(released) => released, - None => match self.release.poll_next(waiter) { - Poll::Ready(released) => released, + let ready = match self.held.take() { + Some(ready) => ready, + None => match self.jitter.poll_next(waiter) { + Poll::Ready(ready) => ready, Poll::Pending => break, }, }; - if released.generation > self.generation { + if ready.generation > self.generation { if !self.pending.is_empty() { // A boundary ends valid media rather than reneging it. // Return that tail under the old generation before adopting the new one. self.emit(None)?; - self.held = Some(released); + self.held = Some(ready); continue; } self.rewind(); - self.generation = released.generation; + self.generation = ready.generation; } let name = self .tracks .iter() - .find(|(_, t)| t.pid == released.track) + .find(|(_, t)| t.pid == ready.track) .map(|(name, _)| name.clone()) - .context("released frame for an unknown PID")?; - self.last_timestamp = Some(released.item.frame.timestamp); - self.advance(released.item.decode)?; - self.mux(&name, released.item)?; + .context("frame for an unknown PID")?; + self.last_timestamp = Some(ready.item.frame.timestamp); + self.advance(ready.item.decode)?; + self.mux(&name, ready.item)?; } // 5. Once every track has drained, no later timestamp is coming to measure @@ -919,7 +919,7 @@ impl Export { // holding its tail until the catalog closed would strand it indefinitely. let drained = !self.tracks.is_empty() && self.tracks.values().all(|t| t.finished) - && self.release.is_empty() + && self.jitter.is_empty() && self.held.is_none(); if drained { self.emit(None)?; @@ -1013,7 +1013,7 @@ impl Export { track.pending = Some(pending); break; } - track.queue(name, pending, &mut self.release); + track.queue(name, pending, &mut self.jitter); } Poll::Ready(None) => { track.finished = true; @@ -1300,16 +1300,16 @@ impl Export { ); } - /// How many frames were dropped for missing their release deadline. + /// How many frames were dropped for missing their deadline. #[cfg(test)] pub(super) fn dropped(&self) -> u64 { - self.release.dropped() + self.jitter.dropped() } - /// When the next queued frame is due for release. + /// When the next queued frame is due. #[cfg(test)] - pub(super) fn next_release(&self) -> Option { - self.release.next_deadline() + pub(super) fn next_due(&self) -> Option { + self.jitter.next_deadline() } /// The discontinuity counter of the most recently returned output frame. @@ -1349,7 +1349,7 @@ impl Export { self.stale.insert(name.clone()); } // The replacement's clock starts afresh at its own first frame. - self.release.clear(); + self.jitter.clear(); self.held = None; self.generation = 0; self.rewind(); @@ -1709,7 +1709,7 @@ impl Export { /// `ts` is the decode time of the frame about to be muxed. Passing the watermark /// means the span that decode time opened is done: everything buffered since is /// exactly the bytes it carried, and the distance from it measures how long it ran. - /// The release stage hands frames over in decode order, so only a tie, or a frame + /// The jitter buffer hands frames over in decode order, so only a tie, or a frame /// read out of order under a zero delay, trails the watermark; it closes nothing, /// and its bytes join the open span. /// diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 7037b6ca918..d3ff512a30a 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -3568,16 +3568,16 @@ async fn drain_frames(export: &mut Export) -> Vec { let mut out = Vec::new(); loop { out.extend(poll_frames(export)); - // Run the paused clock only as far as the next release, so a test that writes + // Run the paused clock only as far as the next frame due, so a test that writes // more afterwards is still on time for it. - if let Some(deadline) = export.next_release() { + if let Some(deadline) = export.next_due() { tokio::time::sleep_until(deadline).await; continue; } match tokio::time::timeout(Duration::from_millis(10), export.next()).await { Ok(Ok(Some(frame))) => out.push(frame), Ok(Ok(None)) => break, - Err(_) if export.next_release().is_none() => break, + Err(_) if export.next_due().is_none() => break, Err(_) => {} Ok(Err(err)) => panic!("exporter error: {err}"), } @@ -5182,8 +5182,8 @@ impl Reordered { } /// Export a [`Reordered`] broadcast twice, like [`export_twice`]: the second exporter joins at -/// [`JOIN`]. Ticks run on the paused clock as a live source's would, and both exporters release -/// on a fixed delay. `jitter` is published at its tick when given. +/// [`JOIN`]. Ticks run on the paused clock as a live source's would, and both exporters hold +/// frames for a fixed delay. `jitter` is published at its tick when given. async fn export_reordered( sps: &'static [u8], pps: &'static [u8], @@ -5336,7 +5336,7 @@ async fn declared_reorder_sizes_the_decode_clock_from_the_first_frame() { } /// Reordered video (each pyramid in decode order: PTS 0, 160, 80, 40, 120 ms) loses nothing on -/// a clean path. The release deadline is on decode time, so no B-frame is stranded behind its +/// a clean path. The deadline is on decode time, so no B-frame is stranded behind its /// reference or sent ahead of it, and every track interleaves in decode order. #[tokio::test(start_paused = true)] async fn reordered_video_loses_nothing_on_a_clean_path() { diff --git a/rs/moq-mux/src/release.rs b/rs/moq-mux/src/jitter.rs similarity index 75% rename from rs/moq-mux/src/release.rs rename to rs/moq-mux/src/jitter.rs index 041d0834c7b..10d2ac53615 100644 --- a/rs/moq-mux/src/release.rs +++ b/rs/moq-mux/src/jitter.rs @@ -1,4 +1,5 @@ -//! Fixed-delay release of several tracks' frames in one decode order. +//! A jitter buffer that releases several tracks' frames in one decode order, each a fixed +//! delay after its decode time. use std::collections::{BTreeMap, VecDeque}; use std::pin::Pin; @@ -8,7 +9,7 @@ use std::time::Duration; use moq_net::Timestamp; use web_async::time::{Instant, Sleep}; -/// A frame handed to [`Release::push`]. +/// A frame handed to [`Buffer::push`]. pub(crate) struct Arrival { /// When the frame was read from its source. pub arrived: Instant, @@ -21,8 +22,8 @@ pub(crate) struct Arrival { pub item: T, } -/// A frame [`Release::poll_next`] let go. -pub(crate) struct Released { +/// A frame [`Buffer::poll_next`] let go. +pub(crate) struct Ready { pub track: K, /// Counts up each time a track's discontinuity started a new anchor. Tracks that /// cross the same discontinuity share one. @@ -30,7 +31,7 @@ pub(crate) struct Released { pub item: T, } -/// What [`Release::push`] did with a frame. +/// What [`Buffer::push`] did with a frame. #[derive(Debug, PartialEq, Eq)] pub(crate) enum Push { Queued, @@ -40,11 +41,11 @@ pub(crate) enum Push { Waiting, } -/// Releases each track's frames at a fixed delay past their decode time, like an SRT +/// Holds each track's frames until a fixed delay past their decode time, like an SRT /// receiver's TSBPD, in `(deadline, track)` order across tracks. /// /// The clock is anchored at the first frame's arrival. Within one anchor the deadline -/// order is the decode order, so two releases that saw the same frames arrive with +/// order is the decode order, so two buffers that saw the same frames arrive with /// different skew, each within its deadline, emit them in the same order. A frame that /// arrives past its deadline would break that order, so it is dropped and counted. /// @@ -56,7 +57,7 @@ pub(crate) enum Push { /// /// A zero delay holds nothing and drops nothing: each frame goes out as soon as it is /// read, ordered only among the frames read together. -pub(crate) struct Release { +pub(crate) struct Buffer { delay: Duration, /// Each generation's anchor: the first frame's arrival and decode time. anchors: BTreeMap, @@ -76,7 +77,7 @@ struct Track { waiting: bool, } -impl Release { +impl Buffer { pub fn new(delay: Duration) -> Self { Self { delay, @@ -88,7 +89,7 @@ impl Release { } } - /// Queue a frame for release, or drop it if it cannot make its deadline. + /// Queue a frame, or drop it if it cannot make its deadline. pub fn push(&mut self, key: K, arrival: Arrival) -> Push { // A new track counts from the start, as if it had always been there. let (mut generation, discontinuity) = self @@ -171,7 +172,7 @@ impl Release { } /// The next frame whose deadline has come, earliest deadline first, ties by track. - pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll> { + pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll> { let Some((deadline, key)) = self .tracks .iter() @@ -194,7 +195,7 @@ impl Release { } let track = key.clone(); let (_, generation, item) = self.tracks.get_mut(&track).and_then(|t| t.queue.pop_front()).unwrap(); - Poll::Ready(Released { + Poll::Ready(Ready { track, generation, item, @@ -210,7 +211,7 @@ impl Release { .min() } - /// Whether no frame is waiting for release. + /// Whether no frame is waiting. pub fn is_empty(&self) -> bool { self.tracks.values().all(|track| track.queue.is_empty()) } @@ -249,12 +250,12 @@ mod tests { } } - /// Every frame due by now, in release order. - fn due(release: &mut Release) -> Vec<&'static str> { + /// Every frame due by now, in order. + fn due(buffer: &mut Buffer) -> Vec<&'static str> { let waiter = kio::Waiter::noop(); let mut out = Vec::new(); - while let Poll::Ready(released) = release.poll_next(&waiter) { - out.push(released.item); + while let Poll::Ready(ready) = buffer.poll_next(&waiter) { + out.push(ready.item); } out } @@ -262,35 +263,35 @@ mod tests { #[tokio::test(start_paused = true)] async fn releases_at_the_delay_in_decode_order() { let start = Instant::now(); - let mut release = Release::new(DELAY); - assert_eq!(release.push(1, arrival(start, 0, "v0")), Push::Queued); - assert_eq!(release.push(1, arrival(start, 40, "v40")), Push::Queued); + let mut buffer = Buffer::new(DELAY); + assert_eq!(buffer.push(1, arrival(start, 0, "v0")), Push::Queued); + assert_eq!(buffer.push(1, arrival(start, 40, "v40")), Push::Queued); // Audio arrives later than the video, with an earlier decode time. assert_eq!( - release.push(2, arrival(start + Duration::from_millis(30), 20, "a20")), + buffer.push(2, arrival(start + Duration::from_millis(30), 20, "a20")), Push::Queued ); assert_eq!( - release.push(2, arrival(start + Duration::from_millis(30), 40, "a40")), + buffer.push(2, arrival(start + Duration::from_millis(30), 40, "a40")), Push::Queued ); - assert!(due(&mut release).is_empty(), "nothing is due before the delay"); + assert!(due(&mut buffer).is_empty(), "nothing is due before the delay"); tokio::time::advance(DELAY).await; - assert_eq!(due(&mut release), ["v0"]); + assert_eq!(due(&mut buffer), ["v0"]); tokio::time::advance(Duration::from_millis(20)).await; - assert_eq!(due(&mut release), ["a20"]); + assert_eq!(due(&mut buffer), ["a20"]); tokio::time::advance(Duration::from_millis(20)).await; - assert_eq!(due(&mut release), ["v40", "a40"], "a tie goes to the lower track"); - assert!(release.is_empty()); + assert_eq!(due(&mut buffer), ["v40", "a40"], "a tie goes to the lower track"); + assert!(buffer.is_empty()); } - /// Two releases fed the same frames with different arrival skew, every frame inside + /// Two buffers fed the same frames with different arrival skew, every frame inside /// its deadline, emit the same order. #[tokio::test(start_paused = true)] async fn arrival_skew_does_not_change_the_order() { let start = Instant::now(); - let (mut early, mut late) = (Release::new(DELAY), Release::new(DELAY)); + let (mut early, mut late) = (Buffer::new(DELAY), Buffer::new(DELAY)); let frames = [ (1, 0, "v0"), (1, 40, "v40"), @@ -320,22 +321,22 @@ mod tests { #[tokio::test(start_paused = true)] async fn late_frames_are_dropped_until_a_sync_frame() { let start = Instant::now(); - let mut release = Release::new(DELAY); - release.push(1, arrival(start, 0, "v0")); - release.push(2, arrival(start, 0, "a0")); + let mut buffer = Buffer::new(DELAY); + buffer.push(1, arrival(start, 0, "v0")); + buffer.push(2, arrival(start, 0, "a0")); let late = start + Duration::from_millis(200); let video = |decode, sync, item| Arrival { sync, ..arrival(late, decode, item) }; - assert_eq!(release.push(1, video(40, false, "v40")), Push::Late); - assert_eq!(release.push(1, video(200, false, "v200")), Push::Waiting); - assert_eq!(release.push(1, video(240, true, "v240")), Push::Queued); - assert_eq!(release.push(2, arrival(late, 180, "a180")), Push::Queued); - assert_eq!(release.dropped(), 2); + assert_eq!(buffer.push(1, video(40, false, "v40")), Push::Late); + assert_eq!(buffer.push(1, video(200, false, "v200")), Push::Waiting); + assert_eq!(buffer.push(1, video(240, true, "v240")), Push::Queued); + assert_eq!(buffer.push(2, arrival(late, 180, "a180")), Push::Queued); + assert_eq!(buffer.dropped(), 2); tokio::time::advance(Duration::from_secs(1)).await; - assert_eq!(due(&mut release), ["v0", "a0", "a180", "v240"]); + assert_eq!(due(&mut buffer), ["v0", "a0", "a180", "v240"]); } /// A discontinuity re-anchors the track, and a track reaching the same discontinuity @@ -343,9 +344,9 @@ mod tests { #[tokio::test(start_paused = true)] async fn a_discontinuity_re_anchors() { let start = Instant::now(); - let mut release = Release::new(DELAY); - release.push(1, arrival(start, 5_000, "v5000")); - release.push(2, arrival(start, 5_000, "a5000")); + let mut buffer = Buffer::new(DELAY); + buffer.push(1, arrival(start, 5_000, "v5000")); + buffer.push(2, arrival(start, 5_000, "a5000")); // The publisher restarts its timeline at zero, a second in. let restart = start + Duration::from_secs(1); @@ -353,34 +354,34 @@ mod tests { discontinuity: 1, ..arrival(arrived, decode, item) }; - assert_eq!(release.push(1, reset(restart, 0, "v0")), Push::Queued); + assert_eq!(buffer.push(1, reset(restart, 0, "v0")), Push::Queued); // Audio still carries its old timeline for a moment, then restarts too. - assert_eq!(release.push(2, arrival(restart, 6_000, "a6000")), Push::Queued); + assert_eq!(buffer.push(2, arrival(restart, 6_000, "a6000")), Push::Queued); let later = restart + Duration::from_millis(10); - assert_eq!(release.push(2, reset(later, 0, "a0")), Push::Queued); + assert_eq!(buffer.push(2, reset(later, 0, "a0")), Push::Queued); tokio::time::advance(Duration::from_secs(1) + DELAY).await; - assert_eq!(due(&mut release), ["v5000", "a5000", "v0", "a6000", "a0"]); + assert_eq!(due(&mut buffer), ["v5000", "a5000", "v0", "a6000", "a0"]); } /// A backlog read in one go releases a new generation after the one before it. #[tokio::test(start_paused = true)] async fn a_backlog_releases_generations_in_turn() { let start = Instant::now(); - let mut release = Release::new(DELAY); - release.push(1, arrival(start, 5_000, "v5000")); - release.push(1, arrival(start, 5_040, "v5040")); + let mut buffer = Buffer::new(DELAY); + buffer.push(1, arrival(start, 5_000, "v5000")); + buffer.push(1, arrival(start, 5_040, "v5040")); let reset = Arrival { discontinuity: 1, ..arrival(start, 0, "v0") }; - release.push(1, reset); - release.push(2, arrival(start, 5_020, "a5020")); + buffer.push(1, reset); + buffer.push(2, arrival(start, 5_020, "a5020")); tokio::time::advance(DELAY).await; - assert_eq!(due(&mut release), ["v5000"]); + assert_eq!(due(&mut buffer), ["v5000"]); tokio::time::advance(Duration::from_millis(40)).await; - assert_eq!(due(&mut release), ["a5020", "v5040", "v0"]); + assert_eq!(due(&mut buffer), ["a5020", "v5040", "v0"]); } /// Zero holds nothing: frames read together go out at once in decode order, and a @@ -388,30 +389,30 @@ mod tests { #[tokio::test(start_paused = true)] async fn zero_delay_releases_on_arrival() { let start = Instant::now(); - let mut release = Release::new(Duration::ZERO); - release.push(1, arrival(start, 0, "v0")); - release.push(1, arrival(start, 40, "v40")); - release.push(2, arrival(start, 20, "a20")); - assert_eq!(due(&mut release), ["v0", "a20", "v40"]); + let mut buffer = Buffer::new(Duration::ZERO); + buffer.push(1, arrival(start, 0, "v0")); + buffer.push(1, arrival(start, 40, "v40")); + buffer.push(2, arrival(start, 20, "a20")); + assert_eq!(due(&mut buffer), ["v0", "a20", "v40"]); let late = start + Duration::from_secs(1); - assert_eq!(release.push(2, arrival(late, 40, "a40")), Push::Queued); - assert_eq!(due(&mut release), ["a40"]); - assert_eq!(release.dropped(), 0); + assert_eq!(buffer.push(2, arrival(late, 40, "a40")), Push::Queued); + assert_eq!(due(&mut buffer), ["a40"]); + assert_eq!(buffer.dropped(), 0); } #[tokio::test(start_paused = true)] async fn clear_starts_a_fresh_clock() { let start = Instant::now(); - let mut release = Release::new(DELAY); - release.push(1, arrival(start, 5_000, "old")); - release.clear(); - assert!(release.is_empty()); + let mut buffer = Buffer::new(DELAY); + buffer.push(1, arrival(start, 5_000, "old")); + buffer.clear(); + assert!(buffer.is_empty()); tokio::time::advance(Duration::from_secs(1)).await; let now = Instant::now(); - assert_eq!(release.push(1, arrival(now, 0, "new")), Push::Queued); + assert_eq!(buffer.push(1, arrival(now, 0, "new")), Push::Queued); tokio::time::advance(DELAY).await; - assert_eq!(due(&mut release), ["new"]); + assert_eq!(due(&mut buffer), ["new"]); } } diff --git a/rs/moq-mux/src/lib.rs b/rs/moq-mux/src/lib.rs index 087eb6f93e9..ad184c5efa6 100644 --- a/rs/moq-mux/src/lib.rs +++ b/rs/moq-mux/src/lib.rs @@ -36,7 +36,7 @@ pub mod import; pub mod json; mod pace; pub mod rate; -mod release; +mod jitter; pub mod select; mod source; pub mod timeline; diff --git a/rs/moq-srt/src/ts.rs b/rs/moq-srt/src/ts.rs index 186656b3011..0accf1853d6 100644 --- a/rs/moq-srt/src/ts.rs +++ b/rs/moq-srt/src/ts.rs @@ -168,7 +168,7 @@ pub struct Subscriber { impl Subscriber { /// Resolve the broadcast at `path` in the origin and prepare to mux it to TS. /// - /// `latency` is the muxer's release delay: each frame is muxed that long after + /// `latency` is the muxer's jitter-buffer delay: each frame is muxed that long after /// its decode time, a frame arriving later is dropped, and a stalled group is /// skipped after it. We reuse the locally configured SRT receive latency for /// it, the same budget an SRT hop gives a packet. It's the configured value, From a11aea3d4d136b4f67bd02eb3d5124641f5b1af7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 18:39:06 -0700 Subject: [PATCH 06/40] fix(moq-mux): space reordered DTS a frame period apart The decode clock hands presentation times out in display order a reorder depth late, like ffmpeg's pts_buffer, instead of nudging each B-frame one tick past its reference. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export.rs | 145 ++++++++++++++------- rs/moq-mux/src/container/ts/export_test.rs | 9 +- 2 files changed, 107 insertions(+), 47 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 90763765b63..1b6c8760c19 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -226,7 +226,7 @@ struct Track { /// no later than it is read; the jitter buffer judges it on the earlier bound, so a caller /// that polls late (a sink sleeping to pace its writes) does not make it late. empty: web_async::time::Instant, - /// The source's discontinuity counter the decode clock (`last_dts`, `timeline`) runs under. + /// The source's discontinuity counter the decode clock (`clock`, `timeline`) runs under. discontinuity: u64, finished: bool, pid: u16, @@ -234,9 +234,8 @@ struct Track { /// PMT ES-level descriptors to re-announce, captured verbatim on import (language, /// registration, ...). Empty for non-TS sources; AC-3/E-AC-3 then synthesize one. descriptors: Vec, - /// Last decode timestamp (continuous 90 kHz ticks) authored for this track, keeping the - /// decode clock monotonic across reordered (B-frame) video. Only video uses it. - last_dts: Option, + /// Authors each video frame's DTS from its PTS. + clock: DecodeClock, /// High-water mark of the timestamps read within this rendition, which sizes the /// video [`Reserve`] from how far a reordered frame lands below it. timeline: Option, @@ -260,7 +259,7 @@ impl Track { if discontinuity != self.discontinuity { // The source may have restarted its timeline, so the decode clock restarts too. self.discontinuity = discontinuity; - self.last_dts = None; + self.clock = DecodeClock::default(); self.timeline = None; } @@ -277,8 +276,9 @@ impl Track { } else { self.reserve.peak(); } - let dts = author_dts(to_ticks(frame.timestamp), self.reserve.ticks, &mut self.last_dts); - (dts, frame.keyframe, self.source.description().cloned()) + let pts = to_ticks(frame.timestamp); + let dts = self.clock.author(pts, self.reserve.ticks, self.reserve.period()); + ((dts != pts).then_some(dts), frame.keyframe, self.source.description().cloned()) } _ => (None, true, None), }; @@ -391,14 +391,21 @@ impl Reserve { self.since_peak = 0; } + /// The picture period in ticks the SPS or the catalog `framerate` declares. + fn period(&self) -> Option { + match self.declared?.period { + Some((units, scale)) => Some(units.saturating_mul(90_000).div_ceil(scale)), + None => Some((90_000.0 / self.framerate?).ceil() as u64), + } + .filter(|&period| period > 0) + } + fn settle(&mut self, name: &str, source: &'static str) { // As in [`Self::observe`], a reordered frame needs a tick for each frame decoded between // it and the one above it; the declared depth stands in for that count. + let period = self.period(); let declared = self.declared.and_then(|reorder| { - let period = match reorder.period { - Some((units, scale)) => units.saturating_mul(90_000).div_ceil(scale), - None => (90_000.0 / self.framerate?).ceil() as u64, - }; + let period = period?; let depth = u64::from(reorder.depth); let ticks = depth.saturating_mul(period).saturating_add(depth.max(1)); Some(ticks.min(MAX_DTS_RESERVE)) @@ -1293,7 +1300,7 @@ impl Export { pid, kind, descriptors, - last_dts: None, + clock: DecodeClock::default(), timeline: None, reserve, }, @@ -1344,7 +1351,7 @@ impl Export { track.finished = true; track.pending = None; track.discontinuity = 0; - track.last_dts = None; + track.clock = DecodeClock::default(); track.timeline = None; self.stale.insert(name.clone()); } @@ -2423,33 +2430,65 @@ fn ensure_raw(container: &Container, kind: &str, name: &str) -> anyhow::Result<( } } -/// Author a monotonic decode timestamp (DTS) for a reordered (B-frame) video frame. +/// A video rendition's decode clock: a decode timestamp (DTS) for each frame from its PTS. /// -/// [`Frame`] carries only a presentation timestamp (PTS) and frames reach the muxer in -/// decode order (MoQ groups and frames are delivered in decode order), so a B-frame stream -/// arrives with valid but non-monotonic PTS and no decode time. MPEG-TS players need a -/// monotonic DTS to schedule decoding; without it they choke on the out-of-order PTS (the -/// `ffplay -fflags +igndts` workaround). +/// [`Frame`] carries only a presentation timestamp and frames reach the muxer in decode order, +/// so a B-frame stream arrives with non-monotonic PTS and no decode time. The clock hands out +/// the presentation times in display order, `depth` frames late, the way a decoder's reorder +/// buffer of that depth would (ffmpeg's `pts_buffer`): with depth 2, decode-order PTS 0, 120, +/// 40, 80 decode at -80, -40, 0, 40, one frame period apart. `depth` is how many frame periods +/// fit in the track's [`Reserve`]; the remainder of the reserve backs every DTS off a little +/// further. So a reference frame decodes a whole reserve before it is presented and a B-frame +/// keeps its own slot rather than bunching one tick behind its reference. /// -/// Since decode order is already the delivery order, the only job is to keep DTS strictly -/// increasing. The clock runs `reserve` ticks behind the PTS and never goes backwards: a -/// reordered frame whose PTS dips below the clock is nudged one tick past the last DTS. A -/// reserve larger than the reordering keeps `DTS <= PTS`; the [`DEFAULT_DTS_RESERVE`] fallback -/// keeps DTS monotonic but lets it sit above a B-frame's own PTS. -/// -/// `reserve` is the track's [`Reserve`]. `pts` and `last` are continuous (unwrapped) 90 kHz -/// ticks, so the clock never wraps mid-stream; the 33-bit wire wrap happens once at emission. -/// `last` is the previous DTS, updated in place. Returns `None` when the DTS equals the PTS -/// (PES stays PTS-only). -fn author_dts(pts: u64, reserve: u64, last: &mut Option) -> Option { - let mut dts = pts.saturating_sub(reserve); - if let Some(prev) = *last - && dts <= prev - { - dts = prev + 1; - } - *last = Some(dts); - (dts != pts).then_some(dts) +/// The period is the one the stream declares, else the smallest step seen between consecutive +/// PTS. Without either the clock runs a reserve behind the PTS. A DTS that would not advance +/// (reordering deeper than the reserve) is nudged one tick past the last, so the clock stays +/// strictly increasing either way. Ticks are continuous (unwrapped) 90 kHz, so the clock never +/// wraps mid-stream; the 33-bit wire wrap happens once at emission. +#[derive(Default)] +struct DecodeClock { + /// Presentation times read but not yet handed out, ascending. + window: Vec, + /// The last DTS handed out. + last: Option, + /// The previous frame's PTS, to measure the frame period. + previous: Option, + /// The smallest positive step seen between consecutive PTS. + period: Option, +} + +impl DecodeClock { + /// The DTS of a frame presented at `pts`, the next in decode order. + fn author(&mut self, pts: u64, reserve: u64, declared: Option) -> u64 { + if let Some(step) = self.previous.map(|previous| pts.abs_diff(previous)).filter(|&step| step > 0) { + self.period = Some(self.period.map_or(step, |period| period.min(step))); + } + self.previous = Some(pts); + + let dts = match declared.or(self.period) { + Some(period) => { + let depth = (reserve / period) as usize; + let at = self.window.partition_point(|&p| p < pts); + self.window.insert(at, pts); + // Until the window holds `depth + 1` frames, the earliest is still to come, so + // count back from the earliest seen as a decoder that ran all along would have. + let held = self.window.len(); + let earliest = match held > depth { + true => self.window.drain(..held - depth).last().unwrap_or(pts), + false => self.window[0].saturating_sub((depth + 1 - held) as u64 * period), + }; + earliest.saturating_sub(reserve % period) + } + None => pts.saturating_sub(reserve), + }; + let dts = match self.last { + Some(last) if dts <= last => last + 1, + _ => dts, + }; + self.last = Some(dts); + dts + } } /// The reordering declared by the first SPS in a video rendition's avcC/hvcC. @@ -2472,20 +2511,19 @@ mod tests { use moq_net::Timestamp; use super::{ - DEFAULT_DTS_RESERVE, PCR_INTERVAL, PSI_INTERVAL, author_dts, due, is_complete_section, si_due, slot, slot_ticks, + DEFAULT_DTS_RESERVE, DecodeClock, PCR_INTERVAL, PSI_INTERVAL, due, is_complete_section, si_due, slot, + slot_ticks, }; fn ms(value: u64) -> Timestamp { Timestamp::from_millis(value).unwrap() } - /// Push a decode-order PTS stream (90 kHz) through the decode clock with a given reserve and - /// return the effective DTS per frame (the authored DTS, or the PTS when none is authored). + /// Push a decode-order PTS stream (90 kHz, 25 fps declared) through the decode clock with a + /// given reserve and return the DTS per frame. fn run_clock(pts: &[u64], reserve: u64) -> Vec { - let mut last = None; - pts.iter() - .map(|&p| author_dts(p, reserve, &mut last).unwrap_or(p)) - .collect() + let mut clock = DecodeClock::default(); + pts.iter().map(|&p| clock.author(p, reserve, Some(3_600))).collect() } /// Decode-order PTS for a constant-frame-rate display timeline with `b` B-frames between @@ -2573,6 +2611,23 @@ mod tests { } } + /// B-frames decode a frame period apart, not one tick behind their reference. + #[test] + fn reordered_dts_is_frame_spaced() { + let dur = 3_600; + for b in [1, 2, 3] { + let reserve = b as u64 * dur + 1; + let pts = decode_order(40, b, dur, 10_000_000); + let dts = run_clock(&pts, reserve); + for (i, win) in dts.windows(2).enumerate() { + assert_eq!(win[1] - win[0], dur, "b={b}: DTS not one frame apart at {i}: {dts:?}"); + } + for (i, (&d, &p)) in dts.iter().zip(pts.iter()).enumerate() { + assert!(d <= p, "b={b}: DTS {d} after PTS {p} at {i}"); + } + } + } + #[test] fn author_dts_is_join_independent_at_a_peak() { // An exporter that has been running and one that just joined author the same decode diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index d3ff512a30a..37242f79e0d 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -5320,7 +5320,8 @@ async fn declared_reorder_sizes_the_decode_clock_from_the_first_frame() { let timing = video_timing(&out_late, ..); assert_decodes_before_presenting(&timing); - // Depth 1 at 25 fps, and one tick so the B-frame decodes after the P-frame above it. + // Depth 1 at 25 fps, plus one tick of reserve: every frame decodes one period after the + // last, the keyframe a whole reserve before it is presented. let (pts, dts) = timing[1]; assert_eq!( pts, @@ -5329,9 +5330,13 @@ async fn declared_reorder_sizes_the_decode_clock_from_the_first_frame() { ); assert_eq!( dts, - Some(pts - FRAME_TICKS - 1), + Some(REORDERED_BASE * FRAME_TICKS - 1), "the first P-frame already runs on the declared reserve" ); + let decode: Vec = timing.iter().map(|&(pts, dts)| dts.unwrap_or(pts)).collect(); + for (i, step) in decode.windows(2).map(|w| w[1] - w[0]).enumerate() { + assert_eq!(step, FRAME_TICKS, "frame {i} does not decode one period after the last: {decode:?}"); + } assert_decodes_after_the_clock(&out_late); } From c38c1339e3126c5ef99056f62cce77b2e8ef062d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 19:08:57 -0700 Subject: [PATCH 07/40] feat(moq-mux): lay TS export packets on a constant-rate schedule A new ts::Schedule packs each access unit onto the PCR grid. With a multiplex rate every slot carries what the rate allows, padded with nulls, and a unit goes out as late as the rate lets it while still arriving by its DTS, spreading a keyframe over the slots before it, back as far as the delay. A burst that does not fit fails the export. Within a slot each PID's packets spread among the others and the nulls. Continuity counters are numbered as packets go out, which deletes the span counters, counter_before, and the stuffing balance. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export.rs | 470 +++++---------------- rs/moq-mux/src/container/ts/export_test.rs | 140 +++--- rs/moq-mux/src/container/ts/import_test.rs | 9 +- rs/moq-mux/src/container/ts/mod.rs | 1 + rs/moq-mux/src/container/ts/schedule.rs | 403 ++++++++++++++++++ 5 files changed, 590 insertions(+), 433 deletions(-) create mode 100644 rs/moq-mux/src/container/ts/schedule.rs diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 1b6c8760c19..7f571b6bfa6 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -3,7 +3,7 @@ //! [`Export`] subscribes to a MoQ broadcast and produces MPEG-TS: PAT/PMT program //! tables and PES packets, packetized into 188-byte TS packets, with the PCR //! riding its own adaptation-field-only packets on a fixed media-time grid. -//! Output is sliced on that grid rather than per media frame ([`Export::emit`]): +//! Output is sliced on that grid rather than per media frame ([`Schedule`]): //! each [`Frame`] is one slot's clock packet plus the bytes belonging to it, //! stamped at the slot boundary, so the clock a receiver recovers from byte //! position agrees with the values, and a pacing caller releases each slot at @@ -46,6 +46,7 @@ use crate::container::{ExportSource, Frame}; use crate::jitter::{self, Arrival, Push}; use super::adts; +use super::schedule::{self, Schedule}; use super::catalog; /// PID of the single program's PMT. @@ -61,16 +62,9 @@ const PSI_INTERVAL: Duration = Duration::from_millis(500); /// enforced on the media timeline. Clamped to the entry's own interval, so a /// table asking for faster repetition than this still gets it. const SI_REVISION_INTERVAL: Duration = Duration::from_secs(1); -/// Emit a PCR on every crossing of this media-time grid ([`Export::emit`]). +/// Emit a PCR on every crossing of this media-time grid ([`Schedule`]). /// TR 101 290 flags a gap over 40 ms; broadcast muxes emit every 25-40 ms. pub(super) const PCR_INTERVAL: Duration = Duration::from_millis(25); -/// How many missed PCR slots to backfill at most: one second's worth. Frames -/// coarser than the grid cross several slots at a time and every one is filled so -/// the ramp stays uniform, down to a 1 fps cadence. Past this cap the media -/// didn't have a coarse cadence, it had an outage, and reconstructing a dense -/// clock history for a span that carried no bytes only stalls anything pacing on -/// the asserted values. -const PCR_BACKFILL: u128 = 40; /// A null packet: PID 0x1FFF, payload only, all stuffing. Its continuity counter /// is don't-care (ISO 13818-1), so one template serves every one. pub(super) const NULL_PACKET: [u8; TsPacket::SIZE] = { @@ -81,18 +75,6 @@ pub(super) const NULL_PACKET: [u8; TsPacket::SIZE] = { packet[3] = 0x10; packet }; -/// Grid slots per second, so the multiplex rate in bits per second is also the -/// per-slot allowance in [`STUFFING_UNIT`]s. -const SLOTS_PER_SECOND: i64 = (Duration::from_secs(1).as_nanos() / PCR_INTERVAL.as_nanos()) as i64; -const _: () = assert!( - Duration::from_secs(1) - .as_nanos() - .is_multiple_of(PCR_INTERVAL.as_nanos()) -); -/// Fixed-point unit of the stuffing balance: one packet is this many units, so that -/// one slot at `mux_rate` bits per second is exactly `mux_rate` units and no slot -/// rounds on its own. The remainder carries across slots instead. -const STUFFING_UNIT: i64 = TsPacket::SIZE as i64 * 8 * SLOTS_PER_SECOND; /// Upper bound on an accepted multiplex rate, in bits per second: far above any /// broadcast contribution multiplex, while bounding one slot's null allocation /// to a few megabytes. Zero and anything past it are refused where they enter, @@ -131,10 +113,8 @@ pub struct Export { generation: u64, tracks: HashMap, - /// Continuity counter per PID (PAT, PMT, and each elementary stream). - counters: HashMap, - /// Counter state before the uncommitted span, restored if a rewind discards it. - span_counters: Option>, + /// The next continuity counter per PID, numbered as packets go out. + counters: HashMap, /// PMT program-level descriptors captured on import, re-emitted in the PMT. program_descriptors: Vec, /// Transport/service identity captured on import, used to rebuild a consistent @@ -154,37 +134,20 @@ pub struct Export { psi: Option, /// Media timestamp of the last PAT/PMT emission ([`due`]). last_psi: Option, - /// Grid slot of the last PCR emission ([`Self::emit`]). - last_pcr: Option, /// Program generation being muxed; source counters are local to each rendition. epoch: u64, /// Generation of the last returned frame, updated only at the output boundary. emitted_epoch: u64, pcr_discontinuity: bool, - /// TS packets muxed into the span that is still open. - pending: Vec, - /// Offsets into [`pending`](Self::pending) where a keyframe's packets begin, so - /// the output frame carrying one keeps the flag. - keyframes: Vec, - /// Output frames ready to hand out, one per grid slot the last span covered. + /// Lays the muxed packets onto the PCR grid. + schedule: Schedule, + /// Output frames ready to hand out, one per grid slot. queue: VecDeque, - /// Continuity counter of the last packet emitted on the PCR PID. A clock packet - /// carries no payload, so it repeats whatever preceded it on the wire rather - /// than advancing the counter ([`Export::pcr_at`]). - pcr_cc: Option, - /// Media time the next span's bytes are transmitted from ([`Export::emit`]). - clock: Option, - /// Earliest decode time muxed into the open span: what its bytes have to arrive - /// before, so it bounds how far the clock may run. - low: Option, - /// Decode time that opened the current span, which closes on a later one. - watermark: Option, /// The rate to pad the output to with null packets, in bits per second: the /// builder override when set, else the catalog's recorded multiplex rate, else - /// none and the output is unpadded ([`Self::stuff`]). + /// none and the output is unpadded ([`Schedule`]). mux_rate: Option, mux_rate_override: Option, - stuffing: Stuffing, /// Tune-in point: the first video keyframe's timestamp, captured when the program /// tables are built. Non-video frames before it are dropped so the keyframe leads /// the stream. @@ -461,18 +424,6 @@ enum Kind { }, } -/// The null stuffing owed to the multiplex rate ([`Export::stuff`]). -#[derive(Default)] -struct Stuffing { - /// Packets the rate has allowed minus packets sent, in [`STUFFING_UNIT`]s. - /// Negative while the media alone exceeds the rate. - balance: i64, - /// Packets sent since the last clock packet, that packet included. - since_pcr: u64, - /// Whether the debt cap has been hit and reported in the current overrun. - overrun: bool, -} - /// The program tables plus the resolved PID layout. struct Psi { pat: Pat, @@ -728,7 +679,6 @@ impl Export { generation: 0, tracks: HashMap::new(), counters: HashMap::new(), - span_counters: None, program_descriptors: Vec::new(), program: None, si: BTreeMap::new(), @@ -736,21 +686,14 @@ impl Export { si_flushed: false, psi: None, last_psi: None, - last_pcr: None, epoch: 0, emitted_epoch: 0, pcr_discontinuity: false, - pending: Vec::new(), - keyframes: Vec::new(), + schedule: Schedule::new(Duration::ZERO), queue: VecDeque::new(), - pcr_cc: None, - clock: None, - low: None, - watermark: None, video_start: None, mux_rate: None, mux_rate_override: None, - stuffing: Stuffing::default(), }) } @@ -766,6 +709,7 @@ impl Export { } self.mux_rate_override = Some(mux_rate); self.mux_rate = sanitize_mux_rate(mux_rate); + self.schedule.set_rate(self.mux_rate); self } @@ -776,10 +720,17 @@ impl Export { /// between them. A frame that arrives after its deadline is dropped, and a video /// track that dropped one resumes at its next keyframe. The delay is also each /// source's staleness budget (see [`Consumer`](crate::container::Consumer)). - /// Defaults to [`Duration::ZERO`]. + /// + /// With a multiplex rate it is also how far ahead of its decode time a frame may go + /// out: a keyframe bigger than one PCR interval at the rate spreads over the intervals + /// before it, back as far as the delay, so the output trails the source by up to twice + /// the delay. A frame that does not fit there fails the export. Defaults to + /// [`Duration::ZERO`], which pads no further ahead than one interval. pub fn with_delay(mut self, delay: Duration) -> Self { self.delay = delay; self.jitter = jitter::Buffer::new(delay); + self.schedule = Schedule::new(delay); + self.schedule.set_rate(self.mux_rate); self } @@ -880,12 +831,10 @@ impl Export { self.fill(waiter)?; } - // 4. Mux each frame the jitter buffer lets go into the open span (the first - // one carries the buffered PAT/PMT). Nothing goes out until a later - // timestamp measures that span: only then is it known how many bytes it - // carried, which is what puts the clock packets at the byte position their - // own values imply and lets the caller's pacer release each at the instant - // it asserts. See [`Self::advance`]. + // 4. Mux each frame the jitter buffer lets go (the first carries the buffered + // PAT/PMT) and lay out every grid slot it settles. A slot is settled once every + // frame that could ride it has been muxed: the jitter buffer hands frames over in + // decode order, so a frame settles every slot whose window ends before its own. loop { if let Some(out) = self.queue.pop_front() { self.emitted_epoch = self.epoch; @@ -899,10 +848,10 @@ impl Export { }, }; if ready.generation > self.generation { - if !self.pending.is_empty() { + if !self.schedule.is_empty() { // A boundary ends valid media rather than reneging it. // Return that tail under the old generation before adopting the new one. - self.emit(None)?; + self.lay(None)?; self.held = Some(ready); continue; } @@ -916,20 +865,21 @@ impl Export { .map(|(name, _)| name.clone()) .context("frame for an unknown PID")?; self.last_timestamp = Some(ready.item.frame.timestamp); - self.advance(ready.item.decode)?; + let decode = ready.item.decode.as_nanos(); self.mux(&name, ready.item)?; + self.lay(Some(schedule::slot(decode)))?; } - // 5. Once every track has drained, no later timestamp is coming to measure - // the open span, so its bytes go out whole. That's independent of the - // catalog: a retained track finishes while the broadcast stays live, and - // holding its tail until the catalog closed would strand it indefinitely. + // 5. Once every track has drained, nothing more can ride the slots still open, so + // they go out. That's independent of the catalog: a retained track finishes while + // the broadcast stays live, and holding its tail until the catalog closed would + // strand it indefinitely. let drained = !self.tracks.is_empty() && self.tracks.values().all(|t| t.finished) && self.jitter.is_empty() && self.held.is_none(); if drained { - self.emit(None)?; + self.lay(None)?; if let Some(out) = self.queue.pop_front() { self.emitted_epoch = self.epoch; return Poll::Ready(Ok(Some(out))); @@ -975,6 +925,7 @@ impl Export { self.write_section(&mut out, pid, section)?; } } + self.number(&mut out); Ok(Some(Frame { timestamp, duration: None, @@ -1056,10 +1007,8 @@ impl Export { }) }), }; - if self.mux_rate != mux_rate { - self.mux_rate = mux_rate; - self.stuffing = Stuffing::default(); - } + self.mux_rate = mux_rate; + self.schedule.set_rate(mux_rate); // Reconcile the SI subscriptions with the catalog's map. Entries may appear // after the PAT/PMT is built (a table acquired late): they ride standalone @@ -1363,28 +1312,20 @@ impl Export { Ok(()) } - /// Discard uncommitted bytes and restart the program clock. Every rendition + /// Discard what has not gone out and restart the program clock. Every rendition /// joins the new generation: no track is fenced across a declared marker, since a - /// latency skip on one track says nothing about another's timeline. + /// latency skip on one track says nothing about another's timeline. The continuity + /// counters run on, since they number only what went out. fn rewind(&mut self) { self.epoch += 1; - if let Some(counters) = self.span_counters.take() { - self.counters = counters; - } - self.pending.clear(); - self.keyframes.clear(); + self.schedule.clear(); self.queue.clear(); - self.watermark = None; - self.clock = None; - self.low = None; - self.last_pcr = None; self.last_psi = None; for si in self.si.values_mut() { si.last_emit = None; } self.video_start = None; self.pcr_discontinuity = true; - self.stuffing = Stuffing::default(); } /// Header is ready when every track's [`ExportSource`] has resolved its @@ -1567,12 +1508,8 @@ impl Export { Ok(()) } - /// Packetize one media frame into the open span, re-emitting PAT/PMT before - /// video keyframes (and periodically) so receivers can tune in mid-stream. - /// - /// The bytes are buffered rather than returned: which grid slots they belong - /// to isn't known until a later timestamp measures the span (see - /// [`Self::advance`]). + /// Packetize one media frame and queue it on the [`Schedule`], re-emitting PAT/PMT + /// before video keyframes (and periodically) so receivers can tune in mid-stream. fn mux(&mut self, name: &str, queued: Queued) -> anyhow::Result<()> { let Queued { frame, @@ -1580,9 +1517,6 @@ impl Export { dts, description, } = queued; - if self.span_counters.is_none() { - self.span_counters = Some(self.counters.clone()); - } let track = self.tracks.get_mut(name).context("missing track")?; let pid = track.pid; let kind = track.kind.clone(); @@ -1703,241 +1637,73 @@ impl Export { self.write_pes(&mut out, &unit, &es_payload)?; } } - if keyframe { - self.keyframes.push(self.pending.len()); - } - self.pending.extend_from_slice(&out); - self.low = Some(self.low.map_or(decode, |low| low.min(decode))); + self.schedule.push(decode.as_nanos(), out, keyframe); Ok(()) } - /// Close the open span if `ts` passes the watermark, laying its bytes out. + /// Lay out every grid slot the schedule has settled, one [`Frame`] each: `known` is + /// the first slot a frame still to be muxed could be due in, or `None` when nothing + /// more is coming. /// - /// `ts` is the decode time of the frame about to be muxed. Passing the watermark - /// means the span that decode time opened is done: everything buffered since is - /// exactly the bytes it carried, and the distance from it measures how long it ran. - /// The jitter buffer hands frames over in decode order, so only a tie, or a frame - /// read out of order under a zero delay, trails the watermark; it closes nothing, - /// and its bytes join the open span. + /// Each frame opens with its slot's clock packet, so the packet count between + /// consecutive PCRs is what the interval between their values carries, and a consumer + /// holding only the byte stream (which is every MPEG-TS tool) recovers the same clock + /// from byte position that the values assert. Each frame is stamped at its slot + /// boundary, so a pacing caller releases the clock at the instant it asserts. /// - /// This is why nothing goes out on arrival, and why it can't. A span's bytes - /// have to reach a receiver before the units in it decode, so they ride the - /// grid slots *preceding* the span, which are only known once the span has - /// closed. That is the mux buffer: the exporter runs two spans behind the media - /// clock, by a constant amount, so a caller pacing on the stamps still releases - /// every slot at the interval its own PCR value asserts. - fn advance(&mut self, ts: Timestamp) -> anyhow::Result<()> { - let Some(watermark) = self.watermark else { - self.watermark = Some(ts); - return Ok(()); - }; - if ts <= watermark { + /// The PES units cannot carry the clock themselves: a PCR sampled from the DTS would + /// step with the frame cadence, and no downstream CBR stage can repair that, because a + /// groomer can only place the clock samples it receives. So the PCR asserts its own + /// uniform ramp instead: absolute grid slots on the media timeline, shared by every + /// exporter of the broadcast, like [`due`]. + fn lay(&mut self, known: Option) -> anyhow::Result<()> { + let Some(psi) = self.psi.as_ref() else { return Ok(()); - } - self.watermark = Some(ts); - let span = ts.as_nanos().saturating_sub(watermark.as_nanos()); - self.emit(Some(span)) - } - - /// Lay the open span's bytes across the grid slots that run up to its decode - /// time, one [`Frame`] per slot. `span` is how long the span ran, or `None` at - /// end of stream, where nothing measured it. - /// - /// Each frame opens with the clock packets whose slot boundary it starts at, - /// then carries the share of the bytes its slice of the interval earns. Three - /// things follow, and they are the whole point of muxing this way. The packet - /// count between consecutive PCRs is proportional to the difference between - /// their values, so a consumer holding only the byte stream (which is every - /// MPEG-TS tool) recovers the same clock from byte position that the values - /// assert. Each frame is stamped at its own slot boundary, so a pacing caller - /// releases the clock at the instant it asserts rather than when the media that - /// revealed it arrived. And the interval ends at the span's earliest media - /// decode time, so every byte precedes the decode time of the unit it belongs to. - /// - /// The PES units cannot carry the clock themselves: a PCR sampled from the DTS - /// would step with the frame cadence, and no downstream CBR stage can repair - /// that, because a groomer can only place the clock samples it receives. So the - /// PCR asserts its own uniform ramp instead: absolute grid slots on the media - /// timeline (shared by every exporter of the broadcast, like [`due`]). - fn emit(&mut self, span: Option) -> anyhow::Result<()> { - self.span_counters = None; - let bytes = std::mem::take(&mut self.pending); - let keyframes = std::mem::take(&mut self.keyframes); - let Some(to) = self.low.take() else { return Ok(()) }; - let packets = bytes.len() / TsPacket::SIZE; - - // Transmit from where the last span stopped up to this one's decode time, so - // the intervals abut and the clock neither repeats nor reverses. The first - // has nothing to abut, so it takes the span's own measured length. - // - // The clock only ever moves forward. A track skewed far enough behind that it - // decodes before the clock already reached it can't be placed ahead of its own - // decode time, and [`Self::with_delay`] owns that skew, so its bytes go out at the - // clock rather than dragging it backwards. - let start = match self.clock { - Some(clock) => clock.as_nanos(), - None => to.as_nanos().saturating_sub(span.unwrap_or(0)), }; - let end = to.as_nanos().max(start); - let from = stamp(start)?; - self.clock = Some(stamp(end)?); - - // The slot `start` sits in; its boundary is at or before every byte here. - let open = start / PCR_INTERVAL.as_nanos(); - // The last boundary strictly inside the interval: `end` opens the next one's. - let last = slot_before(end, PCR_INTERVAL).max(open); - // Slots still owed a clock packet. Backfill every missed one so the ramp - // stays uniform when frames are coarser than the grid, but cap it: past the - // cap the media itself gapped, and a dense clock history for a span that - // carried no bytes helps nobody. - let first = self - .last_pcr - .map_or(open, |l| l + 1) - .max((last + 1).saturating_sub(PCR_BACKFILL)); - // Spread the bytes only across an interval the grid can describe. Past the - // cap they were muxed before a media gap, so they belong at its start rather - // than smeared across silence. - let spread = last - open < PCR_BACKFILL; - let width = end - start; - - // The first frame opens at `from` rather than a boundary, and carries every - // clock packet whose slot has already begun. - let pcr_pid = self.psi.as_ref().context("PSI not built")?.pcr_pid; - let mut payload = Vec::new(); - for index in first..=open { - self.stuff(index, &mut payload); - let before = counter_before(&bytes, 0, pcr_pid, self.pcr_cc); - let clock = self.pcr_at(index, before)?; - self.send(&mut payload, &clock); - } - let mut cut = 0; - let mut at = from; - - // One frame per boundary inside the interval. `first` can only exceed - // `open + 1` when the backfill cap skipped the slots below it, and that cap - // bounds this range to [`PCR_BACKFILL`] iterations however long the gap was. - for index in (open + 1).max(first)..=last { - let boundary = slot_stamp(index)?; - let next = if spread && width > 0 { - (packets as u128 * (boundary.as_nanos() - start) / width) as usize - } else { - packets - }; - self.send(&mut payload, &bytes[cut * TsPacket::SIZE..next * TsPacket::SIZE]); - self.stuff(index, &mut payload); - self.push(at, payload, &keyframes, cut, next); - cut = next; - at = boundary; - let before = counter_before(&bytes, cut * TsPacket::SIZE, pcr_pid, self.pcr_cc); - let clock = self.pcr_at(index, before)?; - payload = Vec::new(); - self.send(&mut payload, &clock); - } - - self.send(&mut payload, &bytes[cut * TsPacket::SIZE..]); - self.push(at, payload, &keyframes, cut, packets); - if let Some(cc) = counter_before(&bytes, bytes.len(), pcr_pid, None) { - self.pcr_cc = Some(cc); + let (pcr_pid, pmt_pid) = (psi.pcr_pid, psi.pmt_pid); + while let Some(slot) = self.schedule.next(known)? { + let mut payload = self.pcr_at(slot.index, pcr_pid)?; + payload.extend_from_slice(&slot.layout(pmt_pid, &NULL_PACKET)); + self.number(&mut payload); + self.queue.push_back(Frame { + timestamp: slot_stamp(slot.index)?, + duration: None, + payload: Bytes::from(payload), + keyframe: slot.keyframe, + }); } Ok(()) } - /// Append packets to the open output frame, counting them against the rate. - fn send(&mut self, payload: &mut Vec, packets: &[u8]) { - payload.extend_from_slice(packets); - self.stuffing.since_pcr += (packets.len() / TsPacket::SIZE) as u64; - } - - /// Settle the interval that closes at grid slot `index`, the one the clock packet - /// about to be written for it ends, appending the null packets that bring it up - /// to the multiplex rate. Nothing without a rate. - /// - /// The rate credits each slot its exact fractional allowance and every packet - /// sent debits one, so the remainder carries across slots and the long-run count - /// is exact. Media is never delayed or dropped to fit: a slot that already - /// exceeds its allowance gets no nulls and carries the debt forward, and a source - /// that sustains more than the rate simply overruns it. The debt is capped at one - /// second of packets and reported once per overrun, or the rate would never - /// recover after a long burst. Credit is capped the same way, matching the clock - /// backfill: past it the media gapped, and a second of stuffing marks that without - /// filling the whole gap. - fn stuff(&mut self, index: u128, payload: &mut Vec) { - let since = std::mem::take(&mut self.stuffing.since_pcr); - let (Some(rate), Some(last)) = (self.mux_rate, self.last_pcr) else { - return; - }; - // The rate is sanitized where it enters (the builder and the catalog), so - // this holds; refuse to pad rather than wrap if it ever does not. - let Ok(rate) = i64::try_from(rate) else { - return; - }; - let slots = index.saturating_sub(last).min(PCR_BACKFILL) as i64; - let stuffing = &mut self.stuffing; - stuffing.balance = stuffing - .balance - .saturating_add(slots.saturating_mul(rate)) - .saturating_sub((since as i64).saturating_mul(STUFFING_UNIT)); - let floor = rate.saturating_mul(-SLOTS_PER_SECOND); - if stuffing.balance < floor { - if !std::mem::replace(&mut stuffing.overrun, true) { - tracing::warn!(mux_rate = rate, "MPEG-TS output exceeds the multiplex rate"); + /// Number the continuity counters of packets about to go out. A packet without a + /// payload (a clock packet) repeats the counter before it (ISO 13818-1 2.4.3.3); before + /// anything has gone out on its PID, any value starts a valid run. + fn number(&mut self, packets: &mut [u8]) { + for packet in packets.chunks_exact_mut(TsPacket::SIZE) { + let pid = u16::from(packet[1] & 0x1f) << 8 | u16::from(packet[2]); + if pid == 0x1FFF { + continue; } - stuffing.balance = floor; - } else if stuffing.balance >= 0 { - stuffing.overrun = false; - } - let nulls = (stuffing.balance / STUFFING_UNIT).max(0); - // Never emit more than the slots just credited could allow: a bound on one - // call's allocation whatever the balance holds. A no-op on a rate that came - // through [`sanitize_mux_rate`], whose balance carries less than one packet. - // (A manual `div_ceil`: all terms are non-negative, and the toolchain's - // signed `div_ceil` is still unstable.) - let ceiling = slots.saturating_mul(rate).saturating_add(STUFFING_UNIT - 1) / STUFFING_UNIT; - let nulls = nulls.min(ceiling); - stuffing.balance -= nulls * STUFFING_UNIT; - payload.reserve(nulls as usize * TsPacket::SIZE); - for _ in 0..nulls { - payload.extend_from_slice(&NULL_PACKET); - } - } - - /// Queue one output frame, unless it would be empty. `from`..`to` are the packet - /// indices it carries, which decide whether a keyframe begins in it. - fn push(&mut self, timestamp: Timestamp, payload: Vec, keyframes: &[usize], from: usize, to: usize) { - if payload.is_empty() { - return; + let next = self.counters.entry(pid).or_default(); + let cc = match packet[3] & 0x10 != 0 { + true => std::mem::replace(next, (*next + 1) & ContinuityCounter::MAX), + false => next.wrapping_sub(1) & ContinuityCounter::MAX, + }; + packet[3] = (packet[3] & 0xf0) | cc; } - let (from, to) = (from * TsPacket::SIZE, to * TsPacket::SIZE); - let keyframe = keyframes.iter().any(|&at| at >= from && at < to); - self.queue.push_back(Frame { - timestamp, - duration: None, - payload: Bytes::from(payload), - keyframe, - }); } - /// The clock packet for grid slot `index`, and record that the slot is served. + /// The clock packet for grid slot `index`. /// - /// The value runs one slot behind the boundary. A receiver times the bytes - /// between two clock packets by interpolating their values, and the bytes after - /// the last boundary in a span run on toward the next one, past the span's decode - /// time; the slot of slack keeps every byte ahead of the unit it belongs to. Back - /// off through the 33-bit wrap rather than saturating: a timeline that starts in - /// its first slot would otherwise clamp to zero and break the uniform step. The - /// wire field is a circular clock, so the masked wrapped value is the correct - /// mod-2^33 back-off. - fn pcr_at(&mut self, index: u128, before: Option) -> anyhow::Result> { - let pcr_pid = self.psi.as_ref().context("PSI not built")?.pcr_pid; + /// The value runs one slot behind the boundary, so the bytes of slot `index` are timed + /// up to its boundary: a unit due in that slot decodes no earlier ([`schedule::slot`]). + /// Back off through the 33-bit wrap rather than saturating: a timeline that starts in + /// its first slot would otherwise clamp to zero and break the uniform step. The wire + /// field is a circular clock, so the masked wrapped value is the correct mod-2^33 + /// back-off. + fn pcr_at(&mut self, index: u128, pcr_pid: u16) -> anyhow::Result> { let ticks = slot_ticks(index, PCR_INTERVAL).wrapping_sub(slot_ticks(1, PCR_INTERVAL)); - // Nothing has gone out on this PID yet, so there is no counter to repeat and - // any value starts a valid run; take the one before the next to be used. - let cc = match before { - Some(cc) => cc, - None => self.counters.entry(pcr_pid).or_default().as_u8().wrapping_sub(1), - }; - self.last_pcr = Some(index); - let mut packet = pcr_packet(pcr_pid, ticks, cc)?; + let mut packet = pcr_packet(pcr_pid, ticks)?; if std::mem::take(&mut self.pcr_discontinuity) { packet[5] |= 0x80; } @@ -2068,7 +1834,8 @@ impl Export { Ok(()) } - /// Serialize one TS packet (with its continuity counter) into `out`. + /// Serialize one TS packet into `out`. Its continuity counter is numbered when it goes + /// out ([`Self::number`]), since the schedule decides that order. fn write_packet( &mut self, out: &mut Vec, @@ -2076,9 +1843,7 @@ impl Export { adaptation_field: Option, payload: TsPayload, ) -> anyhow::Result<()> { - let counter = self.counters.entry(pid).or_default(); - let continuity_counter = *counter; - counter.increment(); + let continuity_counter = ContinuityCounter::default(); let packet = TsPacket { header: TsHeader { @@ -2103,22 +1868,17 @@ impl Export { /// between PCR base and extension as zeros where ISO 13818-1 requires ones, and /// strict analyzers flag that. /// -/// There is no payload, so the field's stuffing fills the packet and the continuity -/// counter is not incremented (ISO 13818-1 2.4.3.3): `cc` is the counter of -/// whatever preceded this packet on the same PID, which it repeats. The clock rides -/// a PID that also carries media, and the packets around it were numbered when they -/// were muxed rather than when they go out, so this has to come from the wire order -/// rather than from the counter's current value. -fn pcr_packet(pid: u16, ticks: u64, cc: u8) -> anyhow::Result> { +/// There is no payload, so the field's stuffing fills the packet; its continuity counter +/// is numbered when it goes out ([`Export::number`]). +fn pcr_packet(pid: u16, ticks: u64) -> anyhow::Result> { anyhow::ensure!(pid <= 0x1FFF, "PID out of range: {pid}"); - let cc = cc & ContinuityCounter::MAX; let base = ticks & TS_TIMESTAMP_MASK; let mut p = Vec::with_capacity(TsPacket::SIZE); p.push(0x47); p.push((pid >> 8) as u8); p.push(pid as u8); // adaptation_field_control = adaptation field only, no scrambling. - p.push(0x20 | cc); + p.push(0x20); // adaptation_field_length covers the rest of the packet. p.push(183); // PCR_flag alone. @@ -2135,28 +1895,6 @@ fn pcr_packet(pid: u16, ticks: u64, cc: u8) -> anyhow::Result> { Ok(p) } -/// The continuity counter a payload-less packet inserted at byte offset `cut` has to -/// repeat: the last one before it on `pid`, else the one carried in from the last -/// span, else one behind the first packet that follows it, which is what keeps the -/// run continuous where the clock leads the stream and nothing precedes it. -fn counter_before(bytes: &[u8], cut: usize, pid: u16, carried: Option) -> Option { - let on_pid = |p: &[u8]| (u16::from(p[1] & 0x1f) << 8 | u16::from(p[2])) == pid; - let counter = |p: &[u8]| p[3] & ContinuityCounter::MAX; - bytes[..cut] - .rchunks_exact(TsPacket::SIZE) - .find(|p| on_pid(p)) - .map(counter) - .or(carried) - .or_else(|| { - bytes[cut..] - .as_chunks::<{ TsPacket::SIZE }>() - .0 - .iter() - .find(|p| on_pid(p.as_slice())) - .map(|p| counter(p).wrapping_sub(1) & ContinuityCounter::MAX) - }) -} - /// Optional PES header region carrying PTS only: 2 flag bytes + 1 length byte + 5 PTS bytes. const PES_OPTIONAL_LEN: usize = 3 + 5; /// Extra bytes when the optional region also carries a DTS (5 DTS bytes). @@ -2226,18 +1964,6 @@ fn slot(timestamp: Timestamp, interval: Duration) -> u128 { Duration::from(timestamp).as_nanos() / interval.as_nanos() } -/// Index of the last repetition slot to *begin* strictly before `nanos`. -/// -/// [`slot`] rounds down, so a position sitting exactly on a boundary belongs to the slot that -/// boundary opens. When the position is the exclusive end of an interval, that slot is the -/// next interval's, not this one's, hence the offset. -fn slot_before(nanos: u128, interval: Duration) -> u128 { - if nanos == 0 { - return 0; - } - (nanos - 1) / interval.as_nanos() -} - /// A repetition slot's boundary as a media timestamp. fn slot_stamp(index: u128) -> anyhow::Result { stamp(index * PCR_INTERVAL.as_micros() * 1_000) diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 37242f79e0d..a474050cfa4 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -69,9 +69,12 @@ fn length_prefixed(nals: &[&[u8]]) -> Bytes { /// exporter's default [`Duration::ZERO`] collapses to the /// live edge: completeness has to be asked for, exactly as a real recorder does. const RECORDING_MAX_AGE: Duration = Duration::from_secs(30); -/// How long a drain waits for the next frame: past the recording delay, so the first -/// frame goes out, and then until output stops. -const DRAIN: std::time::Duration = RECORDING_MAX_AGE.saturating_add(std::time::Duration::from_secs(1)); +/// How long a drain waits for the next frame: past the recording delay, the mux-ahead +/// window a multiplex rate adds on top of it, and the clip, so the first frame goes out, +/// and then until output stops. +const DRAIN: std::time::Duration = RECORDING_MAX_AGE + .saturating_mul(3) + .saturating_add(std::time::Duration::from_secs(1)); async fn drain(consumer: moq_net::broadcast::Consumer) -> BytesMut { drain_with(Export::new(crate::source::announced(&consumer)).await.unwrap()).await @@ -2842,6 +2845,10 @@ async fn discontinuity_flags_the_break_once_across_tracks() { ); assert_eq!(count_discontinuity(&after), 1, "the break is flagged exactly once"); let bytes: Vec<_> = after.iter().flat_map(|f| f.payload.iter().copied()).collect(); + // The old generation's last frame spreads up to its decode time, so its tail leads + // `after`; read from the new generation's tables. + let tables = bytes.chunks(188).position(|p| p[1] & 0x1f == 0 && p[2] == 0).unwrap() * 188; + let bytes = bytes[tables..].to_vec(); let (video_pts, audio_pts) = collect_pes_pts(&bytes); assert!(!video_pts.is_empty(), "video resumed"); assert!(!audio_pts.is_empty(), "audio resumed"); @@ -3642,8 +3649,8 @@ async fn late_join_matches_a_running_exporter() { async fn late_join_matches_a_running_exporter_without_video() { let (a, b) = export_twice(false).await; - // The joiner's first span leads with PAT/PMT, while the running exporter has no - // reason to repeat them there. The span can cover several PCR slices, so compare + // The joiner's first frame leads with PAT/PMT, while the running exporter has no + // reason to repeat them there. It can spread over several PCR slices, so compare // from the next PAT/PMT refresh, which both exporters anchor to the media grid. let from = b .iter() @@ -3811,6 +3818,17 @@ fn pes_decode_in_order(frames: &[Frame]) -> Vec { out } +/// Whether every PES start decodes no earlier than those ahead of it, up to one grid slot: +/// a slot spreads each PID's packets among the others, so units sharing one may swap. +fn in_decode_order(decode: &[u64]) -> bool { + let slot = PCR_INTERVAL.as_micros() as u64 * 90 / 1_000; + let mut high = 0; + decode.iter().all(|&d| { + high = high.max(d); + d + slot >= high + }) +} + /// Audio lagging video by up to a frame, on a live clock: each video frame lands before /// the audio that precedes it. Returns the decode order the exporter emitted. async fn lagging_audio(delay: Duration) -> Vec { @@ -3834,7 +3852,7 @@ async fn late_audio_still_leads_the_interleave() { let decode = lagging_audio(Duration::from_millis(500)).await; assert!(decode.len() > 100, "too little output to judge: {}", decode.len()); assert!( - decode.is_sorted(), + in_decode_order(&decode), "a frame went out ahead of an earlier one: {decode:?}" ); } @@ -3844,7 +3862,7 @@ async fn late_audio_still_leads_the_interleave() { async fn zero_delay_keeps_arrival_order() { let decode = lagging_audio(Duration::ZERO).await; assert!( - !decode.is_sorted(), + !in_decode_order(&decode), "video should have led the audio that arrived after it" ); } @@ -3904,7 +3922,7 @@ async fn a_late_frame_is_dropped_and_the_rest_keep_their_order() { "only the late frames were dropped" ); assert!( - decode.is_sorted(), + in_decode_order(&decode), "a frame went out ahead of an earlier one: {decode:?}" ); } @@ -3921,7 +3939,7 @@ async fn tune_in_does_not_wait_on_dropped_audio() { rig.video(tick); } out.extend(poll_frames(&mut export)); - // Past the keyframe's deadline, and far enough on that a later frame closes its span. + // Past the keyframe's deadline, and far enough on that a later frame settles its slot. tokio::time::advance(delay + Duration::from_micros(4 * VIDEO_US)).await; out.extend(poll_frames(&mut export)); assert!(!out.is_empty(), "the tune-in waited on audio it had dropped"); @@ -4299,19 +4317,20 @@ async fn si_revision_does_not_wait_for_the_interval() { let mut rig = si_cadence_rig(0x0014, 0x70, Duration::from_secs(30)).await; rig.si_track.write_frame(Timestamp::ZERO, Bytes::from(tick(1))).unwrap(); - assert_eq!(rig.media(0, 0x0014).await, 0, "the first span stays buffered"); + assert_eq!(rig.media(0, 0x0014).await, 0, "the first frame stays buffered"); assert_eq!( rig.media(1_000, 0x0014).await, 1, - "the next span closes the lead emission" + "the next frame lets the lead emission out" ); assert_eq!(rig.media(2_000, 0x0014).await, 0, "unchanged: still held"); // The clock ticks. Nothing about the 30 s interval has elapsed, but the value - // changed: it must go out with the very next frame. + // changed: it must go out with the very next frame, which spreads from the slot + // after the frame before it. rig.si_track.write_frame(Timestamp::ZERO, Bytes::from(tick(2))).unwrap(); - assert_eq!(rig.media(3_000, 0x0014).await, 0, "the revision enters the open span"); - assert_eq!(rig.media(4_000, 0x0014).await, 1, "the next span closes the revision"); + assert_eq!(rig.media(3_000, 0x0014).await, 1, "the revision rides the next frame"); + assert_eq!(rig.media(4_000, 0x0014).await, 0, "and only that one"); } /// #3948: an unchanged snapshot repeats on the absolute media-time grid, like the @@ -4325,32 +4344,32 @@ async fn si_repeats_ride_the_media_grid() { // The lead emission lands wherever this exporter joined. rig.si_track.write_frame(Timestamp::ZERO, Bytes::from(sdt_v1)).unwrap(); - assert_eq!(rig.media(1_500, 0x0011).await, 0, "the first span stays buffered"); + assert_eq!(rig.media(1_500, 0x0011).await, 0, "the first frame stays buffered"); assert_eq!( rig.media(1_900, 0x0011).await, 1, - "the next span closes the lead emission" + "the next frame lets the lead emission out" ); // The 2s boundary is 0.5s after the lead: the grid, not a floor from the join. - assert_eq!(rig.media(2_000, 0x0011).await, 0, "the repeat enters the open span"); - assert_eq!(rig.media(2_500, 0x0011).await, 1, "the next span closes the repeat"); + assert_eq!(rig.media(2_000, 0x0011).await, 1, "the repeat rides the 2s frame"); + assert_eq!(rig.media(2_500, 0x0011).await, 0, "and only that one"); // A revision lands mid-slot: it waits only for the 1s revision floor. rig.si_track.write_frame(Timestamp::ZERO, Bytes::from(sdt_v2)).unwrap(); assert_eq!(rig.media(2_900, 0x0011).await, 0, "within the floor of the repeat"); assert_eq!( rig.media(3_000, 0x0011).await, - 0, - "the revision enters the span once its floor lapses" + 1, + "the revision rides the frame where its floor lapses" ); - assert_eq!(rig.media(3_500, 0x0011).await, 1, "the next span closes the revision"); + assert_eq!(rig.media(3_500, 0x0011).await, 0, "and only that one"); // Repeats stay on the grid after it. - assert_eq!(rig.media(4_000, 0x0011).await, 0, "the 4s repeat enters the open span"); - assert_eq!(rig.media(5_000, 0x0011).await, 1, "the next span closes it"); + assert_eq!(rig.media(4_000, 0x0011).await, 1, "the 4s repeat"); + assert_eq!(rig.media(5_000, 0x0011).await, 0, "nothing until the next grid slot"); assert_eq!(rig.media(5_900, 0x0011).await, 0, "nothing between grid slots"); - assert_eq!(rig.media(6_000, 0x0011).await, 0, "the 6s repeat enters the open span"); - assert_eq!(rig.media(6_100, 0x0011).await, 1, "the next span closes it"); + assert_eq!(rig.media(6_000, 0x0011).await, 1, "the 6s repeat"); + assert_eq!(rig.media(6_100, 0x0011).await, 0, "and only that one"); } /// A clock table (TDT/TOT) never repeats an unchanged snapshot: a repeat re-asserts @@ -4362,11 +4381,11 @@ async fn si_clock_tables_do_not_repeat_unchanged() { let mut rig = si_cadence_rig(0x0014, 0x70, Duration::from_secs(2)).await; rig.si_track.write_frame(Timestamp::ZERO, Bytes::from(tick(1))).unwrap(); - assert_eq!(rig.media(0, 0x0014).await, 0, "the first span stays buffered"); + assert_eq!(rig.media(0, 0x0014).await, 0, "the first frame stays buffered"); assert_eq!( rig.media(1_000, 0x0014).await, 1, - "the next span closes the lead emission" + "the next frame lets the lead emission out" ); let mut repeats = 0; for millis in (2_000..=9_000).step_by(1_000) { @@ -4375,8 +4394,8 @@ async fn si_clock_tables_do_not_repeat_unchanged() { assert_eq!(repeats, 0, "an unchanged time is never re-sent"); rig.si_track.write_frame(Timestamp::ZERO, Bytes::from(tick(2))).unwrap(); - assert_eq!(rig.media(10_000, 0x0014).await, 0, "the new time enters the open span"); - assert_eq!(rig.media(11_000, 0x0014).await, 1, "the next span closes it"); + assert_eq!(rig.media(10_000, 0x0014).await, 1, "the new time rides the next frame"); + assert_eq!(rig.media(11_000, 0x0014).await, 0, "and only that one"); } /// A reordered (B-frame) timestamp below the emission anchor earns no credit: @@ -4391,30 +4410,32 @@ async fn si_reordered_frames_earn_no_emission_credit() { rig.si_track .write_frame(Timestamp::ZERO, Bytes::from(section(0))) .unwrap(); - assert_eq!(rig.media(1_000, 0x0011).await, 0, "the first span stays buffered"); + assert_eq!(rig.media(1_000, 0x0011).await, 0, "the first frame stays buffered"); rig.si_track .write_frame(Timestamp::ZERO, Bytes::from(section(1))) .unwrap(); - assert_eq!(rig.media(2_500, 0x0011).await, 1, "the lead emission closes"); + assert_eq!( + rig.media(2_500, 0x0011).await, + 2, + "the lead emission and the revision riding the 2.5s frame" + ); // The next revision arrives on a frame stepping back behind the 2.5s anchor, - // like a B-frame emitted in decode order: no elapsed time, no emission. It still - // decodes after the frame before it, so it closes the span the prior revision - // rode, and its own span carries nothing. + // like a B-frame emitted in decode order: no elapsed time, no emission. rig.si_track .write_frame(Timestamp::ZERO, Bytes::from(section(2))) .unwrap(); - assert_eq!(rig.media(2_400, 0x0011).await, 1, "only the prior revision closes"); + assert_eq!(rig.media(2_400, 0x0011).await, 0, "a reordered frame earns no credit"); // The floor measures from the 2.5s anchor: not due at 3.4s, due at 3.5s. - assert_eq!(rig.media(3_400, 0x0011).await, 0, "a reordered frame earns no credit"); + assert_eq!(rig.media(3_400, 0x0011).await, 0, "not due within the floor"); assert_eq!( rig.media(3_500, 0x0011).await, - 0, - "the deferred revision enters the span at the floor" + 1, + "the deferred revision rides the frame at the floor" ); - assert_eq!(rig.media(3_600, 0x0011).await, 1, "the next span closes the revision"); + assert_eq!(rig.media(3_600, 0x0011).await, 0, "and only that one"); assert_eq!(rig.exporter.discontinuity(), 0, "B-frame reordering is not a rewind"); } @@ -4433,11 +4454,16 @@ async fn si_anchor_survives_a_zero_interval_reorder() { rig.si_track .write_frame(Timestamp::ZERO, Bytes::from(section(0))) .unwrap(); - assert_eq!(rig.media(1_000, 0x0011).await, 0, "the first span stays buffered"); - assert_eq!(rig.media(3_000, 0x0011).await, 1, "the anchor advances to 3s"); + assert_eq!(rig.media(1_000, 0x0011).await, 0, "the first frame stays buffered"); + assert_eq!( + rig.media(3_000, 0x0011).await, + 2, + "the 1s emission, and the anchor advancing to 3s" + ); // A reordered frame steps back behind the anchor, into the previous 1s slot; - // zero interval still emits. It decodes after the 3s frame, so it closes that span. - assert_eq!(rig.media(2_900, 0x0011).await, 1, "the 3s span closes"); + // zero interval still emits. It decodes just after the 3s frame, so it shares that + // frame's last slot and waits with it. + assert_eq!(rig.media(2_900, 0x0011).await, 0, "the reordered frame waits"); // The catalog raises the interval to 1s. The grid slot must be the 3s anchor's, // not the reordered 2.9s emission's. @@ -4452,10 +4478,12 @@ async fn si_anchor_survives_a_zero_interval_reorder() { .get_mut(&0x42) .unwrap() .interval = Some(Duration::from_secs(1)); - assert_eq!(rig.media(3_400, 0x0011).await, 1, "the reordered span closes"); - assert_eq!(rig.media(3_900, 0x0011).await, 0, "still in the anchor's slot"); - assert_eq!(rig.media(4_000, 0x0011).await, 0, "the due table enters the open span"); - assert_eq!(rig.media(4_100, 0x0011).await, 1, "the next span closes the due table"); + // The decode clock catches up on the reorder over the next frames, so count the + // emissions across them rather than per frame. + let held = rig.media(3_400, 0x0011).await + rig.media(3_900, 0x0011).await; + assert_eq!(held, 1, "only the reordered emission, still in the anchor's slot"); + let due = rig.media(4_000, 0x0011).await + rig.media(4_100, 0x0011).await + rig.media(4_200, 0x0011).await; + assert_eq!(due, 1, "the 4s table, once"); assert_eq!(rig.exporter.discontinuity(), 0, "B-frame reordering is not a rewind"); } @@ -4476,10 +4504,9 @@ async fn si_rapid_revisions_are_rate_bounded() { let frames = rig.media_frames(u64::from(i) * 250).await; let count = count_pid(&frames, 0x0011); emitted += count; - if i == 5 { - // The 1s floor lapses here; the emission carries the newest revision, - // not the revisions that arrived after it while its span was buffered. - assert_eq!(count, 1, "the next span closes the 1s emission"); + if i == 4 { + // The 1s floor lapses here, and the emission carries the newest revision. + assert_eq!(count, 1, "the 1s emission"); let needle = section(4); assert!( frames @@ -4729,10 +4756,8 @@ async fn pcr_stamps_step_by_the_grid() { /// rendition, where the exporter has no uniform cadence to lean on: frames arrive in /// decode order, and the two tracks advance the media clock at different rates. The /// clock still lands among the bytes it labels rather than clustering, though the -/// gaps are no longer uniform: a span is however long it took the next timestamp to -/// arrive, which for interleaved tracks is nothing like the media a frame's bytes -/// represent. Evening that out needs the muxer to hold a byte buffer and drain it at -/// a measured rate, which this does not do. +/// gaps are not uniform: without a multiplex rate each frame spreads only over the +/// slots since the frame before it. #[tokio::test(start_paused = true)] async fn pcr_stays_among_the_bytes_across_reordered_tracks() { let data = include_bytes!("test_data/scte35/kyrion_dirtystart.ts"); @@ -4758,8 +4783,7 @@ async fn pcr_stays_among_the_bytes_across_reordered_tracks() { "no clock packet may sit adjacent to the previous one: {gaps:?}" ); - // Stamps still step by exactly one slot, apart from the first interval, which - // covers the leading span rather than a whole slot. + // Stamps step by exactly one slot. let interval = PCR_INTERVAL.as_micros() as i128; let off = pcrs .windows(2) @@ -5374,7 +5398,7 @@ async fn reordered_video_loses_nothing_on_a_clean_path() { ); let decode = pes_decode_in_order(&out); assert!( - decode.is_sorted(), + in_decode_order(&decode), "a frame went out ahead of an earlier one: {decode:?}" ); } diff --git a/rs/moq-mux/src/container/ts/import_test.rs b/rs/moq-mux/src/container/ts/import_test.rs index f9dc05ccfa2..ebe9c54d0af 100644 --- a/rs/moq-mux/src/container/ts/import_test.rs +++ b/rs/moq-mux/src/container/ts/import_test.rs @@ -12,9 +12,12 @@ use bytes::BytesMut; /// [`std::time::Duration::ZERO`](std::time::Duration::ZERO) budget collapses to the live edge: /// completeness has to be asked for. const RECORDING_MAX_AGE: std::time::Duration = std::time::Duration::from_secs(30); -/// How long a drain waits for the next frame: past the recording delay, so the first -/// frame goes out, and then until output stops. -const DRAIN: std::time::Duration = RECORDING_MAX_AGE.saturating_add(std::time::Duration::from_secs(1)); +/// How long a drain waits for the next frame: past the recording delay, the mux-ahead +/// window a multiplex rate adds on top of it, and the clip, so the first frame goes out, +/// and then until output stops. +const DRAIN: std::time::Duration = RECORDING_MAX_AGE + .saturating_mul(3) + .saturating_add(std::time::Duration::from_secs(1)); /// Decode a whole TS buffer into a fresh broadcast and return the catalog. fn import_ts(data: &[u8]) -> crate::catalog::hang::Catalog { diff --git a/rs/moq-mux/src/container/ts/mod.rs b/rs/moq-mux/src/container/ts/mod.rs index 05858df90f1..1e7e95b1930 100644 --- a/rs/moq-mux/src/container/ts/mod.rs +++ b/rs/moq-mux/src/container/ts/mod.rs @@ -22,6 +22,7 @@ mod adts; mod export; mod import; mod mux_rate; +mod schedule; mod programs; mod si; diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs new file mode 100644 index 00000000000..14ba3ec9e1e --- /dev/null +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -0,0 +1,403 @@ +//! Lays a TS export's packets onto its PCR grid. + +use std::collections::{HashMap, VecDeque}; +use std::time::Duration; + +use mpeg2ts::ts::TsPacket; + +use super::export::PCR_INTERVAL; + +/// Grid slots per second, so a multiplex rate in bits per second is also the per-slot +/// allowance in [`PACKET`] units. +const SLOTS_PER_SECOND: u64 = (Duration::from_secs(1).as_nanos() / PCR_INTERVAL.as_nanos()) as u64; +const _: () = assert!( + Duration::from_secs(1) + .as_nanos() + .is_multiple_of(PCR_INTERVAL.as_nanos()) +); +/// One packet in the fixed-point unit of the rate credit: one slot at `rate` bits per second +/// is exactly `rate` units, so no slot rounds on its own and the remainder carries over. +const PACKET: u64 = TsPacket::SIZE as u64 * 8 * SLOTS_PER_SECOND; +/// How many empty slots to lay at most before the next packets: one second's worth. Past +/// this the media had an outage rather than a coarse cadence, and a dense clock history for +/// a span that carried no bytes only stalls anything pacing on the asserted values. +const BACKFILL: u128 = 40; + +/// The grid slot whose clock packet a unit decoding at `nanos` must be sent by. +pub(super) fn slot(nanos: u128) -> u128 { + nanos / PCR_INTERVAL.as_nanos() +} + +/// One access unit's packets, with the tables muxed ahead of it, still to go out. +struct Unit { + /// The last slot its packets may ride: the one that ends by its decode time. + due: u128, + /// The first slot an unpadded stream spreads it from: the slot after the unit before it. + start: u128, + packets: Vec, + /// Packets already laid out. + sent: usize, + keyframe: bool, +} + +impl Unit { + fn remaining(&self) -> usize { + self.packets.len() / TsPacket::SIZE - self.sent + } +} + +/// One grid slot's worth of output. +pub(super) struct Slot { + pub index: u128, + /// The media packets, in the order they were muxed. + pub packets: Vec, + /// How many null packets pad the slot to the multiplex rate, its clock packet included. + pub nulls: usize, + /// Whether a keyframe's first packet rides in the slot. + pub keyframe: bool, +} + +impl Slot { + /// The slot's packets after its clock packet: the media spread evenly among the null + /// packets, and each PID's packets spread evenly among the others, every PID keeping its + /// own order. + /// + /// A low-rate stream muxed whole, like an audio frame's few packets or two frames back to + /// back, would otherwise arrive at the multiplex's full rate and overflow its 512-byte + /// transport buffer, which drains at 2 Mb/s (ISO 13818-1 2.4.2.3). The program tables (PAT + /// and the PMT on `pmt_pid`) move ahead of every packet muxed after them, so they still + /// lead the keyframe they were written for and a reader knows each PID before its first + /// packet. + pub fn layout(&self, pmt_pid: u16, null: &[u8]) -> Vec { + let packets: Vec<&[u8]> = self.packets.chunks_exact(TsPacket::SIZE).collect(); + let pid = |packet: &[u8]| u16::from(packet[1] & 0x1f) << 8 | u16::from(packet[2]); + let mut counts: HashMap = HashMap::new(); + for packet in &packets { + *counts.entry(pid(packet)).or_default() += 1; + } + + // The k-th of a PID's n packets sits at (2k + 1) / 2n of the way through. + let mut seen: HashMap = HashMap::new(); + let mut keys: Vec<(u64, u64)> = packets + .iter() + .map(|packet| { + let pid = pid(packet); + let k = seen.entry(pid).or_default(); + *k += 1; + (2 * *k - 1, 2 * counts[&pid]) + }) + .collect(); + let cmp = |(an, ad): (u64, u64), (bn, bd): (u64, u64)| { + (u128::from(an) * u128::from(bd)).cmp(&(u128::from(bn) * u128::from(ad))) + }; + let mut after = (1, 1); + for (packet, key) in packets.iter().zip(keys.iter_mut()).rev() { + let pid = pid(packet); + if pid == 0 || pid == pmt_pid { + *key = after; + } else if cmp(*key, after).is_lt() { + after = *key; + } + } + let mut order: Vec = (0..packets.len()).collect(); + order.sort_by(|&a, &b| cmp(keys[a], keys[b])); + + // Media packet i of k lands at (2i + 1) / 2k of the way through, nulls in between. + let (k, total) = (order.len(), order.len() + self.nulls); + let mut out = Vec::with_capacity(total * TsPacket::SIZE); + let mut next = 0; + for at in 0..total { + if next < k && at == (2 * next + 1) * total / (2 * k) { + out.extend_from_slice(packets[order[next]]); + next += 1; + } else { + out.extend_from_slice(null); + } + } + out + } +} + +/// Packs muxed access units onto the PCR grid. +/// +/// With a multiplex rate each slot carries exactly what the rate allows, padded with null +/// packets, and each unit goes out as late as the rate lets it while still arriving by its +/// decode time. A unit bigger than one slot (a keyframe) spreads over the slots before its +/// decode time, back as far as the `window`; one that cannot fit there fails the export +/// rather than overrun the rate or arrive late. Sending no earlier than needed keeps a +/// receiver's buffers as empty as the schedule allows. +/// +/// Without a rate the stream is unpadded, and each unit spreads evenly over the slots +/// between the unit before it and its own decode time. +pub(super) struct Schedule { + units: VecDeque, + /// Slots before its due slot a unit may go out: the export's delay. + window: u128, + /// The multiplex rate, in bits per second. + rate: Option, + /// The rate's allowance not yet spent, in [`PACKET`] units. + credit: u64, + /// The next slot to lay out. + next: Option, + /// The due slot of the last unit pushed. + last: Option, + /// Slots before this one may overrun the rate ([`Self::set_rate`]). + grace: Option, +} + +impl Schedule { + pub fn new(window: Duration) -> Self { + Self { + units: VecDeque::new(), + window: slot(window.as_nanos()), + rate: None, + credit: 0, + next: None, + last: None, + grace: None, + } + } + + /// Pad to `rate` bits per second from here on, or stop padding. + /// + /// A rate that turns up mid-stream (import records one only once it has measured it) + /// finds units already queued with less than a window of room before them, so for one + /// window those may overrun the rate rather than fail the export. + pub fn set_rate(&mut self, rate: Option) { + if self.rate != rate { + self.rate = rate; + self.credit = 0; + self.grace = self.next.map(|next| next + self.window); + } + } + + /// Queue an access unit (whole TS packets) that decodes at `decode` nanoseconds. + /// + /// Units go out in the order they are pushed, which keeps the program tables ahead of + /// what they describe. The jitter buffer hands them over in decode order; a unit that + /// decodes before one pushed ahead of it (frames read in arrival order under a zero + /// delay) is due with it. + pub fn push(&mut self, decode: u128, packets: Vec, keyframe: bool) { + let due = slot(decode); + for unit in self.units.iter_mut().rev() { + if unit.due <= due { + break; + } + unit.due = due; + unit.start = unit.start.min(due); + } + let start = self.last.map_or(due, |last| (last + 1).min(due)); + self.last = Some(due); + self.units.push_back(Unit { + due, + start, + packets, + sent: 0, + keyframe, + }); + } + + pub fn is_empty(&self) -> bool { + self.units.is_empty() + } + + /// Drop everything queued and start the grid afresh. + pub fn clear(&mut self) { + self.units.clear(); + self.next = None; + self.last = None; + self.grace = None; + self.credit = 0; + } + + /// Lay out the next slot, if it is settled. + /// + /// `known` is the first slot a unit still to be pushed could be due in, which settles + /// every slot whose window ends before it; `None` means nothing more is coming, so every + /// queued unit goes out. + pub fn next(&mut self, known: Option) -> anyhow::Result> { + let first = match (&self.rate, self.units.front()) { + (Some(_), Some(unit)) => Some(unit.due.saturating_sub(self.window)), + (None, Some(_)) => self.units.iter().map(|unit| unit.start).min(), + (_, None) => None, + }; + let index = match (self.next, first) { + (None, None) => return Ok(None), + (None, Some(first)) => first, + // Skip the empty slots of a long gap, keeping the second before what comes next. + (Some(next), first) => { + let window = self.rate.map_or(0, |_| self.window); + let target = first.or(known.map(|known| known.saturating_sub(window))); + next.max(target.map_or(0, |target| target.saturating_sub(BACKFILL))) + } + }; + // An unpadded unit spreads only back to the one before it, so it needs no window. + let window = self.rate.map_or(0, |_| self.window); + match known { + Some(known) if index + window >= known => return Ok(None), + None if self.units.is_empty() => return Ok(None), + _ => {} + } + self.next = Some(index + 1); + + let (take, nulls) = match self.rate { + Some(rate) => { + self.credit += rate; + let allowed = (self.credit / PACKET).max(1); + self.credit -= (self.credit / PACKET).min(allowed) * PACKET; + let media = self.needed(index, rate / PACKET); + let overrun = media >= allowed as usize; + anyhow::ensure!( + !overrun || self.grace.is_some_and(|grace| index < grace), + "MPEG-TS output needs {media} packets in a {}ms slot, more than {rate} b/s allows within the delay; raise the delay or the multiplex rate", + PCR_INTERVAL.as_millis() + ); + let mut left = media; + let take: Vec = self + .units + .iter() + .map(|unit| { + let take = unit.remaining().min(left); + left -= take; + take + }) + .collect(); + (take, (allowed as usize - 1).saturating_sub(media)) + } + None => { + let take = self + .units + .iter() + .map(|unit| match unit.start <= index { + true => unit.remaining().div_ceil((unit.due.max(index) - index + 1) as usize), + false => 0, + }) + .collect(); + (take, 0) + } + }; + + let mut packets = Vec::new(); + let mut keyframe = false; + for (unit, take) in self.units.iter_mut().zip(take) { + if take == 0 { + continue; + } + keyframe |= unit.keyframe && unit.sent == 0; + let from = unit.sent * TsPacket::SIZE; + packets.extend_from_slice(&unit.packets[from..from + take * TsPacket::SIZE]); + unit.sent += take; + } + self.units.retain(|unit| unit.remaining() > 0); + Ok(Some(Slot { + index, + packets, + nulls, + keyframe, + })) + } + + /// The fewest packets slot `index` must carry so every queued unit still arrives by its + /// due slot, with `per_slot` packets of room in each slot after it. + fn needed(&self, index: u128, per_slot: u64) -> usize { + let mut queued = 0; + let mut needed = 0; + for unit in &self.units { + queued += unit.remaining(); + let room = (unit.due.saturating_sub(index) as usize).saturating_mul(per_slot.saturating_sub(1) as usize); + needed = needed.max(queued.saturating_sub(room)); + } + needed + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A unit of `count` packets on PID `pid`. + fn unit(pid: u8, count: usize) -> Vec { + let mut packet = [0u8; TsPacket::SIZE]; + packet[0] = 0x47; + packet[2] = pid; + packet[3] = 0x10; + packet.repeat(count) + } + + fn ms(ms: u128) -> u128 { + ms * 1_000_000 + } + + /// 40 packets per slot, the clock packet included. + const RATE: u64 = 40 * PACKET; + + #[test] + fn a_unit_goes_out_in_its_due_slot() { + let mut schedule = Schedule::new(Duration::from_millis(100)); + schedule.set_rate(Some(RATE)); + schedule.push(ms(1_000), unit(1, 3), true); + // Nothing past the window is settled until a later unit shows up. + let mut slots = Vec::new(); + while let Some(slot) = schedule.next(Some(slot(ms(1_200)))).unwrap() { + slots.push((slot.index, slot.packets.len() / TsPacket::SIZE, slot.nulls, slot.keyframe)); + } + assert_eq!(slots.first().map(|s| s.0), Some(36), "starts a window ahead of the unit"); + assert!(slots.iter().all(|s| s.1 + s.2 == 39), "every slot is padded to the rate"); + let sent: Vec<_> = slots.iter().filter(|s| s.1 > 0).collect(); + assert_eq!(sent, [&(40, 3, 36, true)], "as late as it can go: {slots:?}"); + } + + #[test] + fn a_burst_spreads_over_the_slots_before_it() { + let mut schedule = Schedule::new(Duration::from_millis(100)); + schedule.set_rate(Some(RATE)); + schedule.push(ms(1_000), unit(1, 100), true); + let mut sent = Vec::new(); + while let Some(slot) = schedule.next(None).unwrap() { + sent.push((slot.index, slot.packets.len() / TsPacket::SIZE)); + } + // 39 media packets fit each slot, so 100 packets take the last three before its due. + assert_eq!(sent, [(36, 0), (37, 0), (38, 22), (39, 39), (40, 39)]); + } + + #[test] + fn a_burst_too_big_for_the_window_fails() { + let mut schedule = Schedule::new(Duration::from_millis(50)); + schedule.set_rate(Some(RATE)); + schedule.push(ms(1_000), unit(1, 200), true); + let err = std::iter::from_fn(|| schedule.next(None).transpose()).find_map(Result::err); + assert!(err.is_some(), "a burst past the window must fail the export"); + } + + /// A rate that turns up mid-stream lets the units already queued overrun it for one + /// window, rather than fail the export. + #[test] + fn a_late_rate_overruns_for_one_window() { + let mut schedule = Schedule::new(Duration::from_millis(100)); + schedule.push(ms(1_000), unit(1, 4), true); + while schedule.next(Some(slot(ms(1_050)))).unwrap().is_some() {} + schedule.set_rate(Some(RATE)); + schedule.push(ms(1_100), unit(1, 150), true); + let mut sent = Vec::new(); + while let Some(slot) = schedule.next(None).unwrap() { + sent.push((slot.index, slot.packets.len() / TsPacket::SIZE, slot.nulls)); + } + assert_eq!(sent, [(42, 72, 0), (43, 39, 0), (44, 39, 0)]); + + schedule.push(ms(3_000), unit(1, 2_000), true); + let err = std::iter::from_fn(|| schedule.next(None).transpose()).find_map(Result::err); + assert!(err.is_some(), "past the window, a burst that does not fit fails"); + } + + #[test] + fn unpadded_units_spread_up_to_their_decode_time() { + let mut schedule = Schedule::new(Duration::ZERO); + schedule.push(ms(1_000), unit(1, 4), true); + schedule.push(ms(1_100), unit(1, 4), false); + let mut sent = Vec::new(); + while let Some(slot) = schedule.next(None).unwrap() { + sent.push((slot.index, slot.packets.len() / TsPacket::SIZE, slot.nulls)); + } + assert_eq!(sent, [(40, 4, 0), (41, 1, 0), (42, 1, 0), (43, 1, 0), (44, 1, 0)]); + } +} From bad8405855998c9a0ef07c8417f76f11c26351d4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 19:20:30 -0700 Subject: [PATCH 08/40] feat(moq-cli): gate the TS byte schedule and pace export with the pacer alone run.sh gates pcr-schedule at 99% whenever the generated clip's rate is known, grading from the first null packet, and CI adds a 2 Mb/s run whose keyframes outgrow a PCR slot. The export now releases at the source's pace, so the CLI's Delivery shrinks to the pacer. Folds in the byte-schedule quest. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/interop.yml | 9 + doc/bin/cli.md | 16 +- quest/m1/tstd/README.md | 7 +- quest/m1/tstd/byte-schedule.md | 31 --- rs/moq-cli/src/subscribe.rs | 257 ++++----------------- rs/moq-mux/src/container/ts/export.rs | 18 +- rs/moq-mux/src/container/ts/export_test.rs | 5 +- rs/moq-mux/src/container/ts/mod.rs | 2 +- rs/moq-mux/src/container/ts/schedule.rs | 26 ++- rs/moq-mux/src/lib.rs | 2 +- test/ts/README.md | 31 +-- test/ts/pcr-timing.py | 13 ++ test/ts/run.sh | 18 +- 13 files changed, 149 insertions(+), 286 deletions(-) delete mode 100644 quest/m1/tstd/byte-schedule.md diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index 74a22e98bd1..b194db3a15b 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -122,6 +122,15 @@ jobs: env: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + # The same round-trip at 2 Mb/s, where every keyframe outgrows a 25 ms PCR slot, + # so the export has to spread it over the slots before its DTS and still keep + # the constant-rate schedule. + - name: TS keyframe bursts + run: nix develop --command just test ts --bitrate 2000000 + shell: bash -leo pipefail {0} + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + # The same round-trip over an open-GOP clip: every leading picture after a # recovery point must come back, in decode order. - name: TS open GOP diff --git a/doc/bin/cli.md b/doc/bin/cli.md index af5e0b114a4..2c112c22781 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -104,11 +104,17 @@ profile or channel layout ADTS cannot label is refused rather than mislabeled. A constant-rate MPEG-TS source records its multiplex rate in the catalog (`mpegts.muxRate`, measured off the PCR clock, null stuffing included), and -`export ts` pads its output with null packets back to that rate so an IRD or -groomer receives a constant-rate stream. `--mux-rate 5000000` pads to an explicit -rate instead, including for a broadcast that recorded none. Media is never delayed -or dropped to fit: a source that sustains more than the rate overruns it, and a -VBR source records nothing, so export without either stays unpadded. +`export ts` pads its output with null packets back to that rate on a constant-rate +schedule, so an IRD or groomer recovering its clock from packet arrival can lock: +every PCR interval carries the bytes the rate implies. `--mux-rate 5000000` pads to +an explicit rate instead, including for a broadcast that recorded none. A frame +goes out as late as the rate lets it while still arriving by its decode time, and a +keyframe bigger than one 25 ms interval spreads over the intervals before it, back +as far as `--delay`. That window adds up to the delay again to the output's +latency, so a padded export trails the source by up to twice the delay. A burst +that does not fit the window, or a source that sustains more than the rate, fails +the export with an error naming both knobs. A VBR source records no rate, so +export without one stays unpadded and adds no window. fMP4 export writes one fragment per publisher group on each track. Audio follows the publisher's cuts; video normally follows GOPs. Closing a group flushes it diff --git a/quest/m1/tstd/README.md b/quest/m1/tstd/README.md index 29b1fbc055f..5532cedd52c 100644 --- a/quest/m1/tstd/README.md +++ b/quest/m1/tstd/README.md @@ -16,6 +16,12 @@ Padding, pacing, and muxing all assume a fixed delay, so the export gets one first. t0ms is testing whether T-STD compliance is feasible at all; record the result here. If it isn't, re-plan this line. +Measured (2026-09-30, #4645): with the fixed-delay jitter buffer and the +constant-rate schedule, a clean-path round trip of the generated clip passes +the strict T-STD check at 10 Mb/s and at 2 Mb/s, where every keyframe outgrows +a PCR interval. A burst that does not fit the delay fails the export rather +than overrun the rate. + This README owns the end-to-end proof: the #4613 netem rig (10% loss, a real ~10 Mb/s broadcast TS) passes the strict T-STD check, and the recipe runs nightly. @@ -23,4 +29,3 @@ nightly. ## Required - [T-STD check](/quest/m1/tstd/check.md) - the harness grades the full buffer model instead of the transport buffer alone -- [TS byte schedule](/quest/m1/tstd/byte-schedule.md) - PCRs sit on the byte grid the mux rate implies, paced against the fixed delay diff --git a/quest/m1/tstd/byte-schedule.md b/quest/m1/tstd/byte-schedule.md deleted file mode 100644 index eebe67bf8fe..00000000000 --- a/quest/m1/tstd/byte-schedule.md +++ /dev/null @@ -1,31 +0,0 @@ -# [L] moq export ts: PCRs sit on the byte grid the mux rate implies - -## Goal - -When the catalog carries `mpegts.muxRate`, `moq export ts` emits a -constant-rate stream: the bytes between consecutive PCRs are what the declared -rate implies for that interval, so a receiver recovering its clock from packet -arrival can lock. Today the average rate is right (#3831) but the bytes clump: -188 B to 870 KB between PCRs where 31 KB is needed, and only 3.3 % of intervals -within 1 % of nominal (#3925). Streams without a mux rate stay VBR as today. A -UDP sink is out of scope; delivery stays with an external tool. - -## Plan - -- Padding and PCR placement become one scheduling decision in - `rs/moq-mux/src/container/ts/export.rs` (`advance`, `emit`, `stuff`): a PCR's - value follows its byte position at the mux rate, and a keyframe burst is - spread over the slots before its DTS instead of landing between two PCRs. -- That makes the mux run ahead of decode by a buffer delay. Decided - (2026-09-30): the delay is the fixed `--delay` from - release (`moq export ts --delay`, `ts::Export::with_delay`), not one that grows - to fit bursts. Say what happens when a burst does not fit the delay (fail - loud, or drop to VBR with a warning) and record the choice here. -- The CLI `Delivery` pacer releases one slice per PCR interval; confirm it - still writes on the schedule the PCRs describe. -- `test/ts/pcr-timing.py`'s `pcr-schedule` grades the bytes between - consecutive PCRs, report-only in the TS recipe. Once the schedule lands, gate - it there with `--schedule-pct-min`, against a fixture whose keyframes outgrow - a PCR slot; the generated clip mostly pads and passes already. -- `doc/bin/cli.md`: say that export pads to `mpegts.muxRate` on a constant-rate - schedule and what latency that adds. diff --git a/rs/moq-cli/src/subscribe.rs b/rs/moq-cli/src/subscribe.rs index aede13db84f..cb87f0744b2 100644 --- a/rs/moq-cli/src/subscribe.rs +++ b/rs/moq-cli/src/subscribe.rs @@ -318,30 +318,18 @@ impl Subscribe { // grid and stamps each slice at its slot boundary, on the contract that the // caller writes the bytes at the time the stamp asserts. Draining on arrival // instead collapses the clock into position clusters no downstream stage can - // repair (#2984). See [`Delivery`] for how the pacing stays - // bounded; it needs to know whether each frame was waited for, hence the - // hand-rolled poll instead of `ts.next()`. + // repair (#2984). The export already releases at the source's pace, so the + // pacer only spreads each burst of settled slices over the time they cover. let mut delivery = Delivery::new(self.args.max_age); let linger = self.args.linger; loop { let end = loop { - let mut waited = false; - let frame = hang::moq_net::kio::wait(|waiter| match ts.poll_next(waiter) { - std::task::Poll::Pending => { - waited = true; - std::task::Poll::Pending - } - ready => ready, - }) - .await; - - let frame = match frame { + let frame = match ts.next().await { Ok(Some(frame)) => frame, Ok(None) => break Ok(()), Err(err) => break Err(err), }; - delivery.update(&frame, ts.discontinuity()); - delivery.deliver(&frame, waited, &mut stdout).await?; + delivery.deliver(&frame, ts.discontinuity(), &mut stdout).await?; }; // Any end waits out the linger, and on expiry the last one is the result: a @@ -404,56 +392,15 @@ async fn resume_within( } } -/// Paced stdout delivery for the TS export: sleeps until each frame's send -/// instant, with total delivery lag bounded by the latency budget. +/// Paced stdout delivery for the TS export: sleeps until each frame's send instant. /// -/// The bound is the subtle part. The pacer alone caps how far one frame may be -/// scheduled past the `now` it paces with, but our own sleeps push that `now` -/// forward, so a backlog arriving faster than real time (a tune-in group -/// replaying from its keyframe, a catch-up after a stall) stays within the lead -/// of every individual call while total delivery lag grows without bound. The -/// export's group skipping can't shed that lag either: it fires when the -/// *current group* is blocked with a newer alternative, so it measures producer -/// stalls, never consumer lag. So lag is measured here instead, against -/// `arrived`: the last instant the export made us wait, which is when a frame -/// obtained without waiting could first have been queued. When a frame's -/// schedule overshoots that epoch by more than the lead, it is delivered -/// immediately and becomes the live edge -/// ([`Pacer::hurry`](moq_mux::Pacer::hurry)). The anchor then rides the newest -/// frame through a backlog, so pacing resumes from the live edge. -/// -/// A hurry moves the epoch with it, because the frame it delivers *is* the live -/// edge and the distance it is measured from has to be too. `hurry` returns `now`, -/// so the credit below can never fire on the frame that hurried; leaving the epoch -/// behind would put every later frame the same overshoot past it and shed the whole -/// stream at the arrival cadence, which the buffered export would never correct -/// since it never makes us wait. A sink that is genuinely slow still sheds on every -/// stall: each one leaves the next frame far enough past the epoch to overshoot -/// again. -/// -/// Reaching a scheduled instant also advances the epoch, and has to. The TS export -/// holds a mux buffer: it always has the next grid slot ready, so it stops making -/// us wait, and an epoch that only moved on a wait would freeze for good and take -/// the budget with it (measured: a hurry roughly every second, each shedding the -/// pacing it was there to protect). Sleeping to the schedule is the proof that -/// replaces the wait, since nothing the export queued while we slept could have -/// gone out any earlier. What that gives up is a producer running faster than real -/// time indefinitely: the sink keeps pace with it and falls further behind live -/// without the budget noticing. Bounding *that* is the export's own -/// `--delay`, which sheds media rather than compressing the clock. -/// -/// The budget is the lead plus whatever standing lag the pacer has absorbed -/// ([`Pacer::slack`](moq_mux::Pacer::slack)), which is a distance it is holding on -/// purpose rather than lag to shed. +/// The export holds every frame its delay and lays the multiplex out a slot at a time, +/// so it hands slices over at the source's pace, a few at once. The pacer spreads each +/// handful over the slots it covers, holding up to the delay ahead of the wall clock. +/// A sink that falls behind writes what is overdue at once and catches up. struct Delivery { discontinuity: u64, pacer: moq_mux::Pacer, - /// The delivery-lag bound, and the pacer's lead: both are the export's - /// latency budget. - lead: Duration, - /// The last instant the export made us wait for a frame: the conservative - /// arrival epoch for frames obtained without waiting (see the type docs). - arrived: tokio::time::Instant, } impl Delivery { @@ -461,60 +408,25 @@ impl Delivery { Self { pacer: moq_mux::Pacer::default().with_lead(lead), discontinuity: 0, - lead, - // tokio's clock rather than the bare std one so tests can pause it; in - // production they are identical. - arrived: tokio::time::Instant::now(), } } - fn update(&mut self, frame: &moq_mux::container::Frame, discontinuity: u64) { - if discontinuity == self.discontinuity { - return; - } - self.discontinuity = discontinuity; - self.arrived = tokio::time::Instant::now(); - self.pacer.hurry(frame.timestamp, self.arrived.into_std()); - } - - /// Write one export frame to `out` at its paced instant. `waited` is whether - /// the export made us wait for this frame rather than having it ready. + /// Write one export frame to `out` at its paced instant. A new `discontinuity` + /// (the program clock restarted) re-anchors the pacing on this frame. async fn deliver( &mut self, frame: &moq_mux::container::Frame, - waited: bool, + discontinuity: u64, out: &mut (impl tokio::io::AsyncWrite + Unpin), ) -> anyhow::Result<()> { - let now = tokio::time::Instant::now(); - if waited { - self.arrived = now; - } - - // The pacer's own slack is not lag: it is the producer's standing delivery - // distance, which the pacer discovered and is holding on purpose. Counting - // it here would shed the margin on a fixed cadence and put the writes back - // on the arrival clock, which is the whole thing this is here to avoid. - let budget = self.lead + self.pacer.slack(); - let mut send_at = self.pacer.pace(frame.timestamp, now.into_std()); - if send_at.saturating_duration_since(self.arrived.into_std()) > budget { - send_at = self.pacer.hurry(frame.timestamp, now.into_std()); - // A hurry makes this frame the live edge, so the epoch it is measured - // against becomes now as well. `hurry` returns `now`, so the credit below - // can't do it, and a stale epoch would overshoot the budget on every - // later frame, latching the shed on for good. - self.arrived = now; - } - - let send_at = tokio::time::Instant::from_std(send_at); - tokio::time::sleep_until(send_at).await; - // Reaching a scheduled instant is proof we are not behind, and it is the only - // such proof once the export holds a buffer: it always has the next slot - // ready, so it stops making us wait and the arrival epoch would freeze for - // good, taking the budget with it. Credit the scheduled instant rather than - // `now`, so an overshoot isn't credited as headroom. - if send_at > now { - self.arrived = send_at; - } + // tokio's clock rather than the bare std one so tests can pause it; in + // production they are identical. + let now = tokio::time::Instant::now().into_std(); + let send_at = match std::mem::replace(&mut self.discontinuity, discontinuity) == discontinuity { + true => self.pacer.pace(frame.timestamp, now), + false => self.pacer.hurry(frame.timestamp, now), + }; + tokio::time::sleep_until(tokio::time::Instant::from_std(send_at)).await; out.write_all(&frame.payload).await?; out.flush().await?; Ok(()) @@ -548,22 +460,21 @@ mod tests { // The first frame anchors the pacer and is written immediately. delivery - .deliver(&frame(0, Timescale::MICRO), true, &mut out) + .deliver(&frame(0, Timescale::MICRO), 0, &mut out) .await .unwrap(); assert_eq!(start.elapsed(), Duration::ZERO); - // A PCR slot 25ms later (ready without waiting, like a backfilled grid - // slot) waits for its grid boundary. + // A PCR slot 25ms later waits for its grid boundary. delivery - .deliver(&frame(25_000, Timescale::MICRO), false, &mut out) + .deliver(&frame(25_000, Timescale::MICRO), 0, &mut out) .await .unwrap(); assert_eq!(start.elapsed(), Duration::from_millis(25)); // A media frame at the source's 90 kHz timescale paces on the same clock. delivery - .deliver(&frame(3_600, Timescale::new(90_000).unwrap()), true, &mut out) + .deliver(&frame(3_600, Timescale::new(90_000).unwrap()), 0, &mut out) .await .unwrap(); assert_eq!(start.elapsed(), Duration::from_millis(40)); @@ -571,116 +482,32 @@ mod tests { assert_eq!(out.len(), 3 * 188, "every payload was written"); } - /// A sink that cannot reach its schedule still sheds the lag, which is the case - /// the budget is left guarding. - /// - /// The epoch advances on a wait, on reaching a scheduled instant, or on a hurry. - /// A sink falling behind reaches none of its instants and is never made to wait, - /// so only the hurry moves it, and each fresh stall puts the next frame far - /// enough past that epoch to overshoot the budget and hurry again. - /// - /// What this no longer covers, deliberately, is a one-off pre-queued backlog - /// (a tune-in group replaying from its keyframe). That now paces out at the - /// media rate instead of being shed, because it is indistinguishable from the - /// TS export's own mux buffer from here: both hand over a frame that is ready - /// and ahead of the schedule. The size of such a backlog is bounded by the - /// export's `--delay`, which is where it belongs. + /// A sink that falls behind writes what is overdue at once, then paces again. #[tokio::test(start_paused = true)] - async fn a_sink_that_cannot_keep_up_sheds_the_lag() { + async fn a_slow_sink_catches_up() { let mut delivery = Delivery::new(Duration::from_millis(500)); let mut out = Vec::new(); let start = tokio::time::Instant::now(); delivery - .deliver(&frame(0, Timescale::MICRO), true, &mut out) + .deliver(&frame(0, Timescale::MICRO), 0, &mut out) .await .unwrap(); - // The writer stalls for 2s (a blocked pipe): wall clock runs on, the media - // clock does not, and every frame after this is overdue on arrival. + // The writer stalls for 2s (a blocked pipe): every slot after this is overdue. tokio::time::advance(Duration::from_secs(2)).await; - delivery - .deliver(&frame(400_000, Timescale::MICRO), false, &mut out) + .deliver(&frame(400_000, Timescale::MICRO), 0, &mut out) .await .unwrap(); assert_eq!( start.elapsed(), Duration::from_secs(2), - "an overdue frame writes at once: the overshoot hurries and makes it the live edge" + "an overdue slot writes at once" ); - - // Shedding happens at the re-anchor, once, not on every frame after it. This - // one is 200ms of media past the new edge and 200ms past the epoch the hurry - // set with it, so it is inside the budget and paces. - delivery - .deliver(&frame(600_000, Timescale::MICRO), false, &mut out) - .await - .unwrap(); - assert_eq!(start.elapsed(), Duration::from_secs(2) + Duration::from_millis(200)); - - // A sink that goes on stalling goes on shedding, which is the property this - // test is here for: each stall leaves the next frame far enough past the - // epoch to overshoot the budget again, however recently the last hurry moved - // it. Media steps 25ms per iteration while the wall clock steps 2s. - for slot in 1..=3u64 { - tokio::time::advance(Duration::from_secs(2)).await; - let before = start.elapsed(); - delivery - .deliver(&frame(600_000 + slot * 25_000, Timescale::MICRO), false, &mut out) - .await - .unwrap(); - assert_eq!(start.elapsed(), before, "stall {slot} must shed, not pace"); - } } - /// A hurry has to move the arrival epoch with it, or it latches. - /// - /// `hurry` returns `now`, so the credit for reaching a scheduled instant can - /// never fire on the frame that hurried. Left there, the epoch stays wherever it - /// was before the shed while the schedule walks forward from the new edge, so the - /// next frame overshoots the budget too, and so does every one after it. The - /// buffered export never waits, so nothing else would move the epoch back. - #[tokio::test(start_paused = true)] - async fn pacing_resumes_after_a_hurry_without_a_wait() { - let mut delivery = Delivery::new(Duration::from_millis(500)); - let mut out = Vec::new(); - - let start = tokio::time::Instant::now(); - delivery - .deliver(&frame(0, Timescale::MICRO), true, &mut out) - .await - .unwrap(); - - // Stall long enough that the schedule outruns the budget and sheds. - tokio::time::advance(Duration::from_secs(2)).await; - delivery - .deliver(&frame(400_000, Timescale::MICRO), false, &mut out) - .await - .unwrap(); - let hurried = start.elapsed(); - assert_eq!(hurried, Duration::from_secs(2), "the overshoot sheds"); - - // Grid slots from the new edge, every one already queued and none waited on. - // Against a stale epoch each is a whole stall past it, so each would hurry and - // write at once, pinning `start.elapsed()` at `hurried` for the whole loop. - for slot in 1..=8u64 { - delivery - .deliver(&frame(400_000 + slot * 25_000, Timescale::MICRO), false, &mut out) - .await - .unwrap(); - assert_eq!( - start.elapsed(), - hurried + Duration::from_millis(slot * 25), - "slot {slot} must be paced, not shed" - ); - } - } - - /// The TS export holds a mux buffer, so it always has the next grid slot ready - /// and stops making the sink wait. Reaching a scheduled instant has to advance - /// the epoch too, or the budget freezes and hurries on a fixed cadence, shedding - /// the pacing it exists to protect. + /// Slots the export has ready at once still go out on their grid. #[tokio::test(start_paused = true)] async fn a_buffered_producer_keeps_pacing() { let mut delivery = Delivery::new(Duration::from_millis(500)); @@ -688,15 +515,13 @@ mod tests { let start = tokio::time::Instant::now(); delivery - .deliver(&frame(0, Timescale::MICRO), true, &mut out) + .deliver(&frame(0, Timescale::MICRO), 0, &mut out) .await .unwrap(); - // A grid slot every 25ms, each already queued: an epoch that only moved on a - // wait would freeze here and hurry once the schedule passed 500ms. for slot in 1..=40u64 { delivery - .deliver(&frame(slot * 25_000, Timescale::MICRO), false, &mut out) + .deliver(&frame(slot * 25_000, Timescale::MICRO), 0, &mut out) .await .unwrap(); assert_eq!( @@ -711,17 +536,19 @@ mod tests { let mut delivery = Delivery::new(Duration::from_millis(500)); let mut out = tokio::io::sink(); delivery - .deliver(&frame(10_000_000, Timescale::MICRO), true, &mut out) + .deliver(&frame(10_000_000, Timescale::MICRO), 0, &mut out) .await .unwrap(); - let first = frame(0, Timescale::MICRO); let now = tokio::time::Instant::now(); - delivery.update(&first, 1); - delivery.deliver(&first, false, &mut out).await.unwrap(); + delivery + .deliver(&frame(0, Timescale::MICRO), 1, &mut out) + .await + .unwrap(); assert_eq!(now.elapsed(), Duration::ZERO); - let next = frame(40_000, Timescale::MICRO); - delivery.update(&next, 1); - delivery.deliver(&next, false, &mut out).await.unwrap(); + delivery + .deliver(&frame(40_000, Timescale::MICRO), 1, &mut out) + .await + .unwrap(); assert_eq!(now.elapsed(), Duration::from_millis(40)); } diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 7f571b6bfa6..98026801d5c 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -46,8 +46,8 @@ use crate::container::{ExportSource, Frame}; use crate::jitter::{self, Arrival, Push}; use super::adts; -use super::schedule::{self, Schedule}; use super::catalog; +use super::schedule::{self, Schedule}; /// PID of the single program's PMT. const PMT_PID: u16 = 0x1000; @@ -241,7 +241,11 @@ impl Track { } let pts = to_ticks(frame.timestamp); let dts = self.clock.author(pts, self.reserve.ticks, self.reserve.period()); - ((dts != pts).then_some(dts), frame.keyframe, self.source.description().cloned()) + ( + (dts != pts).then_some(dts), + frame.keyframe, + self.source.description().cloned(), + ) } _ => (None, true, None), }; @@ -1679,7 +1683,7 @@ impl Export { /// payload (a clock packet) repeats the counter before it (ISO 13818-1 2.4.3.3); before /// anything has gone out on its PID, any value starts a valid run. fn number(&mut self, packets: &mut [u8]) { - for packet in packets.chunks_exact_mut(TsPacket::SIZE) { + for packet in packets.as_chunks_mut::<{ TsPacket::SIZE }>().0 { let pid = u16::from(packet[1] & 0x1f) << 8 | u16::from(packet[2]); if pid == 0x1FFF { continue; @@ -2187,7 +2191,11 @@ struct DecodeClock { impl DecodeClock { /// The DTS of a frame presented at `pts`, the next in decode order. fn author(&mut self, pts: u64, reserve: u64, declared: Option) -> u64 { - if let Some(step) = self.previous.map(|previous| pts.abs_diff(previous)).filter(|&step| step > 0) { + if let Some(step) = self + .previous + .map(|previous| pts.abs_diff(previous)) + .filter(|&step| step > 0) + { self.period = Some(self.period.map_or(step, |period| period.min(step))); } self.previous = Some(pts); @@ -2201,7 +2209,7 @@ impl DecodeClock { // count back from the earliest seen as a decoder that ran all along would have. let held = self.window.len(); let earliest = match held > depth { - true => self.window.drain(..held - depth).last().unwrap_or(pts), + true => self.window.drain(..held - depth).next_back().unwrap_or(pts), false => self.window[0].saturating_sub((depth + 1 - held) as u64 * period), }; earliest.saturating_sub(reserve % period) diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index a474050cfa4..545aa8749c4 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -5359,7 +5359,10 @@ async fn declared_reorder_sizes_the_decode_clock_from_the_first_frame() { ); let decode: Vec = timing.iter().map(|&(pts, dts)| dts.unwrap_or(pts)).collect(); for (i, step) in decode.windows(2).map(|w| w[1] - w[0]).enumerate() { - assert_eq!(step, FRAME_TICKS, "frame {i} does not decode one period after the last: {decode:?}"); + assert_eq!( + step, FRAME_TICKS, + "frame {i} does not decode one period after the last: {decode:?}" + ); } assert_decodes_after_the_clock(&out_late); } diff --git a/rs/moq-mux/src/container/ts/mod.rs b/rs/moq-mux/src/container/ts/mod.rs index 1e7e95b1930..b7fbb7443b1 100644 --- a/rs/moq-mux/src/container/ts/mod.rs +++ b/rs/moq-mux/src/container/ts/mod.rs @@ -22,8 +22,8 @@ mod adts; mod export; mod import; mod mux_rate; -mod schedule; mod programs; +mod schedule; mod si; // The `mpegts` catalog section (per-track PID + descriptors plus verbatim carriage diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index 14ba3ec9e1e..f3ba0f322f1 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -69,7 +69,13 @@ impl Slot { /// lead the keyframe they were written for and a reader knows each PID before its first /// packet. pub fn layout(&self, pmt_pid: u16, null: &[u8]) -> Vec { - let packets: Vec<&[u8]> = self.packets.chunks_exact(TsPacket::SIZE).collect(); + let packets: Vec<&[u8]> = self + .packets + .as_chunks::<{ TsPacket::SIZE }>() + .0 + .iter() + .map(|p| p.as_slice()) + .collect(); let pid = |packet: &[u8]| u16::from(packet[1] & 0x1f) << 8 | u16::from(packet[2]); let mut counts: HashMap = HashMap::new(); for packet in &packets { @@ -339,10 +345,22 @@ mod tests { // Nothing past the window is settled until a later unit shows up. let mut slots = Vec::new(); while let Some(slot) = schedule.next(Some(slot(ms(1_200)))).unwrap() { - slots.push((slot.index, slot.packets.len() / TsPacket::SIZE, slot.nulls, slot.keyframe)); + slots.push(( + slot.index, + slot.packets.len() / TsPacket::SIZE, + slot.nulls, + slot.keyframe, + )); } - assert_eq!(slots.first().map(|s| s.0), Some(36), "starts a window ahead of the unit"); - assert!(slots.iter().all(|s| s.1 + s.2 == 39), "every slot is padded to the rate"); + assert_eq!( + slots.first().map(|s| s.0), + Some(36), + "starts a window ahead of the unit" + ); + assert!( + slots.iter().all(|s| s.1 + s.2 == 39), + "every slot is padded to the rate" + ); let sent: Vec<_> = slots.iter().filter(|s| s.1 > 0).collect(); assert_eq!(sent, [&(40, 3, 36, true)], "as late as it can go: {slots:?}"); } diff --git a/rs/moq-mux/src/lib.rs b/rs/moq-mux/src/lib.rs index ad184c5efa6..85faca69b5e 100644 --- a/rs/moq-mux/src/lib.rs +++ b/rs/moq-mux/src/lib.rs @@ -33,10 +33,10 @@ pub mod codec; pub mod container; mod error; pub mod import; +mod jitter; pub mod json; mod pace; pub mod rate; -mod jitter; pub mod select; mod source; pub mod timeline; diff --git a/test/ts/README.md b/test/ts/README.md index 7fe35e01623..3aa8cad533a 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -41,8 +41,8 @@ runs (see [CI](#ci)). The default arm runs `pcr-timing.py` over its capture too, after `compliance.py`, for the one thing the IRD model does not grade: whether the bytes between consecutive PCRs are the ones the mux rate implies -([`pcr-schedule`](#byte-schedule)). It is a shape check, so it reports without -gating unless `--strict`. +([`pcr-schedule`](#byte-schedule)). It gates whenever the rate is known, which is +the generated clip's. The live arm passes only when the grader's verdict *and* the publisher's exit status are clean. The grader can only speak for what reached it, and the sample floor @@ -221,17 +221,22 @@ Give the rate whenever it is known. The estimate is total bytes over total time, so a transient biases every interval by the same amount: over a 20 s live window, the exporter's unpadded first half-second pulled it ~3 % low and read every padded interval after it as off schedule. `run.sh` passes the generated clip's rate -(`--bitrate`), and lets the grader estimate for `--source`. - -The generated clip is a weak fixture for this check. It compresses to almost -nothing, so padding dominates and no keyframe outgrows its slot: measured against -the exporter this check was written for, 92-97 % of intervals were on schedule -over three 20 s runs, the misses being near-empty intervals from the unpadded -start. A real constant-rate capture is the case that -discriminates. A 60 s cut of a 9.95 Mb/s broadcast clip round-tripped through the -same harness came back with a median of 1,316 B between PCRs against 31,081 B -nominal and 3.1 % of intervals within tolerance, while its aggregate rate was -within 16 b/s of nominal: +(`--bitrate`) with `--schedule-pct-min 99`, and lets the grader estimate, and only +report, for `--source`. + +With the rate given, the intervals before the first null packet are not graded. The +exporter pads only once its catalog records the rate, which import measures over the +source's first two seconds, so a subscriber that starts with the publisher begins +unpadded; the report counts those intervals as `unpadded_lead`. + +At the default 10 Mb/s the generated clip compresses to almost nothing, so padding +dominates and no keyframe outgrows its 31 kB slot. At 2 Mb/s its 13-19 kB keyframes +outgrow a 6 kB slot, so the exporter has to spread each over the slots before its +DTS; CI runs that too (`just test ts --bitrate 2000000`). A real constant-rate +capture discriminates further. A 60 s cut of a 9.95 Mb/s broadcast clip +round-tripped through the harness before the export kept a schedule came back with +a median of 1,316 B between PCRs against 31,081 B nominal and 3.1 % of intervals +within tolerance, while its aggregate rate was within 16 b/s of nominal: ```bash just test ts --source cap.ts --duration 60 # reports the schedule, estimating the rate diff --git a/test/ts/pcr-timing.py b/test/ts/pcr-timing.py index aed69a962c0..b26214aa36f 100755 --- a/test/ts/pcr-timing.py +++ b/test/ts/pcr-timing.py @@ -74,6 +74,7 @@ class Scan: def __init__(self): self.pcr = [] # (packet_index, ticks, arrival_or_None, pid) self.packets = 0 + self.first_null = None # the first null packet: a padded stream's schedule starts there self.bad_sync = 0 self.transport_error = 0 self.cc_errors = [] @@ -95,6 +96,8 @@ def feed(self, p, arrival=None): if p[1] & 0x80: self.transport_error += 1 pid = ((p[1] & 0x1F) << 8) | p[2] + if pid == 0x1FFF and self.first_null is None: + self.first_null = index # ISO 13818-1 2.4.3.3 allows the counter to jump when the adaptation field sets # discontinuity_indicator, and allows one packet to be sent twice, repeating its @@ -486,6 +489,11 @@ def check_schedule(scan, args): as a start that is not yet padded, biases every interval's error by the same amount; pass the rate whenever it is known. + With --mux-rate, the intervals before the first null packet are not graded: an + exporter pads only once its catalog records the rate, which import measures over the + source's first two seconds, so a subscriber that starts with the publisher begins + unpadded. The report counts them. + Report-only unless --schedule-pct-min is given: a VBR stream has no schedule to keep, and `export ts` is VBR unless a mux rate is declared, so only the caller knows whether this property was promised. @@ -502,11 +510,15 @@ def check_schedule(scan, args): on_pid = [(k, e) for k, e in enumerate(scan.pcr) if e[3] == pid] graded = [] skipped = 0 + unpadded = 0 for (_, a), (kb, b) in zip(on_pid, on_pid[1:]): seconds = (b[1] - a[1]) / TICKS_PER_MS / 1000.0 # An interval spanning a signalled new time base measures nothing, as in the value # check. A non-positive one is a duplicate packet (legal) or a backwards clock (the # value check's defect to report); neither has a rate to compare against. + if args.mux_rate and scan.first_null is not None and b[0] <= scan.first_null: + unpadded += 1 + continue if kb in scan.new_base or seconds <= 0: skipped += 1 continue @@ -533,6 +545,7 @@ def check_schedule(scan, args): "other_pcr_pids": sorted(p for p in pids if p != pid), "count": len(graded), "skipped": skipped, + "unpadded_lead": unpadded, "rate_bps": round(rate), "rate_source": "declared" if args.mux_rate else "estimated", "aggregate_bps": round(aggregate), diff --git a/test/ts/run.sh b/test/ts/run.sh index 0858b89dadc..9a5fbbd0cf5 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -299,15 +299,15 @@ harness_endpoint relay "$URL" # (a file has none left in it). It stops itself after its own window, so the # round-trip below still bounds the run. -# SCHEDULE carries the rate pcr-timing.py's schedule check grades against. The -# generated clip is muxed at $BITRATE, which is the rate the catalog records and the -# exporter pads to. Left to estimate, the grader divides total bytes by the PCR span, -# and any transient drags that off the true rate: an unpadded first half-second put it -# ~3 % low over a 20 s window, which then read every correctly padded interval as off -# schedule. A --source capture's rate is not known here, so for that the grader -# estimates it and says so. +# SCHEDULE carries the rate pcr-timing.py's schedule check grades against, and gates +# on it. The generated clip is muxed at $BITRATE, which is the rate the catalog records +# and the exporter pads to, on a constant-rate schedule from the first null packet on: +# every interval is the bytes the rate implies, to one packet. Left to estimate, the +# grader divides total bytes by the PCR span, and the unpadded start drags that off the +# true rate. A --source capture's rate is not known here, so for that the grader +# estimates it, says so, and only reports. SCHEDULE=() -[[ -z "$SOURCE" ]] && SCHEDULE=(--mux-rate "$BITRATE") +[[ -z "$SOURCE" ]] && SCHEDULE=(--mux-rate "$BITRATE" --schedule-pct-min 99) # Both halves matter and `wait` can only report one, so record each. The # exporter's own status is not incidental here: it decides whether the grader saw @@ -521,7 +521,7 @@ fi # compliance.py grades rate in aggregate and over fixed windows, neither of which says # whether the bytes between consecutive PCRs are the ones the mux rate implies, so the # capture goes through pcr-timing.py as well. Its hard checks gate here as they do under -# --live; pcr-schedule is a shape check, so it reports without gating unless --strict. +# --live, and so does pcr-schedule when the rate is known (SCHEDULE). echo if ! python3 "$DIR/pcr-timing.py" "$SUB_TS" ${SCHEDULE[@]+"${SCHEDULE[@]}"} $STRICT; then echo >&2 From 17d8a2f49812d5175d1a78c75fa966c66d373fa1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 19:35:14 -0700 Subject: [PATCH 09/40] fix(moq-mux): re-buffer instead of dropping a late sync frame A source that falls behind the delay and stays there would lose every frame after the stall. A keyframe or audio frame past its deadline now re-anchors its generation on itself, pausing the output for the stall. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 5 ++- rs/moq-mux/src/container/ts/export.rs | 10 ++++-- rs/moq-mux/src/container/ts/export_test.rs | 18 +++++----- rs/moq-mux/src/jitter.rs | 40 +++++++++++++++++++++- 4 files changed, 61 insertions(+), 12 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 2c112c22781..a6942dc05c1 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -335,7 +335,10 @@ with the clock started when the first frame arrived, so the output keeps the source's pace and interleaves all tracks in decode order whatever their arrival skew: two exporters of one broadcast emit them in one order. A frame that arrives later than its deadline is dropped, and video then resumes at its next -keyframe. The delay is also how stale a group may get before it is skipped. +keyframe. A keyframe or audio frame that arrives late restarts the clock on +itself instead, so a source that stalls for longer than the delay pauses the +output rather than losing everything after the stall. The delay is also how +stale a group may get before it is skipped. `--delay 0` holds nothing and drops nothing, writing frames in arrival order. A stdout export ends with the broadcast. `export ts --linger 10s` waits that diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 98026801d5c..07702f7990f 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -266,8 +266,14 @@ impl Track { description, }, }; - if jitter.push(self.pid, arrival) == Push::Late { - tracing::warn!(track = %name, dropped = jitter.dropped(), "frame missed its deadline; dropped"); + match jitter.push(self.pid, arrival) { + Push::Late => { + tracing::warn!(track = %name, dropped = jitter.dropped(), "frame missed its deadline; dropped") + } + Push::Rebuffered => { + tracing::warn!(track = %name, "source fell behind the delay; restarting the clock on this frame") + } + Push::Queued | Push::Waiting => {} } } } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 545aa8749c4..6c3b3da0e37 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -3893,8 +3893,8 @@ async fn arrival_order_does_not_change_the_output() { assert!(eager == batched, "arrival order changed the rendering"); } -/// Audio that arrives past its deadline is dropped and counted, and everything else still -/// goes out in decode order. +/// Video that arrives past its deadline is dropped and counted until its next keyframe, and +/// everything else still goes out in decode order. #[tokio::test(start_paused = true)] async fn a_late_frame_is_dropped_and_the_rest_keep_their_order() { let delay = Duration::from_millis(100); @@ -3903,19 +3903,21 @@ async fn a_late_frame_is_dropped_and_the_rest_keep_their_order() { let mut out = Vec::new(); for tick in 0..TICKS / 2 { rig.at(tick * VIDEO_US).await; - rig.video(tick); - out.extend(poll_frames(&mut export)); - // Audio stalls for 400ms, then its backlog lands at once. - if !(20..30).contains(&tick) { - rig.audio_until(tick * VIDEO_US, &mut export, &mut out); + // Video stalls for 320ms, then its backlog lands at once, before the next keyframe. + match tick { + 20..28 => {} + 28 => (20..=28).for_each(|tick| rig.video(tick)), + _ => rig.video(tick), } + out.extend(poll_frames(&mut export)); + rig.audio_until(tick * VIDEO_US, &mut export, &mut out); } rig.finish(); out.extend(drain_frames(&mut export).await); let written = rig.audio_index + TICKS / 2; let decode = pes_decode_in_order(&out); - assert!(export.dropped() > 0, "the stalled audio missed its deadline"); + assert!(export.dropped() > 0, "the stalled video missed its deadline"); assert_eq!( decode.len() as u64, written - export.dropped(), diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index 10d2ac53615..96284972b7c 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -39,6 +39,10 @@ pub(crate) enum Push { Late, /// It was dropped because the track is waiting for a sync frame. Waiting, + /// It missed its deadline but decodes on its own, so the clock moved to put it a whole + /// delay early again: the source fell behind (a stall upstream), and dropping every + /// frame after it would never catch up. + Rebuffered, } /// Holds each track's frames until a fixed delay past their decode time, like an SRT @@ -55,6 +59,11 @@ pub(crate) enum Push { /// though never ahead of a deadline already given out. A backlog read in one go still /// releases each generation after the one before it, and a live stream keeps its delay. /// +/// A sync frame that misses its deadline re-anchors its generation on itself rather than +/// being dropped, like a player re-buffering after a stall: a source that falls behind by +/// more than the delay and stays there would otherwise lose every frame from then on. +/// Output pauses for the stall, and the media timeline does not jump. +/// /// A zero delay holds nothing and drops nothing: each frame goes out as soon as it is /// read, ordered only among the frames read together. pub(crate) struct Buffer { @@ -124,13 +133,21 @@ impl Buffer { self.horizon = self.horizon.max(Some(deadline)); Push::Queued } + Some(_) if arrival.sync => { + self.anchors.insert(generation, (arrival.arrived, arrival.decode)); + let deadline = arrival.arrived + self.delay; + track.waiting = false; + track.queue.push_back((deadline, generation, arrival.item)); + self.horizon = self.horizon.max(Some(deadline)); + Push::Rebuffered + } // Past the deadline, or so far off the clock that no instant holds it. _ => { track.waiting = true; Push::Late } }; - if push != Push::Queued { + if matches!(push, Push::Late | Push::Waiting) { self.dropped += 1; } @@ -339,6 +356,27 @@ mod tests { assert_eq!(due(&mut buffer), ["v0", "a0", "a180", "v240"]); } + /// A sync frame past its deadline re-anchors the clock instead of being dropped, so a + /// source that fell behind carries on a delay behind its new position. + #[tokio::test(start_paused = true)] + async fn a_late_sync_frame_rebuffers() { + let start = Instant::now(); + let mut buffer = Buffer::new(DELAY); + buffer.push(2, arrival(start, 0, "a0")); + // The source stalls for a second, then carries on from where it was. + let stalled = start + Duration::from_secs(1); + assert_eq!(buffer.push(2, arrival(stalled, 20, "a20")), Push::Rebuffered); + assert_eq!(buffer.push(2, arrival(stalled, 40, "a40")), Push::Queued); + assert_eq!(buffer.dropped(), 0); + + tokio::time::advance(Duration::from_secs(1)).await; + assert_eq!(due(&mut buffer), ["a0"]); + tokio::time::advance(DELAY).await; + assert_eq!(due(&mut buffer), ["a20"]); + tokio::time::advance(Duration::from_millis(20)).await; + assert_eq!(due(&mut buffer), ["a40"]); + } + /// A discontinuity re-anchors the track, and a track reaching the same discontinuity /// later joins that anchor rather than making its own. #[tokio::test(start_paused = true)] From dbb4e1f04d7218177cab1b88791507eeabd2ce3a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 19:35:54 -0700 Subject: [PATCH 10/40] quest(m1): name the jitter buffer in the FLV and MKV delay entries Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quest/m1/README.md b/quest/m1/README.md index 0a35642776c..7af0f8092ab 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -147,8 +147,8 @@ QUIC studies there on that rule. - [Audio warmup](/quest/m1/audio-warmup.md) - a viewer joining an Opus rendition mid-stream never hears the unconverged first 80 ms - [#3021](/quest/m1/3021-moq-gst-anchor-generated-media-timelines-to-wall-clock.md) - moq-gst picks the broadcast wall epoch; a restarted source is a new epoch, not a forward re-anchor - [T-STD TS export](/quest/m1/tstd/README.md) - `moq export ts` is a proper remux that passes the T-STD buffer model, starting with a fixed `--delay` -- [FLV export delay](/quest/m1/flv-export-delay.md) - FLV interleaves through the shared fixed-delay release stage -- [MKV export delay](/quest/m1/mkv-export-delay.md) - MKV interleaves through the shared fixed-delay release stage +- [FLV export delay](/quest/m1/flv-export-delay.md) - FLV interleaves through the shared jitter buffer +- [MKV export delay](/quest/m1/mkv-export-delay.md) - MKV interleaves through the shared jitter buffer - [Release profile](/quest/m1/release-profile.md) - every release build gets fat LTO, one codegen unit, and stripping from the workspace profile instead of three script exports - [Size report](/quest/m1/size-report.md) - a nightly job reports every shipped artifact's size, native and JS, and alerts when one grows - [Publish lazy file source](/quest/m1/publish-lazy-file.md) - a camera or screen `` stops downloading mediabunny's ~99 KB gzip From 402500b31e4ac31a30476a32a9e440ddd6af34fc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 20:30:11 -0700 Subject: [PATCH 11/40] fix(moq-mux): drop every late TS frame, as SRT does Reverts the re-buffer on a late keyframe or audio frame. A frame that misses its deadline is dropped and counted, and video waits for its next keyframe. A source that falls permanently behind the delay produces nothing. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 5 +-- rs/moq-mux/src/container/ts/export.rs | 10 ++---- rs/moq-mux/src/container/ts/export_test.rs | 18 +++++----- rs/moq-mux/src/jitter.rs | 40 +--------------------- 4 files changed, 12 insertions(+), 61 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index a6942dc05c1..2c112c22781 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -335,10 +335,7 @@ with the clock started when the first frame arrived, so the output keeps the source's pace and interleaves all tracks in decode order whatever their arrival skew: two exporters of one broadcast emit them in one order. A frame that arrives later than its deadline is dropped, and video then resumes at its next -keyframe. A keyframe or audio frame that arrives late restarts the clock on -itself instead, so a source that stalls for longer than the delay pauses the -output rather than losing everything after the stall. The delay is also how -stale a group may get before it is skipped. +keyframe. The delay is also how stale a group may get before it is skipped. `--delay 0` holds nothing and drops nothing, writing frames in arrival order. A stdout export ends with the broadcast. `export ts --linger 10s` waits that diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 07702f7990f..98026801d5c 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -266,14 +266,8 @@ impl Track { description, }, }; - match jitter.push(self.pid, arrival) { - Push::Late => { - tracing::warn!(track = %name, dropped = jitter.dropped(), "frame missed its deadline; dropped") - } - Push::Rebuffered => { - tracing::warn!(track = %name, "source fell behind the delay; restarting the clock on this frame") - } - Push::Queued | Push::Waiting => {} + if jitter.push(self.pid, arrival) == Push::Late { + tracing::warn!(track = %name, dropped = jitter.dropped(), "frame missed its deadline; dropped"); } } } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 6c3b3da0e37..545aa8749c4 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -3893,8 +3893,8 @@ async fn arrival_order_does_not_change_the_output() { assert!(eager == batched, "arrival order changed the rendering"); } -/// Video that arrives past its deadline is dropped and counted until its next keyframe, and -/// everything else still goes out in decode order. +/// Audio that arrives past its deadline is dropped and counted, and everything else still +/// goes out in decode order. #[tokio::test(start_paused = true)] async fn a_late_frame_is_dropped_and_the_rest_keep_their_order() { let delay = Duration::from_millis(100); @@ -3903,21 +3903,19 @@ async fn a_late_frame_is_dropped_and_the_rest_keep_their_order() { let mut out = Vec::new(); for tick in 0..TICKS / 2 { rig.at(tick * VIDEO_US).await; - // Video stalls for 320ms, then its backlog lands at once, before the next keyframe. - match tick { - 20..28 => {} - 28 => (20..=28).for_each(|tick| rig.video(tick)), - _ => rig.video(tick), - } + rig.video(tick); out.extend(poll_frames(&mut export)); - rig.audio_until(tick * VIDEO_US, &mut export, &mut out); + // Audio stalls for 400ms, then its backlog lands at once. + if !(20..30).contains(&tick) { + rig.audio_until(tick * VIDEO_US, &mut export, &mut out); + } } rig.finish(); out.extend(drain_frames(&mut export).await); let written = rig.audio_index + TICKS / 2; let decode = pes_decode_in_order(&out); - assert!(export.dropped() > 0, "the stalled video missed its deadline"); + assert!(export.dropped() > 0, "the stalled audio missed its deadline"); assert_eq!( decode.len() as u64, written - export.dropped(), diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index 96284972b7c..10d2ac53615 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -39,10 +39,6 @@ pub(crate) enum Push { Late, /// It was dropped because the track is waiting for a sync frame. Waiting, - /// It missed its deadline but decodes on its own, so the clock moved to put it a whole - /// delay early again: the source fell behind (a stall upstream), and dropping every - /// frame after it would never catch up. - Rebuffered, } /// Holds each track's frames until a fixed delay past their decode time, like an SRT @@ -59,11 +55,6 @@ pub(crate) enum Push { /// though never ahead of a deadline already given out. A backlog read in one go still /// releases each generation after the one before it, and a live stream keeps its delay. /// -/// A sync frame that misses its deadline re-anchors its generation on itself rather than -/// being dropped, like a player re-buffering after a stall: a source that falls behind by -/// more than the delay and stays there would otherwise lose every frame from then on. -/// Output pauses for the stall, and the media timeline does not jump. -/// /// A zero delay holds nothing and drops nothing: each frame goes out as soon as it is /// read, ordered only among the frames read together. pub(crate) struct Buffer { @@ -133,21 +124,13 @@ impl Buffer { self.horizon = self.horizon.max(Some(deadline)); Push::Queued } - Some(_) if arrival.sync => { - self.anchors.insert(generation, (arrival.arrived, arrival.decode)); - let deadline = arrival.arrived + self.delay; - track.waiting = false; - track.queue.push_back((deadline, generation, arrival.item)); - self.horizon = self.horizon.max(Some(deadline)); - Push::Rebuffered - } // Past the deadline, or so far off the clock that no instant holds it. _ => { track.waiting = true; Push::Late } }; - if matches!(push, Push::Late | Push::Waiting) { + if push != Push::Queued { self.dropped += 1; } @@ -356,27 +339,6 @@ mod tests { assert_eq!(due(&mut buffer), ["v0", "a0", "a180", "v240"]); } - /// A sync frame past its deadline re-anchors the clock instead of being dropped, so a - /// source that fell behind carries on a delay behind its new position. - #[tokio::test(start_paused = true)] - async fn a_late_sync_frame_rebuffers() { - let start = Instant::now(); - let mut buffer = Buffer::new(DELAY); - buffer.push(2, arrival(start, 0, "a0")); - // The source stalls for a second, then carries on from where it was. - let stalled = start + Duration::from_secs(1); - assert_eq!(buffer.push(2, arrival(stalled, 20, "a20")), Push::Rebuffered); - assert_eq!(buffer.push(2, arrival(stalled, 40, "a40")), Push::Queued); - assert_eq!(buffer.dropped(), 0); - - tokio::time::advance(Duration::from_secs(1)).await; - assert_eq!(due(&mut buffer), ["a0"]); - tokio::time::advance(DELAY).await; - assert_eq!(due(&mut buffer), ["a20"]); - tokio::time::advance(Duration::from_millis(20)).await; - assert_eq!(due(&mut buffer), ["a40"]); - } - /// A discontinuity re-anchors the track, and a track reaching the same discontinuity /// later joins that anchor rather than making its own. #[tokio::test(start_paused = true)] From 70e738359263b7de6fece68f721dde94e1dd34fb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 20:49:35 -0700 Subject: [PATCH 12/40] test(ts): grade the full T-STD buffer model (#4643) Co-authored-by: Claude Opus 5.5 --- .github/workflows/interop.yml | 6 + quest/m1/tstd/README.md | 7 +- quest/m1/tstd/check.md | 25 - quest/m1/tstd/delay.md | 3 + test/justfile | 11 +- test/ts/README.md | 160 ++++- test/ts/compliance.py | 1132 +++++++++++++++++++++++---------- test/ts/pcr-timing.py | 74 +-- test/ts/run.sh | 6 +- test/ts/tstd-controls.py | 220 +++++++ 10 files changed, 1206 insertions(+), 438 deletions(-) delete mode 100644 quest/m1/tstd/check.md create mode 100755 test/ts/tstd-controls.py diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index 74a22e98bd1..5a2fa6b7171 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -138,6 +138,12 @@ jobs: env: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + # The T-STD model must pass a broadcast capture and fail the same capture + # delivered too slowly, too early, or in a burst. + - name: T-STD controls + run: nix develop --command just test ts-tstd + shell: bash -leo pipefail {0} + # A failing harness run keeps its directory, holding the process logs, the # relay config, and a Playwright trace of the failing page. Upload it for a # short while so a red cell is diagnosable; a passing run deletes its own. diff --git a/quest/m1/tstd/README.md b/quest/m1/tstd/README.md index ec8f9dc2cff..f681316236a 100644 --- a/quest/m1/tstd/README.md +++ b/quest/m1/tstd/README.md @@ -16,6 +16,12 @@ Padding, pacing, and muxing all assume a fixed delay, so the export gets one first. t0ms is testing whether T-STD compliance is feasible at all; record the result here. If it isn't, re-plan this line. +The full-model check (`tstd` in `test/ts/compliance.py`) fails today's export +for two reasons, both delivery timing: 7-10 % of video access units finish +arriving after their DTS, and audio runs ~230 ms ahead in bursts that overflow +its TB and B. The media itself fits its buffers, so nothing yet says the bar +is out of reach. + This README owns the end-to-end proof: the #4613 netem rig (10% loss, a real ~10 Mb/s broadcast TS) passes the strict T-STD check, and the recipe runs nightly. @@ -23,5 +29,4 @@ nightly. ## Required - [Fixed-delay release](/quest/m1/tstd/delay.md) - frames go out at media time plus a fixed `--delay`, in one order under loss -- [T-STD check](/quest/m1/tstd/check.md) - the harness grades the full buffer model instead of the transport buffer alone - [TS byte schedule](/quest/m1/tstd/byte-schedule.md) - PCRs sit on the byte grid the mux rate implies, paced against the fixed delay diff --git a/quest/m1/tstd/check.md b/quest/m1/tstd/check.md deleted file mode 100644 index a7c42f511f8..00000000000 --- a/quest/m1/tstd/check.md +++ /dev/null @@ -1,25 +0,0 @@ -# [M] The TS harness grades the full T-STD buffer model - -## Goal - -`test/ts/compliance.py` models every T-STD buffer an elementary stream passes -through, not only the transport buffer. It fails a stream where any buffer -overflows or where an access unit hasn't fully arrived by its DTS, and the TS -recipe runs it strictly against `moq export ts` output in CI. - -## Plan - -Today the `tstd` check models only the transport-buffer (TB) smoothing stage. -It's a shape check that warns unless `--strict` is set (`test/ts/README.md`). -Extend it to the multiplex buffer (MB, for video) and elementary buffer (EB) -with the ISO 13818-1 leak rates and sizes for each stream type and level, and -remove each access unit at its DTS. Timing stays on the stream's own PCR -clock, so the result is deterministic for a file. - -Validate the model before gating: a known-compliant reference TS passes, and -a deliberately bursty one fails. Where TSDuck or another maintained tool -already implements T-STD, prefer it over hand-rolled math. - -Gate: strict `tstd` in the TS recipe for export output. Until -[fixed-delay release](/quest/m1/tstd/delay.md) lands, the check will likely -fail, so land it report-only and flip the gate in that PR. diff --git a/quest/m1/tstd/delay.md b/quest/m1/tstd/delay.md index e07d26616e6..df8b826567a 100644 --- a/quest/m1/tstd/delay.md +++ b/quest/m1/tstd/delay.md @@ -62,6 +62,9 @@ compare with #4618's numbers. Update `doc/bin/cli.md` and the `moq export ts` examples. +Promote `tstd` in `test/ts/compliance.py` from shape to hard, so `just test +ts` fails a round-trip the T-STD model rejects; it reports only until then. + Public API: `ts::Export` takes the delay in place of its max age and loses the hold; breaking, on `dev`. Wire: none. diff --git a/test/justfile b/test/justfile index 4b466640516..523fd57f9c1 100644 --- a/test/justfile +++ b/test/justfile @@ -77,8 +77,8 @@ drill-sensitivity *args: ./drill/sensitivity.sh "$@" # Round-trips a PCR-paced TS through a relay and checks the subscriber's `export -# ts` output with TSDuck plus a custom analyzer (PCR jitter/repetition, -# burstiness, instantaneous bitrate, T-STD). Flags: `--analyze-only file.ts` +# ts` output with TSDuck plus a custom analyzer (the T-STD buffer model, PCR +# spacing and byte schedule). Flags: `--analyze-only file.ts` # (skip the round-trip), `--strict` (fail on shape warnings), `--source file.ts` # (use a real capture), `--live` (grade PCR release timing and byte position off # the pipe instead, which a capture cannot carry; nightly runs this arm), @@ -96,6 +96,13 @@ ts *args: ts-eit *args: ./ts/eit-roundtrip.sh "$@" +# The T-STD model's controls: a broadcast capture must pass, and the same capture +# delivered too slowly, too early, or in a burst must fail. Needs only TSDuck. + +# Prove the T-STD model passes a compliant stream and fails broken ones. +ts-tstd: + ./ts/tstd-controls.py + # Runs the @moq/wasm bindings in headless Chromium against a real relay, one per # protocol flavour. The crate is `#![cfg(target_arch = "wasm32")]`, so nothing in # `just check` or `just rs wasm` gets past compiling it. See wasm/README.md. diff --git a/test/ts/README.md b/test/ts/README.md index 4fb5950b9d8..b875227fd8e 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -7,8 +7,8 @@ runs [TSDuck](https://tsduck.io) plus a custom analyzer over it. This is a diagnostic gate, not just a pass/fail: the exporter ([`rs/moq-mux/src/container/ts/export.rs`](../../rs/moq-mux/src/container/ts/export.rs)) -is VBR, inserts no null packets, and paces PCR once per media frame, so several -broadcast-shape checks are expected to flag. The report quantifies exactly where +pads to a constant rate only once the catalog carries the source's mux rate, so +several broadcast-shape checks are expected to flag. The report quantifies exactly where and by how much. Four instruments live here. `compliance.py` (via `run.sh`) grades a captured file @@ -39,10 +39,10 @@ That is the only arm that can see release timing at all, and it is what nightly runs (see [CI](#ci)). The default arm runs `pcr-timing.py` over its capture too, after `compliance.py`, -for the one thing the IRD model does not grade: whether the bytes between -consecutive PCRs are the ones the mux rate implies -([`pcr-schedule`](#byte-schedule)). It is a shape check, so it reports without -gating unless `--strict`. +for the PCR checks `compliance.py` leaves to it: the value interval (hard), and +whether the bytes between consecutive PCRs are the ones the mux rate implies +([`pcr-schedule`](#byte-schedule), a shape check that reports without gating +unless `--strict`). The live arm passes only when the grader's verdict *and* the publisher's exit status are clean. The grader can only speak for what reached it, and the sample floor @@ -57,7 +57,7 @@ moq --connect http://localhost:4443 --broadcast live.hang export ts > sub.ts ./run.sh --analyze-only sub.ts ``` -Requirements: `tsp` and `tsanalyze` (TSDuck) and `python3` for every mode; the +Requirements: `tsp`, `tsanalyze` and `tstables` (TSDuck) and `python3` for every mode; the round-trip modes also need `cargo`, `ffmpeg`, `curl`, and `timeout`. ## Checks @@ -80,14 +80,8 @@ Severities: **hard** checks fail the run by default; **shape** checks report as | `pcr-presence` | hard | a PCR PID is declared and carries PCR | | `pcr-monotonic` | hard | PCR strictly increases (one 33-bit wrap tolerated), except into a PCR that signals `discontinuity_indicator` | | `duration-fidelity` | hard | exported PCR span tracks the source's duration (round-trip only) | -| `pcr-repetition` | shape | consecutive PCRs within the limit (default 40 ms) | -| `pcr-jitter` | shape | per-interval PCR jitter vs the nominal bitrate (pcrverify model) | -| `null-ratio` | shape | null/stuffing fraction (flags only a pathological excess) | | `service-descriptors` | shape | an SDT naming the service is present | -| `bitrate-consistency` | shape | instantaneous-bitrate spread over 1 ms / 10 ms windows (CBR-ness) | -| `burstiness` | shape | peak/mean of windowed delivery | -| `inter-arrival` | shape | packet inter-arrival spread on the PCR clock (informational) | -| `tstd` | shape | transport-buffer smoothing (TB fills on arrival, leaks at Rx) | +| `tstd` | shape | the full T-STD buffer model: no TB, MB, EB or B overflow, no access unit incomplete at its decoding time (see [T-STD](#t-std)) | Every timing check reads the stream's own PCR, so a PCR emitted on the wrong clock rate stays internally consistent and passes them all. `duration-fidelity` @@ -96,10 +90,111 @@ independent duration, which pins the absolute rate. It runs only on a round-trip (where a source exists); `run.sh` passes the source automatically, and `--analyze-only` skips it. -Thresholds are CLI flags forwarded through `run.sh` (e.g. -`--pcr-repetition-ms`, `--pcr-jitter-us`, `--bitrate-cov-max`, `--burstiness-max`, -`--tb-size-bytes`, `--video-leak-bps`, `--audio-leak-bps`). `--report-json ` -writes the full machine-readable report. +`compliance.py` grades what TSDuck parses and the T-STD model, and leaves PCR +spacing, byte schedule and release timing to `pcr-timing.py`, which honours +signalled discontinuities. `--report-json ` writes the full +machine-readable report. + +## T-STD + +`tstd` runs every audio and video stream through the ISO 13818-1 system target +decoder (2.4.2; Rec. ITU-T H.222.0, whose 10/2014 edition is a free download), +fed on the stream's own PCR clock: + +```text +video (AVC 2.14.3.1, HEVC 2.17.2) TB --Rx--> MB --Rbx (leak)--> EB --DTS--> decoder +audio (2.4.2.3) TB --Rx--> B ----------------PTS--> decoder +``` + +It fails a stream where TB, MB, EB or B overflows, where TB stays occupied for a +second, where an access unit is not wholly in EB/B at its decoding time +(underflow), or where a byte waits longer than the STD delay bound (1 s, 10 s for +AVC/HEVC). The report gives each stream's peak fill per buffer, how late the worst +underflowed access unit finished arriving, and the longest any access unit waited. + +No maintained tool implements this. TSDuck has no T-STD analyzer, and nothing else +in nixpkgs does either, so the model is hand-rolled and its parameters are +transcribed from the specs: H.264 Table A-1 and H.265 Table A.8 for the level, the +ADTS and "other audio" rates and sizes in H.222.0 2.4.2.3, and ATSC A/52 and A/53 +Part 5 for AC-3 and E-AC-3. TSDuck does the parsing: `tstables` decodes the PMT +and `tsp -P pes --avc-access-unit` the SPS, AVC and HEVC alike. + +Video takes its buffers from the HRD the SPS declares, as H.222.0 2.14.3.1 +(AVC) and 2.17.2 (HEVC) specify. A NAL HRD sets Rx from its bit rate and EB to +its CPB size, and MB grows by whatever the level's CPB leaves over; Rbx stays the +level's. Without one, the level's limits stand in. A VCL HRD describes the VCL +alone, not the byte stream EB holds, so a stream declaring only that takes the +level defaults too. An `AVC_timing_and_HRD_descriptor` or +`HEVC_timing_and_HRD_descriptor` with `hrd_management_valid` switches MB-to-EB +transfer to the HRD's own schedule, which is not modelled, so such a stream is +refused. + +Opus is graded against ADTS's buffers for the same channel count: the Opus-in-TS +draft gives Rx (2 Mb/s for 1-2 channels, matching ADTS) but leaves the buffer size +unset, so that size is borrowed rather than specified. A stream with no parameters +at all is refused by name rather than skipped, which fails the check: MPEG-1/2 +video, DVB E-AC-3, and HEVC beyond Main/Main 10. Sections and private data +(SCTE-35, teletext) have no elementary-stream buffers and are listed as not +graded. A signalled PCR discontinuity starts fresh buffers, since the timestamps +on either side of it are on different clocks. + +Every packet on an elementary stream's PID enters TB, including adaptation-only +ones (a PCR, stuffing) and legal duplicates (2.4.3.3: the same counter and +payload twice); only PES bytes of a first copy go on to MB/B. Video access units +are read from the ES rather than taken from PES boundaries: one opens at the first +delimiter, parameter set or SEI after the previous picture's slices (H.264 +7.4.1.2.3, H.265 7.4.2.4.4), and H.222.0 requires a delimiter in each. A PES may +carry several, but only the first takes its timestamp, and deriving the rest from +the stream's own timing is not modelled, so that layout is refused. Video decode +times must strictly increase. + +One simplification, toward strictness: a packet's bytes reach MB/B when its last +byte leaves TB, up to one packet's drain time (0.75 ms for audio) later than +byte-by-byte, so underflow is judged that much stricter. + +### Controls + +`just test ts-tstd` (`tstd-controls.py`) proves the model can tell a compliant +stream from a broken one. The positive control is a real broadcast encoder's +output (`kyrion_dirtystart.ts` from the `moq-mux` test data: AVC High@4.0 with a +1.935 Mb/s CBR NAL HRD and a 755 kbit CPB, plus two MPEG-1 Layer II tracks), which +passes against its own declared buffer, filling EB to the brim as a CBR stream +should. The negatives restamp its PCRs with +`tsp -P pcradjust`, leaving every PES and timestamp alone, so only delivery +changes: + +| Case | Expected | +|---|---| +| as captured | pass | +| PCRs restamped at the capture's own rate | pass | +| delivered at 0.7x | EB and B underflow | +| delivered at 4x | TB and B overflow | +| delivered at 15x (a burst) | TB and B overflow | + +No restamp can overflow the video MB: it holds the level's whole CPB less the +declared one, about 3.6 MB, more than the 4 s capture carries. + +The packet layouts a capture cannot be edited into are built synthetically: a +10 Mb/s single-video stream carrying the Kyrion SPS, one access unit per PES, with +a PCR packet between them. Each case fails without the handling it names: + +| Case | Expected | +|---|---| +| as built | pass | +| two access units in one PES | refused | +| four adaptation-only packets after an access unit | TB overflow | +| an access unit's first packet sent twice | pass | + +The ffmpeg clip `run.sh` generates is not a positive control: its muxer sends +audio 0.7 s ahead by default (`-muxdelay`), which overflows the 3,584-byte ADTS +buffer, and even at 0.1 s a four-packet audio burst overflows TB. + +### Gate + +`tstd` is a shape check, so `just test ts` reports it without failing. `export +ts` output does not pass yet: some video access units arrive after their DTS, and +audio runs far enough ahead, in bursts, to overflow both TB and B. The change that +makes the exporter release on a fixed delay promotes `tstd` to a hard check. ## PCR timing (`pcr-timing.py`) @@ -112,11 +207,9 @@ cannot see the other two: | release | the bytes carrying a PCR were handed over when that PCR asserts | arrival stamps | | position | a PCR packet sits among the media bytes it describes | packet offsets | -`compliance.py` grades `value` from a file, deterministically and with no -wall-clock capture, which is the right basis for the model math it does. That -also means it cannot grade `release`: a change to *when* the exporter hands bytes -over is invisible to any harness that does not stamp arrivals. -`pcr-timing.py` reads a pipe and grades all three in one pass. +`pcr-timing.py` grades `value` and `position` from a file, and all three from a +pipe. A file carries no arrival stamps, so a change to *when* the exporter hands +bytes over is invisible to any harness that does not stamp them. A constant-rate stream makes a fourth claim, graded by `pcr-schedule`: that the bytes between consecutive PCRs are the bytes the mux rate implies for that @@ -143,13 +236,19 @@ grades only how evenly the bytes are laid over the PCRs. | Check | Severity | What it verifies | |---|---|---| -| `sync` | hard | no invalid sync bytes / transport-error packets | -| `continuity` | hard | no discontinuities, and a payload-less packet must not advance the counter (ISO 13818-1 2.4.3.3) | -| `pcr-single-pid` | hard | every PCR rides one PID | +| `sync` | hard | no invalid sync bytes / transport-error packets (`--live` only) | +| `continuity` | hard | no discontinuities, and a payload-less packet must not advance the counter (ISO 13818-1 2.4.3.3) (`--live` only) | | `pcr-value-interval` | hard | no interval above `--repetition-ms` (default 40, TR 101 290), within one time base | | `pcr-release-timing` | hard | no more than `--release-pct-max` of intervals arrive further than `--release-ms` from the interval their own values assert, and accumulated drift stays within `--drift-ms`, being the standing lag the sender is allowed to hold; a sample below `--live-min-pcr` PCRs or `--live-cover-pct` of the window is a failure, not a pass (`--live` only) | | `pcr-position` | shape | share of PCR packets within `--adjacent-packets` of the previous one | -| `pcr-schedule` | shape | share of PCR intervals, on the busiest PCR PID, whose bytes are within `--schedule-tolerance-pct` (default 1) or one packet of what `--mux-rate` implies (estimated from the capture if not given); hard, at that share, when `--schedule-pct-min` is given | +| `pcr-schedule` | shape | share of PCR intervals whose bytes are within `--schedule-tolerance-pct` (default 1) or one packet of what `--mux-rate` implies (estimated from the capture if not given); hard, at that share, when `--schedule-pct-min` is given | + +A stream carrying several PCR PIDs (one per program) is graded on the busiest, +since two correct grids offset from one another pool into one that neither keeps. +`sync` and `continuity` run only under `--live`: on a file, `compliance.py` +grades both through TSDuck's `tsanalyze`, which also catches a payload-less +packet advancing the counter. A pipe cannot go through TSDuck first without +rebuffering the arrivals `release` stamps. Accumulated drift has two shapes and only one is a defect, so the check bounds the total and reports the rate over the tail of the sample beside it. A sender @@ -444,8 +543,8 @@ exporter re-emits SI on its own repetition cadence rather than the source's. ## CI `.github/workflows/interop.yml` runs `just test ts`, `just test ts --open-gop`, -and `just test ts-eit` after the interop matrix (nightly, on demand, and on PRs -touching `test/ts/`). +`just test ts-eit`, and `just test ts-tstd` after the interop matrix (nightly, on +demand, and on PRs touching `test/ts/`). `ts-eit` is `eit-roundtrip.sh`: it builds the sparse-schedule and pending-version fixtures from a generated clip, round-trips them through a relay, and censuses the capture, so a break in the generators or in the SI @@ -469,6 +568,3 @@ that from a pipe running slow. - Wall-clock delivery jitter/burstiness is out of scope *for `compliance.py`*: all of its timing is derived from the stream's PCR, not from arrival times. `pcr-timing.py --live` covers that axis separately, by stamping a pipe. -- `tstd` models only the transport-buffer (TB) smoothing stage of the ISO 13818-1 - T-STD, not the full multiplex/elementary decode buffers. Its leak rates are - defaults, not level-derived, so treat overflow as a smell rather than proof. diff --git a/test/ts/compliance.py b/test/ts/compliance.py index 66aae98fe7d..0609857cf72 100755 --- a/test/ts/compliance.py +++ b/test/ts/compliance.py @@ -6,19 +6,17 @@ summary, exiting non-zero on failure. Division of labour: TSDuck does the transport-stream parsing (we shell out to -`tsanalyze --json` for PSI/service/structure), and this script does the model -math TSDuck does not cover directly (PCR jitter/repetition, packet -inter-arrival, burstiness, instantaneous bitrate, and a transport-buffer model). -The PCR/PTS/DTS timeline the timing model needs comes from a 188/204-byte -packet-header scan done here, which also gives the per-packet PID that -`tsp -P pcrextract` does not expose. +`tsanalyze --json` for PSI/service/structure and `tstables` for the PMT), and +this script does what TSDuck does not cover: the ISO 13818-1 T-STD buffer +model, which no maintained tool implements, and the PCR checks that need the +time-base discontinuities a 188/204-byte header scan here recovers. PCR value +intervals, byte schedule and release timing live in `pcr-timing.py`. Checks split into two severities: - HARD (structural): fail the run by default. PAT/PMT, packet size, sync, continuity counters, PSI CRC, PCR presence, PCR monotonicity. - SHAPE (broadcast profile): reported as WARN and only fail the run under - `--strict`. PCR repetition interval, PCR jitter, null-packet ratio, bitrate - consistency / burstiness, service descriptors (SDT), transport-buffer model. + `--strict`. Service descriptors (SDT), T-STD buffer model. Timing basis is the stream's own PCR clock (an IRD locks to PCR), so the harness needs no wall-clock capture and results are deterministic for a given file. @@ -39,7 +37,6 @@ PTS_HZ = 90_000 # The PCR base field is 33 bits at 90 kHz, so the full 27 MHz PCR wraps here. PCR_WRAP = (1 << 33) * 300 -NULL_PID = 0x1FFF class Status(Enum): @@ -68,22 +65,6 @@ class Check: metrics: dict = field(default_factory=dict) -@dataclass -class Thresholds: - """IRD limits and T-STD model parameters, all overridable from the CLI.""" - - pcr_repetition_ms: float = 40.0 - pcr_jitter_us: float = 500.0 - null_ratio_max: float = 0.90 - bitrate_cov_max: float = 0.10 - burstiness_max: float = 3.0 - tb_size_bytes: int = 512 - video_leak_bps: float = 1.8e7 - audio_leak_bps: float = 2.0e6 - data_leak_bps: float = 1.0e6 - inst_windows_ms: tuple[float, ...] = (1.0, 10.0) - - # --------------------------------------------------------------------------- IO @@ -106,10 +87,7 @@ def run_tsanalyze(ts_path: str) -> dict: class Scan: """Per-packet facts recovered from a raw header scan of the TS file.""" - packet_size: int total_packets: int - # (ts_index, payload_bytes) per PID, in file order. - pid_packets: dict[int, list[tuple[int, int]]] # (ts_index, pcr_27mhz) samples for every PID that carries PCR. pcr_by_pid: dict[int, list[tuple[int, int]]] # ts_index of every PCR that starts a new time base (discontinuity_indicator). @@ -117,15 +95,14 @@ class Scan: def scan_packets(ts_path: str, packet_size: int) -> Scan: - """Walk the TS packet by packet, recovering PID, payload size, and PCR values. + """Walk the TS packet by packet, recovering the PCR timeline. - This is a header-only scan (no PES/PSI parsing): enough for the timing and - buffer models, which need per-packet PID and the PCR timeline. + A header-only scan: the PCR samples and which of them start a new time base, + which `tsp -P pcrextract` does not report. """ with open(ts_path, "rb") as handle: data = handle.read() - pid_packets: dict[int, list[tuple[int, int]]] = {} pcr_by_pid: dict[int, list[tuple[int, int]]] = {} pcr_new_base: set[int] = set() # PIDs whose discontinuity_indicator rode a packet without a PCR, so the new time @@ -147,8 +124,6 @@ def scan_packets(ts_path: str, packet_size: int) -> Scan: pid = ((b1 & 0x1F) << 8) | b2 afc = (b3 >> 4) & 0x3 - payload_len = 0 - af_len = 0 if afc in (2, 3): af_len = data[offset + 4] if af_len > 0: @@ -168,21 +143,12 @@ def scan_packets(ts_path: str, packet_size: int) -> Scan: if pid in pending_base: pending_base.discard(pid) pcr_new_base.add(index) - if afc in (1, 3): - # Payload = 184 minus the adaptation field (its length byte + body). - consumed = (1 + af_len) if afc == 3 else 0 - payload_len = 184 - consumed - - if pid != NULL_PID: - pid_packets.setdefault(pid, []).append((index, max(0, payload_len))) index += 1 offset += packet_size return Scan( - packet_size=packet_size, total_packets=index, - pid_packets=pid_packets, pcr_by_pid=pcr_by_pid, pcr_new_base=pcr_new_base, ) @@ -191,25 +157,6 @@ def scan_packets(ts_path: str, packet_size: int) -> Scan: # --------------------------------------------------------------- small helpers -def percentile(values: list[float], pct: float) -> float: - """Linear-interpolated percentile of an unsorted list (0..100). 0 for empty.""" - if not values: - return 0.0 - ordered = sorted(values) - if len(ordered) == 1: - return ordered[0] - rank = (pct / 100.0) * (len(ordered) - 1) - low = int(rank) - high = min(low + 1, len(ordered) - 1) - frac = rank - low - return ordered[low] * (1 - frac) + ordered[high] * frac - - -def mean(values: list[float]) -> float: - """Arithmetic mean, 0 for an empty list.""" - return sum(values) / len(values) if values else 0.0 - - def pcr_seconds(pcr_27mhz: int) -> float: """Convert a 27 MHz PCR value to seconds.""" return pcr_27mhz / PCR_HZ @@ -367,89 +314,6 @@ def check_pcr_monotonic(scan: Scan) -> Check: # ------------------------------------------------------------------- shape checks -def check_pcr_repetition(scan: Scan, th: Thresholds) -> Check: - """Consecutive PCRs on a PID should be no more than `pcr_repetition_ms` apart.""" - worst_ms = 0.0 - intervals: list[float] = [] - over = 0 - for samples in scan.pcr_by_pid.values(): - prev = None - for _i, pcr in samples: - if prev is not None: - delta = pcr - prev - if delta <= 0: - prev = pcr - continue - ms = pcr_seconds(delta) * 1000.0 - intervals.append(ms) - worst_ms = max(worst_ms, ms) - if ms > th.pcr_repetition_ms: - over += 1 - prev = pcr - metrics = { - "max_interval_ms": round(worst_ms, 3), - "mean_interval_ms": round(mean(intervals), 3), - "intervals_over_limit": over, - "limit_ms": th.pcr_repetition_ms, - } - detail = f"max {worst_ms:.1f} ms (limit {th.pcr_repetition_ms:.0f} ms), {over} over" - status = Status.PASS if over == 0 else Status.WARN - return Check("pcr-repetition", Severity.SHAPE, status, detail, metrics) - - -def check_pcr_jitter(scan: Scan, ts_bitrate: float, th: Thresholds) -> Check: - """Per-interval PCR jitter vs the nominal bitrate (pcrverify's model). - - For a true CBR mux the actual PCR delta matches the byte delta clocked at the - stream bitrate; the difference is the jitter. On a VBR stream it is large by - construction, which is exactly the IRD-relevant signal. - """ - if ts_bitrate <= 0: - return Check("pcr-jitter", Severity.SHAPE, Status.WARN, "unknown bitrate", {}) - bits_per_packet = scan.packet_size * 8 - jitters_us: list[float] = [] - for samples in scan.pcr_by_pid.values(): - prev = None - for i, pcr in samples: - if prev is not None: - pi, ppcr = prev - d_pcr = pcr - ppcr - if d_pcr <= 0: - prev = (i, pcr) - continue - expected_s = (i - pi) * bits_per_packet / ts_bitrate - actual_s = pcr_seconds(d_pcr) - jitters_us.append((actual_s - expected_s) * 1e6) - prev = (i, pcr) - if not jitters_us: - return Check("pcr-jitter", Severity.SHAPE, Status.WARN, "no PCR intervals", {}) - abs_jit = [abs(j) for j in jitters_us] - max_us = max(abs_jit) - p95 = percentile(abs_jit, 95) - metrics = { - "max_abs_us": round(max_us, 1), - "p95_abs_us": round(p95, 1), - "limit_us": th.pcr_jitter_us, - } - detail = f"max |jitter| {max_us:.0f} us, p95 {p95:.0f} us (limit {th.pcr_jitter_us:.0f} us)" - status = Status.PASS if max_us <= th.pcr_jitter_us else Status.WARN - return Check("pcr-jitter", Severity.SHAPE, status, detail, metrics) - - -def check_null_ratio(analysis: dict, th: Thresholds) -> Check: - """Report the null-packet (stuffing) fraction; flag only a pathological excess.""" - total = analysis["ts"]["packets"]["total"] or 1 - null = 0 - for pid in analysis.get("pids", []): - if pid["id"] == NULL_PID: - null = pid.get("packets", {}).get("total", 0) - ratio = null / total - metrics = {"null_ratio": round(ratio, 4), "null_packets": null, "limit": th.null_ratio_max} - detail = f"{ratio * 100:.2f}% null packets" - status = Status.PASS if ratio <= th.null_ratio_max else Status.WARN - return Check("null-ratio", Severity.SHAPE, status, detail, metrics) - - def check_service_descriptors(analysis: dict) -> Check: """An IRD expects an SDT naming the service; PAT/PMT-only streams get a WARN.""" tables = analysis.get("tables", []) @@ -468,152 +332,804 @@ def check_service_descriptors(analysis: dict) -> Check: ) -def windowed_bitrates(times: list[float], sizes: list[int], window_s: float) -> list[float]: - """Bytes-per-window converted to bit/s, over contiguous windows spanning the capture.""" - if not times: - return [] - start, end = times[0], times[-1] - if end <= start: - return [] - n_windows = max(1, int((end - start) / window_s) + 1) - buckets = [0] * n_windows - for t, size in zip(times, sizes): - idx = min(n_windows - 1, int((t - start) / window_s)) - buckets[idx] += size - # Drop the last (partial) window so a short tail doesn't skew the minimum. - if n_windows > 1: - buckets = buckets[:-1] - return [b * 8 / window_s for b in buckets] - - -def check_bitrate_and_burstiness(scan: Scan, clock: PcrClock, ts_bitrate: float, th: Thresholds) -> tuple[Check, Check]: - """Instantaneous bitrate spread (CBR-ness) and delivery burstiness, on the PCR clock.""" - # Every non-null packet, timed on the PCR clock, weighted by its full 188 bytes. - events: list[tuple[float, int]] = [] - for _pid, packets in scan.pid_packets.items(): - for i, _payload in packets: - events.append((clock.time_at(i), scan.packet_size)) - events.sort(key=lambda e: e[0]) - times = [t for t, _ in events] - sizes = [s for _, s in events] - - inst_metrics: dict = {"nominal_bps": round(ts_bitrate)} - worst_cov = 0.0 - worst_burst = 0.0 - for window_ms in th.inst_windows_ms: - rates = windowed_bitrates(times, sizes, window_ms / 1000.0) - if not rates: +# ---------------------------------------------------------------- T-STD model +# +# ISO/IEC 13818-1 2.4.2 (Rec. ITU-T H.222.0 10/2014, free from the ITU), fed each +# elementary stream's packets at the times its program's PCR assigns them: +# +# video (AVC 2.14.3.1, HEVC 2.17.2) TB -Rx-> MB -Rbx (leak)-> EB -DTS-> decoder +# audio (2.4.2.3) TB -Rx-> B ---------------PTS-> decoder +# +# and graded on 2.4.2.6 and its AVC/HEVC counterparts: no buffer overflows, TB +# empties at least once a second, every access unit is complete in EB/B at its +# decoding time, and no byte waits longer than the STD delay bound. +# +# No maintained tool implements this (TSDuck has no T-STD analyzer), so the +# parameters are transcribed from the specs named beside each table. + +TB_SIZE = 512 +# 2.4.2.6: TB must empty at least once a second. +TB_EMPTY_S = 1.0 +# PTS/DTS are 33 bits at 90 kHz. +STAMP_WRAP_S = (1 << 33) / PTS_HZ + +# H.264 Table A-1: level_idc -> (MaxBR, MaxCPB) as tabulated. H.222.0 2.14.3.1 +# scales both by 1200 bits for the buffers, and Rx by the profile's cpbBrNalFactor. +AVC_LEVELS = { + 10: (64, 175), + 11: (192, 500), + 12: (384, 1000), + 13: (768, 2000), + 20: (2000, 2000), + 21: (4000, 4000), + 22: (4000, 4000), + 30: (10000, 10000), + 31: (14000, 14000), + 32: (20000, 20000), + 40: (20000, 25000), + 41: (50000, 62500), + 42: (50000, 62500), + 50: (135000, 135000), + 51: (240000, 240000), + 52: (240000, 240000), + 60: (240000, 240000), + 61: (480000, 480000), + 62: (800000, 800000), +} +AVC_LEVEL_1B = (128, 350) +# H.264 Table A-2 cpbBrNalFactor, by profile_idc: the default BitRate that sets Rx. +AVC_NAL_FACTOR = {66: 1200, 77: 1200, 88: 1200, 100: 1500, 110: 3600, 122: 4800, 244: 4800, 44: 4800} + +# H.265 Table A.8: general_level_idc -> ((MaxBR, MaxCPB) Main tier, (MaxBR, MaxCPB) High tier). +HEVC_LEVELS = { + 30: ((128, 350), None), + 60: ((1500, 1500), None), + 63: ((3000, 3000), None), + 90: ((6000, 6000), None), + 93: ((10000, 10000), None), + 120: ((12000, 12000), (30000, 30000)), + 123: ((20000, 20000), (50000, 50000)), + 150: ((25000, 25000), (100000, 100000)), + 153: ((40000, 40000), (160000, 160000)), + 156: ((60000, 60000), (240000, 240000)), + 180: ((60000, 60000), (240000, 240000)), + 183: ((120000, 120000), (480000, 480000)), + 186: ((240000, 240000), (800000, 800000)), +} +# H.265 Table A.9 CpbNalFactor for Main, Main 10 and Main Still Picture (profile_idc 1-3). +HEVC_NAL_FACTOR = 1100 + +# H.222.0 2.4.2.3: ADTS (Rx, BSn) by channels, the LFE not counted. +ADTS_BUFFERS = ((2, 2_000_000, 3584), (8, 5_529_600, 8976), (12, 8_294_400, 12804), (48, 33_177_600, 51216)) +# channel_configuration -> full-bandwidth channels (5.1 and 7.1 drop the LFE). +ADTS_CHANNELS = {1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 5, 7: 7} +ADTS_RATES = (96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350) +# 2.4.2.3 "other audio". +AUDIO_RX = 2_000_000 +MPEG_AUDIO_BS = 3584 +# ATSC A/53 Part 5 5.7 (AC-3) and A/52 Annex G 3.6.1 (E-AC-3: 736 + 64 + 12096). +AC3_ATSC_BS = 2592 +EAC3_ATSC_BS = 12896 +# ATSC A/52 Annex A 5.4: AC-3 carried as DVB private data. +AC3_DVB_BS = 5696 + +MPEG_AUDIO_KBPS = { + (1, 1): (0, 32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, 448), + (1, 2): (0, 32, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384), + (1, 3): (0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320), + (2, 1): (0, 32, 48, 56, 64, 80, 96, 112, 128, 144, 160, 176, 192, 224, 256), + (2, 2): (0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160), + (2, 3): (0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160), +} +# stream_type -> model, for the types identified by stream_type alone. +STREAM_KINDS = { + 0x1B: "avc", + 0x24: "hevc", + 0x0F: "adts", + 0x03: "mpeg-audio", + 0x04: "mpeg-audio", + 0x81: "ac3-atsc", + 0x87: "eac3-atsc", +} +AC3_KBPS = (32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384, 448, 512, 576, 640) + + +class Refused(Exception): + """An elementary stream the model cannot grade: named, never skipped silently.""" + + +@dataclass +class Params: + """One elementary stream's T-STD buffers (bytes) and rates (bit/s).""" + + label: str + rx: float + # Video: MB and EB with the leak rate between them. Audio: the main buffer B alone. + mb: float = 0.0 + eb: float = 0.0 + rbx: float = 0.0 + b: float = 0.0 + # tdn(j) - t(i) bound: 10 s for AVC and HEVC, 1 s otherwise (2.4.2.6, 2.14.3.1, 2.17.2). + max_delay_s: float = 1.0 + + @property + def video(self) -> bool: + return self.rbx > 0 + + +@dataclass +class AccessUnit: + """ES byte span [start, end), where its first byte arrived, and its decode time.""" + + start: int + end: int + first_packet: int + # 90 kHz DTS (or PTS) when the PES stamps it, else derived from the one before. + stamp: int | None + duration_s: float = 0.0 + + +@dataclass +class Stream: + """One elementary stream's packets and PES timestamps, and its ES bytes where needed.""" + + pid: int + stream_type: int + # TSDuck's names for the PMT descriptors, plus "registration:" + # and, for one TSDuck has no name for, "tag:". + descriptors: set[str] + # (ts_index, PES header bytes, ES payload bytes, ES offset before this packet). + packets: list[tuple[int, int, int, int]] = field(default_factory=list) + # (ES offset, 90 kHz PTS or None, DTS or None, ts_index) per PES, from the first timestamped one. + pes: list[tuple[int, int | None, int | None, int]] = field(default_factory=list) + es: bytearray = field(default_factory=bytearray) + es_len: int = 0 + # Video: the first SPS as TSDuck decodes it. + sps: dict[str, str] = field(default_factory=dict) + + +def read_programs(ts_path: str) -> list[tuple[int, list[Stream]]]: + """(PCR PID, elementary streams) of each program's first PMT, as TSDuck decodes it.""" + proc = subprocess.run( + ["tstables", ts_path, "--psi-si", "--tid", "2", "--json-output", "-"], + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + raise RuntimeError(f"tstables failed: {proc.stderr.strip()}") + programs: dict[int, tuple[int, list[Stream]]] = {} + for pmt in json.loads(proc.stdout or "[]"): + if pmt.get("service_id") in programs: continue - avg = mean(rates) - peak = max(rates) - low = min(rates) - var = mean([(r - avg) ** 2 for r in rates]) - cov = (var**0.5 / avg) if avg else 0.0 - burst = (peak / avg) if avg else 0.0 - worst_cov = max(worst_cov, cov) - worst_burst = max(worst_burst, burst) - inst_metrics[f"w{int(window_ms)}ms"] = { - "min_bps": round(low), - "mean_bps": round(avg), - "max_bps": round(peak), - "p95_bps": round(percentile(rates, 95)), - "cov": round(cov, 3), - "peak_over_mean": round(burst, 2), - } + streams = [] + for node in pmt.get("#nodes", []): + if node.get("#name") != "component": + continue + descriptors = [d for d in node.get("#nodes", []) if isinstance(d, dict)] + names = {d["#name"] for d in descriptors} + names |= {f"registration:{d['format_identifier']}" for d in descriptors if "format_identifier" in d} + names |= {"hrd_management_valid" for d in descriptors if d.get("hrd_management_valid") is True} + names |= { + f"tag{d['tag']}:" + "".join(d.get("#nodes", [])).replace(" ", "").lower() + for d in descriptors + if d["#name"] == "generic_descriptor" + } + streams.append(Stream(node["elementary_pid"], node["stream_type"], names)) + programs[pmt.get("service_id")] = (pmt["pcr_pid"], streams) + return list(programs.values()) + + +def _stamp(b: bytes) -> int: + """A 33-bit PTS/DTS from its 5-byte PES encoding.""" + return ((b[0] >> 1) & 0x07) << 30 | b[1] << 22 | (b[2] >> 1) << 15 | b[3] << 7 | b[4] >> 1 + + +def read_pes(data: bytes, packet_size: int, streams: dict[int, Stream]) -> dict[int, str]: + """Split each stream's packets into PES header and ES bytes, and note every PES start. + + Every packet on the PID enters TB (H.222.0 2.4.2.3), but only PES bytes go on to + MB/B. So an adaptation-only packet (a PCR, stuffing) costs TB and nothing after it, + as does a duplicate (2.4.3.3: the same counter and payload twice), which "is not + delivered" downstream. Packets before a PID's first timestamped PES likewise stop at + TB: their bytes belong to an access unit whose start was never captured. + Returns why each stream that could not be read was refused. + """ + refused: dict[int, str] = {} + last: dict[int, tuple[int, bytes]] = {} # PID -> (continuity_counter, payload) + for index, offset in enumerate(range(0, len(data) - packet_size + 1, packet_size)): + if data[offset] != 0x47: + continue + pid = ((data[offset + 1] & 0x1F) << 8) | data[offset + 2] + stream = streams.get(pid) + if stream is None or pid in refused: + continue + afc = (data[offset + 3] >> 4) & 0x3 + start = offset + 4 + (1 + data[offset + 4] if afc & 0x2 else 0) + payload = data[start : offset + 188] if afc & 0x1 else b"" + cc = data[offset + 3] & 0x0F + duplicate = bool(payload) and last.get(pid) == (cc, payload) + if payload: + last[pid] = (cc, payload) + if not payload or duplicate or not (stream.pes or data[offset + 1] & 0x40): + stream.packets.append((index, 0, 0, stream.es_len)) + continue + length = len(payload) + header = 0 + if data[offset + 1] & 0x40: + if payload[:3] != b"\x00\x00\x01": + refused[pid] = f"packet {index} starts a payload without a PES start code" + continue + if length < 9 or 9 + payload[8] > length: + refused[pid] = f"the PES header at packet {index} spans packets" + continue + header = 9 + payload[8] + flags = payload[7] + pts = _stamp(payload[9:14]) if flags & 0x80 else None + dts = _stamp(payload[14:19]) if flags & 0xC0 == 0xC0 else None + if pts is None and not stream.pes: + stream.packets.append((index, 0, 0, 0)) + continue + stream.pes.append((stream.es_len, pts, dts, index)) + body = length - header + stream.packets.append((index, header, body, stream.es_len)) + stream.es += payload[header:] + stream.es_len += body + return refused - bitrate_status = Status.PASS if worst_cov <= th.bitrate_cov_max else Status.WARN - bitrate = Check( - "bitrate-consistency", - Severity.SHAPE, - bitrate_status, - f"worst CoV {worst_cov:.2f} (limit {th.bitrate_cov_max:.2f})", - inst_metrics | {"worst_cov": round(worst_cov, 3)}, + +def read_sps(ts_path: str, pid: int, nal_type: int) -> dict[str, str]: + """The first SPS on `pid`, field by field, as TSDuck's `pes` plugin decodes it.""" + proc = subprocess.run( + ["tsp", "-I", "file", ts_path, "-P", "pes", "--pid", str(pid), "--avc-access-unit"] + + ["--nal-unit-type", str(nal_type), "--max-dump-count", "1", "-O", "drop"], + capture_output=True, + text=True, + check=False, ) - burst_status = Status.PASS if worst_burst <= th.burstiness_max else Status.WARN - burst = Check( - "burstiness", - Severity.SHAPE, - burst_status, - f"peak/mean {worst_burst:.2f} (limit {th.burstiness_max:.2f})", - {"worst_peak_over_mean": round(worst_burst, 2), "limit": th.burstiness_max}, + if proc.returncode != 0: + raise RuntimeError(f"tsp -P pes failed: {proc.stderr.strip()}") + fields = {} + for line in (proc.stdout + proc.stderr).splitlines(): + key, eq, value = line.strip().partition(" = ") + # A key repeats once per CPB schedule; the last is the one H.222.0 sizes EB from. + if eq: + fields[key] = value + if not fields: + raise Refused("the stream carries no SPS") + return fields + + +def _hrd(sps: dict[str, str], prefix: str, value: str) -> tuple[float, float] | None: + """(BitRate bit/s, CpbSize bits) of the highest-numbered CPB a NAL HRD declares. + + H.264 E.2.2 and H.265 E.3.3: BitRate = (bit_rate_value_minus1 + 1) * 2^(6 + bit_rate_scale) + and CpbSize = (cpb_size_value_minus1 + 1) * 2^(4 + cpb_size_scale). H.222.0 2.14.3.1 + sizes EB from CpbSize[cpb_cnt_minus1], so the last schedule is the one that counts. + """ + rates = [int(v) for k, v in sps.items() if k.startswith(prefix) and "bit_rate_value_minus1" in k] + sizes = [int(v) for k, v in sps.items() if k.startswith(prefix) and "cpb_size_value_minus1" in k] + if not rates or not sizes: + return None + rate = (rates[-1] + 1) << (6 + int(sps[f"{value}bit_rate_scale"])) + size = (sizes[-1] + 1) << (4 + int(sps[f"{value}cpb_size_scale"])) + return rate, size + + +def avc_params(sps: dict[str, str]) -> Params: + """H.222.0 2.14.3.1 buffers from the SPS: level limits, overridden by a declared NAL HRD. + + Only the NAL HRD counts: EB sizes the byte stream, which a VCL HRD does not describe, + so a stream declaring only a VCL HRD takes the level defaults (H.264 E.2.2). + """ + profile, level = int(sps["profile_idc"]), int(sps["level_idc"]) + if (level == 11 and sps.get("constraint_set3_flag") == "1" and profile in (66, 77, 88)) or level == 9: + max_br, max_cpb, name = *AVC_LEVEL_1B, "1b" + elif level in AVC_LEVELS: + max_br, max_cpb = AVC_LEVELS[level] + name = f"{level // 10}.{level % 10}" + else: + raise Refused(f"AVC level_idc {level} has no Table A-1 entry") + if profile not in AVC_NAL_FACTOR: + raise Refused(f"AVC profile_idc {profile} has no Table A-2 cpbBrNalFactor") + # Absent a NAL HRD, BitRate is cpbBrNalFactor * MaxBR (H.264 E.2.2) and cpb_size is + # 1200 * MaxCPB (H.222.0 2.14.3.1). + declared = ( + _hrd(sps, "vui.nal_hrd.", "vui.nal_hrd.") if sps.get("vui.nal_hrd_parameters_present_flag") == "1" else None ) - return bitrate, burst - - -def check_inter_arrival(scan: Scan, clock: PcrClock) -> Check: - """Report the packet inter-arrival spread on the PCR clock (informational).""" - indices = sorted(i for packets in scan.pid_packets.values() for i, _ in packets) - gaps_us: list[float] = [] - prev = None - for i in indices: - t = clock.time_at(i) - if prev is not None and t >= prev: - gaps_us.append((t - prev) * 1e6) - prev = t - if not gaps_us: - return Check("inter-arrival", Severity.SHAPE, Status.WARN, "no packets timed", {}) - metrics = { - "mean_us": round(mean(gaps_us), 2), - "p95_us": round(percentile(gaps_us, 95), 2), - "max_us": round(max(gaps_us), 2), - } - return Check( - "inter-arrival", - Severity.SHAPE, - Status.PASS, - f"mean {metrics['mean_us']:.1f} us, p95 {metrics['p95_us']:.1f} us, max {metrics['max_us']:.1f} us", - metrics, + bit_rate, cpb = declared or (AVC_NAL_FACTOR[profile] * max_br, 1200 * max_cpb) + overhead = (0.004 + 1 / 750) * max(1200 * max_br, 2_000_000) + return Params( + label=f"AVC profile {profile} level {name}, " + + (f"NAL HRD {bit_rate / 1e6:.3f} Mb/s cpb {cpb / 1e3:.0f} kbit" if declared else "level defaults"), + # H.222.0 2.14.3.1: Rx = 1.2 * BitRate[SchedSelIdx]; MBS = BSmux + BSoh + 1200 * + # MaxCPB - cpb_size; EBS = cpb_size; Rbx = 1200 * MaxBR whatever the HRD declares. + rx=1.2 * bit_rate, + mb=(overhead + 1200 * max_cpb - cpb) / 8, + eb=cpb / 8, + rbx=1200 * max_br, + max_delay_s=10.0, ) -def check_tstd(analysis: dict, scan: Scan, clock: PcrClock, th: Thresholds) -> Check: - """Approximate T-STD transport-buffer check: TB fills on arrival, leaks at Rx. +def hevc_params(sps: dict[str, str]) -> Params: + """H.222.0 2.17.2 buffers from the SPS: tier and level limits, overridden by a declared NAL HRD.""" + prefix = "profile_tier_level.general_" + tier, profile, level = (int(sps[f"{prefix}{k}"]) for k in ("tier_flag", "profile_idc", "level_idc")) + if profile not in (1, 2, 3): + raise Refused(f"HEVC general_profile_idc {profile} has no CpbNalFactor here (Main, Main 10, Still only)") + limits = HEVC_LEVELS.get(level, (None, None))[tier] + if limits is None: + raise Refused(f"HEVC level_idc {level} tier {tier} has no Table A.8 entry") + max_br, max_cpb = limits + declared = ( + _hrd(sps, "vui.hrd.nal_hrd_parameters", "vui.hrd.") + if sps.get("vui.hrd.nal_hrd_parameters_present_flag") == "1" + else None + ) + # Absent a NAL HRD, BitRate is CpbBrVclFactor * MaxBR (H.265 E.3.3), which H.222.0 then + # scales by CpbBrNalFactor / CpbBrVclFactor; cpb_size is CpbBrNalFactor * MaxCPB. + bit_rate, cpb = declared or (1000 * max_br, HEVC_NAL_FACTOR * max_cpb) + rate = HEVC_NAL_FACTOR * max_br + return Params( + label=f"HEVC profile {profile} {'high' if tier else 'main'} tier level {level / 30:g}, " + + (f"NAL HRD {bit_rate / 1e6:.3f} Mb/s cpb {cpb / 1e3:.0f} kbit" if declared else "level defaults"), + # H.222.0 2.17.2: Rx = CpbBrNalFactor / CpbBrVclFactor * BitRate[SchedSelIdx]; MBS = + # BSmux + BSoh + CpbBrNalFactor * MaxCPB - cpb_size; EBS = cpb_size; Rbx = + # CpbBrNalFactor * MaxBR. + rx=HEVC_NAL_FACTOR / 1000 * bit_rate, + mb=((0.004 + 1 / 750) * max(rate, 2_000_000) + HEVC_NAL_FACTOR * max_cpb - cpb) / 8, + eb=cpb / 8, + rbx=rate, + max_delay_s=10.0, + ) + - This models only the transport buffer (TB) smoothing stage of ISO 13818-1 - 2.4.2, not the full multiplex/elementary buffer decode model. TB size is - fixed at 512 bytes; the leak rate Rx defaults per stream type. An overflow - means the stream delivers a PID's bytes faster than a receiver drains them. +def adts_frame(es: bytes, pos: int) -> tuple[int, float]: + """(length, duration) of the ADTS frame at `pos`.""" + h = es[pos : pos + 7] + if len(h) < 7 or h[0] != 0xFF or h[1] & 0xF0 != 0xF0: + raise Refused(f"no ADTS sync at ES offset {pos}") + rate_index = (h[2] >> 2) & 0x0F + if rate_index >= len(ADTS_RATES): + raise Refused(f"ADTS sampling_frequency_index {rate_index} at ES offset {pos}") + length = ((h[3] & 0x03) << 11) | (h[4] << 3) | (h[5] >> 5) + return length, 1024 * ((h[6] & 0x03) + 1) / ADTS_RATES[rate_index] + + +def mpeg_audio_frame(es: bytes, pos: int) -> tuple[int, float]: + """(length, duration) of the MPEG-1/2 audio frame at `pos`.""" + h = es[pos : pos + 4] + if len(h) < 4 or h[0] != 0xFF or h[1] & 0xE0 != 0xE0: + raise Refused(f"no MPEG audio sync at ES offset {pos}") + version_bits = (h[1] >> 3) & 0x03 + layer = 4 - ((h[1] >> 1) & 0x03) + rate_index = (h[2] >> 2) & 0x03 + kbps_index = h[2] >> 4 + if version_bits == 1 or layer == 4 or rate_index == 3 or kbps_index in (0, 15): + raise Refused(f"MPEG audio header at ES offset {pos} is reserved or free-format") + # MPEG-1, MPEG-2 (half rate) and MPEG-2.5 (quarter rate). + version = 1 if version_bits == 3 else 2 + rate = (44100, 48000, 32000)[rate_index] >> {3: 0, 2: 1, 0: 2}[version_bits] + bits = MPEG_AUDIO_KBPS[(version, layer)][kbps_index] * 1000 + pad = (h[2] >> 1) & 0x01 + if layer == 1: + return (12 * bits // rate + pad) * 4, 384 / rate + samples = 576 if layer == 3 and version == 2 else 1152 + return samples // 8 * bits // rate + pad, samples / rate + + +def ac3_frame(es: bytes, pos: int) -> tuple[int, float]: + """(length, duration) of the AC-3 or E-AC-3 frame at `pos`; 0 duration for a dependent substream.""" + h = es[pos : pos + 6] + if len(h) < 6 or h[0] != 0x0B or h[1] != 0x77: + raise Refused(f"no AC-3 sync at ES offset {pos}") + if h[5] >> 3 > 10: # bsid 11-16: E-AC-3 (A/52 Annex E) + blocks = (1, 2, 3, 6)[(h[4] >> 4) & 0x03] if h[4] >> 6 != 3 else 6 + fscod = h[4] >> 6 + rate = (48000, 44100, 32000)[fscod] if fscod != 3 else (24000, 22050, 16000)[(h[4] >> 4) & 0x03] + dependent = h[2] >> 6 == 1 + return ((((h[2] & 0x07) << 8) | h[3]) + 1) * 2, 0.0 if dependent else 256 * blocks / rate + fscod, code = h[4] >> 6, h[4] & 0x3F + if fscod == 3 or code >> 1 >= len(AC3_KBPS): + raise Refused(f"AC-3 fscod/frmsizecod reserved at ES offset {pos}") + kbps = AC3_KBPS[code >> 1] + words = (2 * kbps, 320 * kbps // 147 + (code & 1), 3 * kbps)[fscod] + return words * 2, 1536 / (48000, 44100, 32000)[fscod] + + +def opus_frame(es: bytes, pos: int) -> tuple[int, float]: + """(length, duration) of the Opus-in-TS access unit at `pos`: control header, then one packet.""" + if es[pos] != 0x7F or es[pos + 1] & 0xE0 != 0xE0: + raise Refused(f"no Opus control header at ES offset {pos}") + flags = es[pos + 1] + at = pos + 2 + size = 0 + while True: + size += es[at] + at += 1 + if es[at - 1] != 0xFF: + break + at += 2 * bool(flags & 0x10) + 2 * bool(flags & 0x08) + if flags & 0x04: + at += 1 + es[at] + # RFC 6716 3.1: the TOC byte's config sets the frame size, its code the frame count. + config, code = es[at] >> 3, es[at] & 0x03 + if config < 12: + frame_ms = (10, 20, 40, 60)[config % 4] + elif config < 16: + frame_ms = (10, 20)[config % 2] + else: + frame_ms = (2.5, 5, 10, 20)[config % 4] + frames = (1, 2, 2, es[at + 1] & 0x3F)[code] + return at - pos + size, frame_ms * frames / 1000 + + +def stream_kind(stream: Stream) -> str | None: + """Which T-STD model the stream takes, or None for one with no elementary stream buffers (sections, data).""" + kind = stream.stream_type + tags = stream.descriptors + if kind in STREAM_KINDS: + return STREAM_KINDS[kind] + if kind == 0x06 and tags & {"DVB_AC3_descriptor", "AC3_descriptor"}: + return "ac3-dvb" + if kind == 0x06 and f"registration:{int.from_bytes(b'Opus', 'big')}" in tags: + return "opus" + if kind == 0x06 and tags & {"DVB_enhanced_AC3_descriptor", "enhanced_AC3_descriptor"}: + raise Refused("E-AC-3 as DVB private data takes its buffer from ETSI TS 101 154, not modelled") + if kind in (0x01, 0x02, 0x10, 0x11, 0x1C, 0x20, 0x21, 0x42, 0xD1, 0xEA): + raise Refused(f"stream_type 0x{kind:02X} is audio/video this model has no parameters for") + return None + + +def stream_params(kind: str, stream: Stream) -> tuple[Params, object]: + """The stream's T-STD parameters, and its audio frame parser (None for video).""" + es = stream.es + if kind == "avc": + return avc_params(stream.sps), None + if kind == "hevc": + return hevc_params(stream.sps), None + if kind == "adts": + adts_frame(es, 0) + config = ((es[2] & 0x01) << 2) | (es[3] >> 6) + if config not in ADTS_CHANNELS: + raise Refused("ADTS channel_configuration 0 defers the layout to a PCE, which this model does not read") + channels = ADTS_CHANNELS[config] + rx, bs = next((rx, bs) for top, rx, bs in ADTS_BUFFERS if channels <= top) + return Params(label=f"ADTS AAC {channels} ch", rx=rx, b=bs), adts_frame + if kind == "opus": + # Borrowed: the Opus-in-TS draft gives Rx (2 Mb/s for 1-2 channels, as here) but no + # buffer size, so Opus is graded against ADTS's buffers for the same channel count. + config = next((tag[len("tag127:80") :][:2] for tag in stream.descriptors if tag.startswith("tag127:80")), "") + if not config or not 0 <= int(config, 16) <= 8: + raise Refused("Opus without a channel_config_code of 0-8 in its extension descriptor") + channels = int(config, 16) or 2 # 0 is dual mono + rx, bs = next((rx, bs) for top, rx, bs in ADTS_BUFFERS if channels <= top) + return Params(label=f"Opus {channels} ch (ADTS buffers)", rx=rx, b=bs), opus_frame + label, bs, parse = { + "mpeg-audio": ("MPEG audio", MPEG_AUDIO_BS, mpeg_audio_frame), + "ac3-atsc": ("AC-3 (ATSC)", AC3_ATSC_BS, ac3_frame), + "eac3-atsc": ("E-AC-3 (ATSC)", EAC3_ATSC_BS, ac3_frame), + "ac3-dvb": ("AC-3 (DVB)", AC3_DVB_BS, ac3_frame), + }[kind] + return Params(label=label, rx=AUDIO_RX, b=bs), parse + + +# NAL unit types that open a new access unit when they follow the last VCL NAL unit +# of the previous one (H.264 7.4.1.2.3, H.265 7.4.2.4.4), and the VCL types. +AU_OPENERS = {"avc": {6, 7, 8, 9, 14, 15, 16, 17, 18}, "hevc": {32, 33, 34, 35, 39, 41, 42, 43, 44, *range(48, 56)}} +VCL = {"avc": set(range(1, 6)), "hevc": set(range(0, 32))} +DELIMITER = {"avc": 9, "hevc": 35} + + +def video_units(stream: Stream, kind: str) -> list[AccessUnit]: + """One access unit per coded picture, each decoded at its PES's DTS (or PTS). + + An access unit opens at the first delimiter, parameter set or SEI after the previous + picture's last slice (H.264 7.4.1.2.3, H.265 7.4.2.4.4); H.222.0 2.14.1 and 2.17.1 + also put a delimiter in every one, so a stream without them is refused. A PES may + carry several access units, but only the first takes its timestamp; the rest would + need decoding times derived from the stream's own timing (2.4.2.3), which this model + does not do, so such a layout is refused rather than graded as one unit. """ - kinds: dict[int, float] = {} - for pid in analysis.get("pids", []): - if pid.get("video"): - kinds[pid["id"]] = th.video_leak_bps - elif pid.get("audio"): - kinds[pid["id"]] = th.audio_leak_bps - - overflows = 0 - worst_pid = None - worst_occ = 0.0 - for pid, leak_bps in kinds.items(): - packets = scan.pid_packets.get(pid, []) - occ = 0.0 - last_t = None - pid_over = 0 - pid_peak = 0.0 - for i, payload in packets: - t = clock.time_at(i) - if last_t is not None and t > last_t: - occ = max(0.0, occ - leak_bps * (t - last_t) / 8.0) - occ += payload - pid_peak = max(pid_peak, occ) - if occ > th.tb_size_bytes: - pid_over += 1 - last_t = t - overflows += pid_over - if pid_peak > worst_occ: - worst_occ = pid_peak - worst_pid = pid - if not kinds: - return Check("tstd", Severity.SHAPE, Status.WARN, "no video/audio PID to model", {}) + es = bytes(stream.es) + starts = [] + delimited = False + after_vcl = True # the first opener starts the first whole access unit + pos = es.find(b"\x00\x00\x01") + while 0 <= pos < len(es) - 3: + nal = (es[pos + 3] >> 1) & 0x3F if kind == "hevc" else es[pos + 3] & 0x1F + delimited |= nal == DELIMITER[kind] + if nal in AU_OPENERS[kind] and after_vcl: + # The zero_byte ahead of the start code belongs to this NAL unit (H.264 B.1.1, + # H.265 B.2.1), so to this access unit rather than the last. + starts.append(pos - 1 if pos and es[pos - 1] == 0 else pos) + after_vcl = False + elif nal in VCL[kind]: + after_vcl = True + pos = es.find(b"\x00\x00\x01", pos + 3) + if not delimited: + raise Refused("no access unit delimiter, which H.222.0 2.14.1/2.17.1 requires in every access unit") + pes = stream.pes + pes_at = [offset for offset, *_ in pes] + packet_at = [offset for _i, _h, body, offset in stream.packets if body] + packets = [index for index, _h, body, _o in stream.packets if body] + units: list[AccessUnit] = [] + stamped: set[int] = set() + for n, start in enumerate(starts): + k = bisect.bisect_right(pes_at, start) - 1 + _offset, pts, dts, index = pes[k] + if pts is None: + raise Refused(f"an access unit starts in the untimestamped PES at packet {index}") + if k in stamped: + raise Refused(f"the PES at packet {index} carries several access units, whose decode times are not derived") + stamped.add(k) + end = starts[n + 1] if n + 1 < len(starts) else stream.es_len + first = packets[bisect.bisect_right(packet_at, start) - 1] + units.append(AccessUnit(start, end, first, dts if dts is not None else pts)) + return units + + +def access_units(stream: Stream, parse) -> list[AccessUnit]: + """Audio: one access unit per codec frame.""" + units: list[AccessUnit] = [] + pes = [entry for entry in stream.pes if entry[1] is not None] + # A PES timestamp belongs to the first frame that starts in that PES (2.4.3.7). + starts = [(offset, index) for index, _h, body, offset in stream.packets if body] + keys = [offset for offset, _ in starts] + stamps = iter(pes) + nxt = next(stamps, None) + pos = 0 + # A header cut short by the end of the capture is not a malformed frame. + while pos + 8 <= len(stream.es): + try: + length, duration = parse(stream.es, pos) + except IndexError: + break # a header that runs past the end of the capture + if length <= 0: + raise Refused(f"zero-length audio frame at ES offset {pos}") + stamp = None + while nxt is not None and nxt[0] <= pos: + stamp = nxt[1] + nxt = next(stamps, None) + if duration == 0.0 and units: + units[-1].end = pos + length + else: + first = starts[bisect.bisect_right(keys, pos) - 1][1] + units.append(AccessUnit(pos, pos + length, first, stamp, duration)) + pos += length + return units + + +@dataclass +class Grade: + """What one elementary stream did in the model.""" + + pid: int + label: str + violations: dict[str, int] = field(default_factory=dict) + peaks: dict[str, float] = field(default_factory=dict) + worst_late_ms: float = 0.0 + worst_delay_s: float = 0.0 + graded_units: int = 0 + + def flag(self, name: str) -> None: + self.violations[name] = self.violations.get(name, 0) + 1 + + def peak(self, name: str, fill: float, size: float) -> None: + self.peaks[name] = max(self.peaks.get(name, 0.0), fill / size) + + +def simulate( + stream: Stream, params: Params, units: list[AccessUnit], segments: list[tuple[PcrClock, int, int]] +) -> Grade: + """Run one elementary stream through its buffers, with fresh buffers for each time base. + + `segments` are (clock, first packet, end packet) per time base: the timestamps on + either side of a signalled discontinuity are on different clocks. + """ + grade = Grade(stream.pid, params.label) + eps = 1e-9 + for clock, lo, hi in segments: + # TB: packet i arrives over [t(i), t(i+1)] and leaves at Rx. `leave` is when its + # last byte does; the bytes still in TB as the packet finishes arriving are its + # peak, and a stretch where TB never empties may not exceed a second. + deliveries: list[tuple[float, int, int, int]] = [] + leave = float("-inf") + busy_since = None + for index, header, body, offset in stream.packets: + if not lo <= index < hi: + continue + start, end = clock.time_at(index), clock.time_at(index + 1) + if leave <= start: + busy_since = start + leave = max(end, max(leave, start) + 188 * 8 / params.rx) + fill = (leave - end) * params.rx / 8 + grade.peak("TB", fill, TB_SIZE) + if fill > TB_SIZE + 0.5: + grade.flag("TB overflow") + if busy_since is not None and leave - busy_since > TB_EMPTY_S: + grade.flag("TB not emptied within 1 s") + busy_since = None + if header or body: + deliveries.append((leave, header, body, offset)) + + horizon = clock.time_at(hi) + removals: list[tuple[float, AccessUnit]] = [] + td = None + for unit in units: + if not lo <= unit.first_packet < hi: + continue + if unit.stamp is not None: + base = unit.stamp / PTS_HZ + stamped = base + round((clock.time_at(unit.first_packet) - base) / STAMP_WRAP_S) * STAMP_WRAP_S + if removals and stamped <= removals[-1][0]: + grade.flag("decode time does not advance") + td = max(stamped, removals[-1][0]) if removals else stamped + elif td is None: + continue # an audio frame ahead of the first timestamp has no decoding time + if td > horizon: + break + removals.append((td, unit)) + td += unit.duration_s + + # MB -> EB (video, leak method) or B (audio): walk deliveries and removals in time + # order. EB/B is tracked by ES offset: everything below `into` has reached it and + # everything below `out` has been removed, so an access unit whose bytes arrive + # after its decoding time passes through as underflow rather than lingering as fill. + into = out = deliveries[0][3] if deliveries else 0 + mb: list[list[int]] = [] # [header, payload] per delivered packet, FIFO + mb_header = mb_payload = 0 + headers: list[tuple[int, int]] = [] # audio: (ES offset, bytes) of PES headers held in B + b_header = 0 + now = float("-inf") + late: list[tuple[int, float]] = [] + + def settle(t: float, moved_from: int, rate: float) -> None: + # Record how late each underflowed unit finished arriving. + while late and into >= late[0][0]: + end, td = late.pop(0) + done = t if rate <= 0 else now + (end - moved_from) * 8 / rate + grade.worst_late_ms = max(grade.worst_late_ms, (done - td) * 1000) + + def leak(until: float) -> None: + nonlocal into, mb_header, mb_payload, now + if not params.video or until <= now or not mb_payload: + now = max(now, until) + return + room = params.eb - max(0, into - out) + max(0, out - into) + amount = min(params.rbx * (until - now) / 8, mb_payload, room) + before = into + remaining = amount + while remaining > eps and mb: + head = mb[0] + mb_header -= head[0] + head[0] = 0 + take = min(head[1], remaining) + head[1] -= take + remaining -= take + if head[1] <= eps: + mb.pop(0) + mb_payload -= amount + into += amount + settle(until, before, params.rbx) + grade.peak("EB", max(0, into - out), params.eb) + now = until + + events = sorted( + [(t, 1, n) for n, (t, *_rest) in enumerate(deliveries)] + [(t, 0, n) for n, (t, _u) in enumerate(removals)] + ) + for t, kind, n in events: + leak(t) + if kind == 1: + _t, header, body, offset = deliveries[n] + if params.video: + mb.append([header, body]) + mb_header += header + mb_payload += body + grade.peak("MB", mb_header + mb_payload, params.mb) + if mb_header + mb_payload > params.mb + 0.5: + grade.flag("MB overflow") + else: + if header: + headers.append((offset, header)) + b_header += header + into += body + settle(t, into, 0) + fill = max(0, into - out) + b_header + grade.peak("B", fill, params.b) + if fill > params.b + 0.5: + grade.flag("B overflow") + continue + _t, unit = removals[n] + grade.graded_units += 1 + grade.worst_delay_s = max(grade.worst_delay_s, t - clock.time_at(unit.first_packet)) + if t - clock.time_at(unit.first_packet) > params.max_delay_s: + grade.flag(f"held over {params.max_delay_s:g} s") + if into + eps < unit.end: + grade.flag("EB underflow" if params.video else "B underflow") + late.append((unit.end, t)) + out = max(out, unit.end) + while headers and headers[0][0] < unit.end: + b_header -= headers.pop(0)[1] + return grade + + +def check_tstd(ts_path: str, packet_size: int, scan: Scan) -> Check: + """Full T-STD buffer model (ISO 13818-1 2.4.2) for every audio and video stream. + + Each program's streams run on that program's PCR, one time base at a time: a + signalled discontinuity starts fresh buffers, since the timestamps before and + after it are on different clocks. + """ + with open(ts_path, "rb") as handle: + data = handle.read() + programs = read_programs(ts_path) + + grades: list[Grade] = [] + refused: dict[int, str] = {} + skipped: list[int] = [] + for pcr_pid, streams in programs: + kinds: dict[int, str] = {} + for stream in streams: + try: + kind = stream_kind(stream) + except Refused as err: + refused[stream.pid] = str(err) + continue + if kind is None: + skipped.append(stream.pid) + else: + kinds[stream.pid] = kind + modelled = {s.pid: s for s in streams if s.pid in kinds} + for pid, why in read_pes(data, packet_size, modelled).items(): + refused[pid] = why + del modelled[pid] + samples = scan.pcr_by_pid.get(pcr_pid, []) + bounds = [0, *sorted(i for i, _ in samples if i in scan.pcr_new_base), scan.total_packets] + segments = [(PcrClock([s for s in samples if lo <= s[0] < hi]), lo, hi) for lo, hi in zip(bounds, bounds[1:])] + segments = [segment for segment in segments if segment[0].ok()] + for stream in modelled.values(): + try: + if kinds[stream.pid] in ("avc", "hevc"): + if "hrd_management_valid" in stream.descriptors: + raise Refused("the HRD-scheduled MB to EB transfer (H.222.0 2.14.3.1) is not modelled") + stream.sps = read_sps(ts_path, stream.pid, 7 if kinds[stream.pid] == "avc" else 33) + params, parse = stream_params(kinds[stream.pid], stream) + if parse is None: + units = video_units(stream, kinds[stream.pid]) + else: + units = access_units(stream, parse) + except Refused as err: + refused[stream.pid] = str(err) + continue + grades.append(simulate(stream, params, units, segments)) + metrics = { - "tb_overflows": overflows, - "worst_pid": worst_pid, - "worst_peak_bytes": round(worst_occ, 1), - "tb_size_bytes": th.tb_size_bytes, + "streams": { + str(g.pid): { + "type": g.label, + "access_units": g.graded_units, + "violations": g.violations, + "peak_fill_pct": {k: round(v * 100, 1) for k, v in g.peaks.items()}, + "worst_underflow_late_ms": round(g.worst_late_ms, 1), + "worst_delay_s": round(g.worst_delay_s, 3), + } + for g in grades + }, + "refused": {str(pid): why for pid, why in refused.items()}, + "not_elementary": skipped, } - detail = f"TB overflows {overflows}, worst peak {worst_occ:.0f}B on PID {worst_pid} (TB {th.tb_size_bytes}B)" - status = Status.PASS if overflows == 0 else Status.WARN - return Check("tstd", Severity.SHAPE, status, detail, metrics) + faults = [f"PID {g.pid} {name} x{count}" for g in grades for name, count in sorted(g.violations.items())] + faults += [f"PID {pid} refused: {why}" for pid, why in refused.items()] + if not grades and not refused: + return Check("tstd", Severity.SHAPE, Status.WARN, "no audio/video stream to model", metrics) + if not faults and not any(g.graded_units for g in grades): + return Check("tstd", Severity.SHAPE, Status.WARN, "no access unit fell inside a modelled time base", metrics) + if faults: + return Check("tstd", Severity.SHAPE, Status.WARN, "; ".join(faults), metrics) + peaks = ", ".join(f"PID {g.pid} " + "/".join(f"{k} {v * 100:.0f}%" for k, v in g.peaks.items()) for g in grades) + return Check("tstd", Severity.SHAPE, Status.PASS, f"no overflow or underflow; peak fill {peaks}", metrics) def detect_packet_size(analysis: dict) -> int: @@ -660,30 +1176,17 @@ def check_duration_fidelity(captured_s: float, reference_s: float) -> Check: # ------------------------------------------------------------------------ driver -def analyze(ts_path: str, th: Thresholds, reference_seconds: float | None = None) -> list[Check]: +def analyze(ts_path: str, reference_seconds: float | None = None) -> list[Check]: """Run every check against `ts_path` and return the ordered results. `reference_seconds` (the source's PCR span, round-trip only) enables the duration-fidelity check that pins the exported stream's absolute rate. """ analysis = run_tsanalyze(ts_path) - ts = analysis["ts"] packet_size = detect_packet_size(analysis) scan = scan_packets(ts_path, packet_size) clock_by_pid = {pid: PcrClock(samples) for pid, samples in scan.pcr_by_pid.items()} - # The reference clock is the PID with the most PCR samples (the PCR PID). - main_clock = max(clock_by_pid.values(), key=lambda c: len(c.idx), default=PcrClock([])) - - # Nominal bitrate: total bytes clocked over the PCR span (the rate an IRD would - # play the stream at). Self-consistent with the PCR clock used everywhere else, - # and far more stable than tsanalyze's instantaneous PCR bitrate on a bursty - # capture. Fall back to tsanalyze only when there is no usable PCR clock. - span = main_clock.sec[-1] - main_clock.sec[0] if main_clock.ok() else 0.0 - if span > 0: - ts_bitrate = scan.total_packets * scan.packet_size * 8 / span - else: - ts_bitrate = float(ts.get("bitrate") or ts.get("pcr-bitrate") or 0) checks: list[Check] = [] checks.append(check_packet_size(analysis)) @@ -696,22 +1199,10 @@ def analyze(ts_path: str, th: Thresholds, reference_seconds: float | None = None checks.append(check_pcr_presence(analysis, clock_by_pid)) checks.append(check_pcr_monotonic(scan)) if reference_seconds is not None: - checks.append(check_duration_fidelity(span, reference_seconds)) + checks.append(check_duration_fidelity(pcr_span_seconds(scan), reference_seconds)) checks.append(check_service_descriptors(analysis)) - checks.append(check_pcr_repetition(scan, th)) - checks.append(check_pcr_jitter(scan, ts_bitrate, th)) - checks.append(check_null_ratio(analysis, th)) - - if main_clock.ok(): - bitrate, burst = check_bitrate_and_burstiness(scan, main_clock, ts_bitrate, th) - checks.append(bitrate) - checks.append(burst) - checks.append(check_inter_arrival(scan, main_clock)) - checks.append(check_tstd(analysis, scan, main_clock, th)) - else: - for name in ("bitrate-consistency", "burstiness", "inter-arrival", "tstd"): - checks.append(Check(name, Severity.SHAPE, Status.WARN, "not enough PCRs to build a clock", {})) + checks.append(check_tstd(ts_path, packet_size, scan)) return checks @@ -741,20 +1232,6 @@ def verdict(checks: list[Check], strict: bool) -> int: return 0 -def build_thresholds(args: argparse.Namespace) -> Thresholds: - """Assemble a Thresholds from parsed CLI arguments.""" - return Thresholds( - pcr_repetition_ms=args.pcr_repetition_ms, - pcr_jitter_us=args.pcr_jitter_us, - null_ratio_max=args.null_ratio_max, - bitrate_cov_max=args.bitrate_cov_max, - burstiness_max=args.burstiness_max, - tb_size_bytes=args.tb_size_bytes, - video_leak_bps=args.video_leak_bps, - audio_leak_bps=args.audio_leak_bps, - ) - - def main() -> int: """CLI entry point: parse args, analyze the TS, print the report, return the exit code.""" parser = argparse.ArgumentParser(description="MPEG-TS / IRD compliance analyzer") @@ -765,20 +1242,11 @@ def main() -> int: "--reference", help="source TS the capture was muxed from; enables the duration-fidelity check", ) - parser.add_argument("--pcr-repetition-ms", type=float, default=Thresholds.pcr_repetition_ms) - parser.add_argument("--pcr-jitter-us", type=float, default=Thresholds.pcr_jitter_us) - parser.add_argument("--null-ratio-max", type=float, default=Thresholds.null_ratio_max) - parser.add_argument("--bitrate-cov-max", type=float, default=Thresholds.bitrate_cov_max) - parser.add_argument("--burstiness-max", type=float, default=Thresholds.burstiness_max) - parser.add_argument("--tb-size-bytes", type=int, default=Thresholds.tb_size_bytes) - parser.add_argument("--video-leak-bps", type=float, default=Thresholds.video_leak_bps) - parser.add_argument("--audio-leak-bps", type=float, default=Thresholds.audio_leak_bps) args = parser.parse_args() - th = build_thresholds(args) try: reference_seconds = source_duration(args.reference) if args.reference else None - checks = analyze(args.ts, th, reference_seconds) + checks = analyze(args.ts, reference_seconds) except (RuntimeError, FileNotFoundError, json.JSONDecodeError) as err: print(f"error: {err}", file=sys.stderr) return 2 diff --git a/test/ts/pcr-timing.py b/test/ts/pcr-timing.py index b318fe39043..643613d997d 100755 --- a/test/ts/pcr-timing.py +++ b/test/ts/pcr-timing.py @@ -80,6 +80,8 @@ def __init__(self): self.cc_on_empty = [] self.wraps = 0 self.new_base = set() # positions in self.pcr that state a new time base + self.pcr_pid = None # the PID graded, once keep_busiest_pcr_pid has run + self.other_pcr_pids = [] self._cc = {} self._dup = {} self._new_base = set() # PIDs whose next PCR states a new time base @@ -157,6 +159,21 @@ def feed(self, p, arrival=None): elif cc != prev: self.cc_on_empty.append((index, pid, prev, cc)) + def keep_busiest_pcr_pid(self): + """Grade one clock: the PID carrying the most PCRs. + + Each program may carry its own PCR, and two correct grids offset from one another + pool into one grid of half the interval that neither keeps. + """ + pids = collections.Counter(pid for _, _, _, pid in self.pcr) + self.pcr_pid = pids.most_common(1)[0][0] if pids else None + self.other_pcr_pids = sorted(p for p in pids if p != self.pcr_pid) + if not self.other_pcr_pids: + return + kept = [(k, e) for k, e in enumerate(self.pcr) if e[3] == self.pcr_pid] + self.new_base = {n for n, (k, _) in enumerate(kept) if k in self.new_base} + self.pcr = [e for _, e in kept] + def scan_file(path): scan = Scan() @@ -255,18 +272,6 @@ def check_continuity(scan, args): ) -def check_pcr_single_pid(scan, args): - """Every PCR must ride the one PID the PMT declares.""" - pids = collections.Counter(pid for _, _, _, pid in scan.pcr) - return ( - "pcr-single-pid", - HARD, - len(pids) <= 1, - f"PCR carried on {len(pids)} PID(s): " + ", ".join(f"{k} ({v})" for k, v in pids.most_common()), - {"pids": {str(k): v for k, v in pids.items()}}, - ) - - def check_value_interval(scan, args): """PCR values must be spaced within the repetition limit. @@ -493,16 +498,9 @@ def check_schedule(scan, args): hard = args.schedule_pct_min is not None severity = HARD if hard else SHAPE required = args.schedule_pct_min if hard else 99.0 - # One PID only. Two PIDs each on a correct grid, offset from one another, pool into a - # grid of half the interval with half the bytes in each slot, which is a schedule - # neither of them keeps. pcr-single-pid already fails a stream carrying two; grading - # the busier one keeps this check's answer about one clock rather than about both. - pids = collections.Counter(pid for _, _, _, pid in scan.pcr) - pid = pids.most_common(1)[0][0] if pids else None - on_pid = [(k, e) for k, e in enumerate(scan.pcr) if e[3] == pid] graded = [] skipped = 0 - for (_, a), (kb, b) in zip(on_pid, on_pid[1:]): + for kb, (a, b) in enumerate(zip(scan.pcr, scan.pcr[1:]), start=1): seconds = (b[1] - a[1]) / TICKS_PER_MS / 1000.0 # An interval spanning a signalled new time base measures nothing, as in the value # check. A non-positive one is a duplicate packet (legal) or a backwards clock (the @@ -513,7 +511,7 @@ def check_schedule(scan, args): graded.append(((b[0] - a[0]) * PKT, seconds)) if len(graded) < 3: # Asked to gate, an ungradable stream is a failure rather than a clean one. - return ("pcr-schedule", severity, not hard, "not measured (too few PCR intervals on one PID)", {}) + return ("pcr-schedule", severity, not hard, "not measured (too few PCR intervals)", {}) aggregate = 8.0 * sum(n for n, _ in graded) / sum(s for _, s in graded) rate = args.mux_rate if args.mux_rate else aggregate @@ -529,8 +527,8 @@ def check_schedule(scan, args): within = sum(1 for (n, _), w, s in zip(graded, want, slack) if abs(n - w) <= s) median_s = statistics.median(s for _, s in graded) detail = { - "pid": pid, - "other_pcr_pids": sorted(p for p in pids if p != pid), + "pid": scan.pcr_pid, + "other_pcr_pids": scan.other_pcr_pids, "count": len(graded), "skipped": skipped, "rate_bps": round(rate), @@ -565,15 +563,10 @@ def check_schedule(scan, args): ) -CHECKS = [ - check_sync, - check_continuity, - check_pcr_single_pid, - check_value_interval, - check_release, - check_position, - check_schedule, -] +# A file is graded for sync and continuity by compliance.py, through TSDuck's tsanalyze. +# A pipe cannot be: tsp in front of it would rebuffer the very arrivals `release` stamps. +LIVE_CHECKS = [check_sync, check_continuity] +CHECKS = [check_value_interval, check_release, check_position, check_schedule] def coincidence(scan, args): @@ -598,15 +591,11 @@ def coincidence(scan, args): def main(): - ap = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("path", nargs="?", help="TS file; omit with --live to read stdin") ap.add_argument("--live", action="store_true", help="read stdin and stamp arrivals, grading release timing") ap.add_argument("--seconds", type=float, default=45.0, help="live capture window (default 45)") - ap.add_argument( - "--repetition-ms", type=float, default=40.0, help="max PCR value interval, TR 101 290 (default 40)" - ) + ap.add_argument("--repetition-ms", type=float, default=40.0, help="max PCR value interval, TR 101 290 (default 40)") ap.add_argument( "--release-ms", type=float, @@ -630,8 +619,7 @@ def main(): "--live-cover-pct", type=float, default=50.0, - help="share of the --seconds window a --live sample must span, or it is treated as " - "truncated (default 50)", + help="share of the --seconds window a --live sample must span, or it is treated as truncated (default 50)", ) ap.add_argument( "--drift-ms", @@ -684,7 +672,8 @@ def main(): else: ap.error("give a path or --live") - results = [check(scan, args) for check in CHECKS] + scan.keep_busiest_pcr_pid() + results = [check(scan, args) for check in (LIVE_CHECKS if args.live else []) + CHECKS] width = max(len(r[0]) for r in results) print(f"### PCR timing report - {scan.packets} packets, {len(scan.pcr)} PCR") @@ -721,8 +710,7 @@ def main(): "pcr_count": len(scan.pcr), "live": args.live, "checks": [ - {"name": n, "severity": s, "ok": ok, "headline": h, "detail": d} - for n, s, ok, h, d in results + {"name": n, "severity": s, "ok": ok, "headline": h, "detail": d} for n, s, ok, h, d in results ], }, f, diff --git a/test/ts/run.sh b/test/ts/run.sh index 0858b89dadc..3ceb480b7f2 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -7,7 +7,7 @@ # TSDuck + custom analyzer in compliance.py against the capture. The point is to # tell whether what the subscriber emits is something an Integrated # Receiver/Decoder would accept, and to quantify where it diverges (the exporter -# is VBR, emits no null packets, and paces PCR per frame). +# pads to a constant rate only once the catalog carries the source's mux rate). # # Modes: # ./run.sh # generate a clip, round-trip it, analyze @@ -160,7 +160,7 @@ have() { command -v "$1" >/dev/null 2>&1; } require_tools() { local missing=() t - for t in tsp tsanalyze python3; do + for t in tsp tsanalyze tstables python3; do have "$t" || missing+=("$t") done # ffmpeg + cargo are only needed for the round-trip, not for --analyze-only. @@ -174,7 +174,7 @@ require_tools() { fi if [[ ${#missing[@]} -gt 0 ]]; then echo "error: missing required tools: ${missing[*]}" >&2 - echo " TSDuck (tsp, tsanalyze) is required; install from https://tsduck.io" >&2 + echo " TSDuck (tsp, tsanalyze, tstables) is required; install from https://tsduck.io" >&2 exit 1 fi } diff --git a/test/ts/tstd-controls.py b/test/ts/tstd-controls.py new file mode 100755 index 00000000000..ccebb6f8f07 --- /dev/null +++ b/test/ts/tstd-controls.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Positive and negative controls for the T-STD model in compliance.py. + +The positive control is a real broadcast encoder's output (a Kyrion contribution +feed: AVC High@4.0 plus two MPEG-1 Layer II tracks), which a broadcast chain only +carries because it is T-STD compliant. Each negative restamps that capture's PCRs +at a different constant rate with TSDuck's `pcradjust`, leaving every PES, PTS and +DTS untouched, so the only thing that changes is when the bytes arrive: + + as captured the broadcast itself must pass + restamped 1x PCRs rewritten at the capture's own rate must pass (the rewrite is not the fault) + 0.7x delivered too slowly EB and B underflow + 4x delivered too early TB and B overflow + 15x delivered in a burst TB and B overflow + +The video TB drains at 1.2x the bit rate the SPS's NAL HRD declares (1.935 Mb/s), +so any delivery well above real time overflows it. MB holds the level's whole CPB +less the declared one, ~3.6 MB, more than this 4 s capture carries, so no restamp +of it can overflow MB. + +A capture cannot be edited into the packet layouts the model has to get right, so +the rest are built here: a 10 Mb/s single-video stream carrying the Kyrion SPS, +with each AU in its own PES, a PCR packet between AUs and nulls elsewhere. + + synthetic as built must pass + two AUs, one PES an AU without its own timestamp refused + adaptation burst four payload-less packets after an AU TB overflow (they cost TB) + duplicate packet one AU's first packet sent twice (2.4.3.3) must pass (TB only, not MB) + +A model that passes everything, or fails everything, cannot tell these apart. +""" + +import os +import subprocess +import sys +import tempfile + +DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, DIR) + +import compliance # noqa: E402 + +FIXTURE = os.path.join(DIR, "../../rs/moq-mux/src/container/ts/test_data/scte35/kyrion_dirtystart.ts") + +# The Kyrion SPS (High@4.0, NAL HRD 1.935 Mb/s CBR, 755 kbit CPB), so the synthetic +# streams get the same declared buffers. +SPS = bytes.fromhex("6764 0028 acd1 0078 044f de03 6a02 0202 8000 01f4 8000 7530 7500 0762 0002 e11f af7f 072a 1629 92") +VIDEO_PID, PMT_PID = 0x100, 0x1000 +RATE = 10_000_000 +SLOT_S = 188 * 8 / RATE + + +def crc32_mpeg(data: bytes) -> int: + """The MPEG-2 section CRC: polynomial 0x04C11DB7, not reflected, initial all ones.""" + crc = 0xFFFFFFFF + for byte in data: + crc ^= byte << 24 + for _ in range(8): + crc = (crc << 1) ^ 0x04C11DB7 if crc & 0x80000000 else crc << 1 + crc &= 0xFFFFFFFF + return crc + + +def section(table_id: int, extension: int, body: bytes) -> bytes: + """A long-form PSI section, CRC included.""" + head = bytes([table_id, 0xB0 | ((len(body) + 9) >> 8), (len(body) + 9) & 0xFF]) + head += extension.to_bytes(2, "big") + bytes([0xC1, 0, 0]) + return head + body + crc32_mpeg(head + body).to_bytes(4, "big") + + +def packet(pid: int, cc: int, payload: bytes = b"", pusi: bool = False, adaptation: bytes | None = None) -> bytes: + """One 188-byte packet; any room the payload leaves is adaptation-field stuffing.""" + room = 184 - len(payload) + if adaptation is None and room > 0: + adaptation = b"\x00" if room > 1 else b"" + if adaptation is not None: + adaptation = bytes([room - 1]) + adaptation.ljust(room - 1, b"\xff") if room > 1 else b"\x00" + afc = (2 if adaptation is not None else 0) | (1 if payload else 0) + head = bytes([0x47, (0x40 if pusi else 0) | pid >> 8, pid & 0xFF, afc << 4 | cc]) + return head + (adaptation or b"") + payload + + +def stamp(prefix: int, ticks: int) -> bytes: + """A 5-byte PES PTS/DTS field.""" + return bytes( + [ + prefix << 4 | (ticks >> 29) & 0x0E | 1, + (ticks >> 22) & 0xFF, + (ticks >> 14) & 0xFE | 1, + (ticks >> 7) & 0xFF, + (ticks << 1) & 0xFE | 1, + ] + ) + + +def synthetic(path: str, layout: str) -> None: + """Write a compliant 2 s stream, bent into `layout` at one access unit.""" + pat = section(0x00, 1, (1).to_bytes(2, "big") + (0xE000 | PMT_PID).to_bytes(2, "big")) + pmt = section(0x02, 1, bytes([0xE1, 0x00, 0xF0, 0x00, 0x1B, 0xE1, 0x00, 0xF0, 0x00])) + cc: dict[int, int] = {} + out: list[bytes] = [] + + def emit(pid: int, payload: bytes = b"", pusi: bool = False, adaptation: bytes | None = None) -> bytes: + if payload: + cc[pid] = (cc.get(pid, -1) + 1) & 0x0F + made = packet(pid, cc.get(pid, 0), payload, pusi, adaptation) + out.append(made) + return made + + def pes(units: list[bytes], dts_s: float) -> None: + ticks = round((1.0 + dts_s) * compliance.PTS_HZ) + data = bytes.fromhex("000001e0 0000 80c0 0a") + stamp(3, ticks) + stamp(1, ticks) + b"".join(units) + first = True + for at in range(0, len(data), 184): + made = emit(VIDEO_PID, data[at : at + 184], pusi=first) + if first and layout == "duplicate packet" and dts_s == 0.5 + 30 * 0.04: + out.append(made) + first = False + + frames = 50 + pending = None + for k in range(frames): + begin = len(out) + emit(0, b"\x00" + pat, pusi=True) + emit(PMT_PID, b"\x00" + pmt, pusi=True) + unit = b"\x00\x00\x00\x01\x09\xf0" + (b"\x00\x00\x00\x01" + SPS if k == 0 else b"") + unit += b"\x00\x00\x01\x65" + b"\xaa" * 250 + # Each AU decodes 0.5 s after its first byte arrives. + if layout == "two AUs, one PES" and k == 10: + pending = unit + else: + pes([pending, unit] if pending else [unit], 0.5 + (k - bool(pending)) * 0.04) + pending = None + if layout == "adaptation burst" and k == 20: + for _ in range(4): + emit(VIDEO_PID, adaptation=b"\x00") + slots = round(0.04 / SLOT_S) + while len(out) - begin < slots: + if len(out) - begin == slots // 2: + pcr = round((1.0 + len(out) * SLOT_S) * compliance.PCR_HZ) + field = bytes([0x10]) + ((pcr // 300) << 15 | 0x7E00 | pcr % 300).to_bytes(6, "big") + emit(VIDEO_PID, adaptation=field) + else: + out.append(packet(0x1FFF, 0, b"\xff" * 184)) + with open(path, "wb") as handle: + handle.write(b"".join(out)) + + +def restamp(scale: float): + """A builder that restamps the Kyrion capture's PCRs at `scale` times its own rate.""" + + def build(path: str) -> str: + rate = compliance.run_tsanalyze(FIXTURE)["ts"]["bitrate"] + subprocess.run( + ["tsp", "-I", "file", FIXTURE, "-P", "pcradjust", "--bitrate", f"{rate * scale:.0f}"] + + ["--ignore-pts", "--ignore-dts", "-O", "file", path], + check=True, + ) + return path + + return build + + +def built(layout: str): + """A builder for one synthetic layout.""" + + def build(path: str) -> str: + synthetic(path, layout) + return path + + return build + + +# (name, builder, expected): violations that must all appear, or "pass", or a refusal's text. +CASES = [ + ("as captured", lambda _path: FIXTURE, "pass"), + ("restamped 1x", restamp(1.0), "pass"), + ("0.7x", restamp(0.7), {"EB underflow", "B underflow"}), + ("4x", restamp(4.0), {"TB overflow", "B overflow"}), + ("15x", restamp(15.0), {"TB overflow", "B overflow"}), + ("synthetic", built("synthetic"), "pass"), + ("two AUs, one PES", built("two AUs, one PES"), "several access units"), + ("adaptation burst", built("adaptation burst"), {"TB overflow"}), + ("duplicate packet", built("duplicate packet"), "pass"), +] + + +def grade(path: str) -> compliance.Check: + """compliance.py's tstd verdict on one file.""" + size = compliance.detect_packet_size(compliance.run_tsanalyze(path)) + return compliance.check_tstd(path, size, compliance.scan_packets(path, size)) + + +def main() -> int: + """Run every case and exit non-zero if any verdict is not the expected one.""" + failed = 0 + with tempfile.TemporaryDirectory() as tmp: + for n, (name, build, expected) in enumerate(CASES): + check = grade(build(os.path.join(tmp, f"{n}.ts"))) + streams = check.metrics.get("streams", {}) + seen = {v for s in streams.values() for v in s["violations"]} + if expected == "pass": + # Not vacuous: every stream must have had access units to grade. + ok = check.status == compliance.Status.PASS and all(s["access_units"] for s in streams.values()) + elif isinstance(expected, str): + ok = any(expected in why for why in check.metrics.get("refused", {}).values()) + else: + ok = expected <= seen + failed += not ok + want = expected if isinstance(expected, str) else ", ".join(sorted(expected)) + print(f" {'ok ' if ok else 'FAIL'} {name:<18} want {want:<40} got {check.detail}") + if failed: + print(f"tstd controls: {failed} of {len(CASES)} cases wrong", file=sys.stderr) + return 1 + print("tstd controls: PASS") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 02a332fa78cefa60569bae055c0245d10cf0bc73 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 21:39:44 -0700 Subject: [PATCH 13/40] fix(moq-mux): finish each TS unit a millisecond before its DTS A unit's last packet still drains through the receiver's transport and multiplex buffers, no slower than 2 Mb/s, so a unit whose DTS sat on a slot boundary was graded up to 0.1 ms late. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export.rs | 4 +-- rs/moq-mux/src/container/ts/schedule.rs | 38 ++++++++++++++++++------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 98026801d5c..d8f2f99e284 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -871,7 +871,7 @@ impl Export { self.last_timestamp = Some(ready.item.frame.timestamp); let decode = ready.item.decode.as_nanos(); self.mux(&name, ready.item)?; - self.lay(Some(schedule::slot(decode)))?; + self.lay(Some(schedule::due(decode)))?; } // 5. Once every track has drained, nothing more can ride the slots still open, so @@ -1700,7 +1700,7 @@ impl Export { /// The clock packet for grid slot `index`. /// /// The value runs one slot behind the boundary, so the bytes of slot `index` are timed - /// up to its boundary: a unit due in that slot decodes no earlier ([`schedule::slot`]). + /// up to its boundary: a unit due in that slot decodes no earlier ([`schedule::due`]). /// Back off through the 33-bit wrap rather than saturating: a timeline that starts in /// its first slot would otherwise clamp to zero and break the uniform step. The wire /// field is a circular clock, so the masked wrapped value is the correct mod-2^33 diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index f3ba0f322f1..4d55a6eea51 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -23,8 +23,19 @@ const PACKET: u64 = TsPacket::SIZE as u64 * 8 * SLOTS_PER_SECOND; /// a span that carried no bytes only stalls anything pacing on the asserted values. const BACKFILL: u128 = 40; -/// The grid slot whose clock packet a unit decoding at `nanos` must be sent by. -pub(super) fn slot(nanos: u128) -> u128 { +/// How long before its decode time a unit's last packet has to arrive. A receiver still +/// moves that packet through its transport and multiplex buffers, which drain no slower +/// than 2 Mb/s (ISO 13818-1 2.4.2.3), so 0.75 ms a packet. +const DRAIN: Duration = Duration::from_millis(1); + +/// The grid slot a unit decoding at `nanos` must be sent by: its bytes are timed up to the +/// slot's boundary, which has to fall a [`DRAIN`] before the decode time. +pub(super) fn due(nanos: u128) -> u128 { + slot(nanos.saturating_sub(DRAIN.as_nanos())) +} + +/// The grid slot `nanos` falls in. +fn slot(nanos: u128) -> u128 { nanos / PCR_INTERVAL.as_nanos() } @@ -184,7 +195,7 @@ impl Schedule { /// decodes before one pushed ahead of it (frames read in arrival order under a zero /// delay) is due with it. pub fn push(&mut self, decode: u128, packets: Vec, keyframe: bool) { - let due = slot(decode); + let due = due(decode); for unit in self.units.iter_mut().rev() { if unit.due <= due { break; @@ -334,6 +345,11 @@ mod tests { ms * 1_000_000 } + /// A decode time `ms` in, with the drain on top, so the unit is due in `ms`'s slot. + fn at(ms: u128) -> u128 { + self::ms(ms) + DRAIN.as_nanos() + } + /// 40 packets per slot, the clock packet included. const RATE: u64 = 40 * PACKET; @@ -341,7 +357,7 @@ mod tests { fn a_unit_goes_out_in_its_due_slot() { let mut schedule = Schedule::new(Duration::from_millis(100)); schedule.set_rate(Some(RATE)); - schedule.push(ms(1_000), unit(1, 3), true); + schedule.push(at(1_000), unit(1, 3), true); // Nothing past the window is settled until a later unit shows up. let mut slots = Vec::new(); while let Some(slot) = schedule.next(Some(slot(ms(1_200)))).unwrap() { @@ -369,7 +385,7 @@ mod tests { fn a_burst_spreads_over_the_slots_before_it() { let mut schedule = Schedule::new(Duration::from_millis(100)); schedule.set_rate(Some(RATE)); - schedule.push(ms(1_000), unit(1, 100), true); + schedule.push(at(1_000), unit(1, 100), true); let mut sent = Vec::new(); while let Some(slot) = schedule.next(None).unwrap() { sent.push((slot.index, slot.packets.len() / TsPacket::SIZE)); @@ -382,7 +398,7 @@ mod tests { fn a_burst_too_big_for_the_window_fails() { let mut schedule = Schedule::new(Duration::from_millis(50)); schedule.set_rate(Some(RATE)); - schedule.push(ms(1_000), unit(1, 200), true); + schedule.push(at(1_000), unit(1, 200), true); let err = std::iter::from_fn(|| schedule.next(None).transpose()).find_map(Result::err); assert!(err.is_some(), "a burst past the window must fail the export"); } @@ -392,17 +408,17 @@ mod tests { #[test] fn a_late_rate_overruns_for_one_window() { let mut schedule = Schedule::new(Duration::from_millis(100)); - schedule.push(ms(1_000), unit(1, 4), true); + schedule.push(at(1_000), unit(1, 4), true); while schedule.next(Some(slot(ms(1_050)))).unwrap().is_some() {} schedule.set_rate(Some(RATE)); - schedule.push(ms(1_100), unit(1, 150), true); + schedule.push(at(1_100), unit(1, 150), true); let mut sent = Vec::new(); while let Some(slot) = schedule.next(None).unwrap() { sent.push((slot.index, slot.packets.len() / TsPacket::SIZE, slot.nulls)); } assert_eq!(sent, [(42, 72, 0), (43, 39, 0), (44, 39, 0)]); - schedule.push(ms(3_000), unit(1, 2_000), true); + schedule.push(at(3_000), unit(1, 2_000), true); let err = std::iter::from_fn(|| schedule.next(None).transpose()).find_map(Result::err); assert!(err.is_some(), "past the window, a burst that does not fit fails"); } @@ -410,8 +426,8 @@ mod tests { #[test] fn unpadded_units_spread_up_to_their_decode_time() { let mut schedule = Schedule::new(Duration::ZERO); - schedule.push(ms(1_000), unit(1, 4), true); - schedule.push(ms(1_100), unit(1, 4), false); + schedule.push(at(1_000), unit(1, 4), true); + schedule.push(at(1_100), unit(1, 4), false); let mut sent = Vec::new(); while let Some(slot) = schedule.next(None).unwrap() { sent.push((slot.index, slot.packets.len() / TsPacket::SIZE, slot.nulls)); From 4b7158d6c00dab0ca64f9554befeac87e015dd5f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 22:41:18 -0700 Subject: [PATCH 14/40] fix(moq-mux): finish each TS unit one packet's drain before its DTS dev's importers publish the source's own timestamps, so a 25 fps source starting at 1.4 s puts every fifth DTS on a 25 ms PCR slot boundary. An unpadded slot times its last packet up to that boundary, and the T-STD model grades the frame 0.09 ms late while the packet drains from MB to EB. Replaces the 1 ms constant with one packet at the stream's slowest T-STD rate: the level's Rbx for H.264/H.265, the transport buffer's Rx for audio. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export.rs | 235 ++++++++++++++------- rs/moq-mux/src/container/ts/export_test.rs | 109 ++++++++++ rs/moq-mux/src/container/ts/schedule.rs | 44 ++-- 3 files changed, 277 insertions(+), 111 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index d8f2f99e284..7d77064eb07 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -205,9 +205,30 @@ struct Track { /// Decode-clock reserve: how far ahead of its PTS each frame decodes. Only video sizes it; /// every other kind holds [`DEFAULT_DTS_RESERVE`]. reserve: Reserve, + /// How long a receiver takes to pass one of its packets on ([`drain`]), so its last + /// packet has to arrive that long before it decodes. + drain: Duration, } impl Track { + /// A track reading `source`, its PID filled in by the catalog. + fn new(source: ExportSource, kind: Kind) -> Self { + Self { + source, + pending: None, + empty: web_async::time::Instant::now(), + discontinuity: 0, + finished: false, + pid: 0, + kind, + descriptors: Vec::new(), + clock: DecodeClock::default(), + timeline: None, + reserve: Reserve::default(), + drain: Duration::ZERO, + } + } + /// Author the frame's decode time and queue it in the jitter buffer. /// /// The decode clock runs here, as frames are read, because the deadline is @@ -871,7 +892,10 @@ impl Export { self.last_timestamp = Some(ready.item.frame.timestamp); let decode = ready.item.decode.as_nanos(); self.mux(&name, ready.item)?; - self.lay(Some(schedule::due(decode)))?; + // A later frame decodes no earlier, so it can be due no earlier than this, + // less the longest drain of any track. + let drain = self.tracks.values().map(|track| track.drain).max().unwrap_or_default(); + self.lay(Some(schedule::slot(decode.saturating_sub(drain.as_nanos()))))?; } // 5. Once every track has drained, nothing more can ride the slots still open, so @@ -1137,47 +1161,33 @@ impl Export { let mut old = std::mem::take(&mut self.tracks); for (name, config) in catalog.video.renditions.iter() { let kind = video_kind(config, name)?; - let descriptors = track_descriptors(&mpegts, name); - let pid = pids[name]; - match old.remove(name) { - Some(mut track) => { - track.pid = pid; - track.kind = kind; - track.descriptors = descriptors; - track.reserve.configure(config, name); - self.tracks.insert(name.clone(), track); - } - None => { - let Some(source) = ExportSource::for_video(&self.source, name, config, self.delay)? else { - continue; - }; - let mut reserve = Reserve::default(); - reserve.configure(config, name); - self.insert_track(name, source, pid, kind, descriptors, reserve); - } - } + let mut track = match old.remove(name) { + Some(track) => track, + None => match ExportSource::for_video(&self.source, name, config, self.delay)? { + Some(source) => Track::new(source, kind.clone()), + None => continue, + }, + }; + track.kind = kind; + track.drain = drain(video_rate(config, name)); + track.reserve.configure(config, name); + self.refresh(name, track, pids[name], track_descriptors(&mpegts, name)); } for (name, config) in catalog.audio.renditions.iter() { let kind = audio_kind(config, name)?; - let descriptors = track_descriptors(&mpegts, name); - let pid = pids[name]; - match old.remove(name) { - Some(mut track) => { - track.pid = pid; - track.kind = kind; - track.descriptors = descriptors; - self.tracks.insert(name.clone(), track); - } - None => { - let Some(source) = ExportSource::for_audio(&self.source, name, config, self.delay)? else { - continue; - }; - self.insert_track(name, source, pid, kind, descriptors, Reserve::default()); - } - } + let mut track = match old.remove(name) { + Some(track) => track, + None => match ExportSource::for_audio(&self.source, name, config, self.delay)? { + Some(source) => Track::new(source, kind.clone()), + None => continue, + }, + }; + track.kind = kind; + track.drain = drain(audio_rate(config)); + self.refresh(name, track, pids[name], track_descriptors(&mpegts, name)); } - for (name, track) in mpegts.tracks.iter() { - let Some(verbatim) = &track.verbatim else { + for (name, entry) in mpegts.tracks.iter() { + let Some(verbatim) = &entry.verbatim else { continue; }; let kind = Kind::Verbatim { @@ -1185,24 +1195,23 @@ impl Export { framing: verbatim.framing, stream_id: verbatim.stream_id, }; - let descriptors = track.descriptors.clone(); - let pid = pids[name]; - match old.remove(name) { - Some(mut existing) => { - existing.pid = pid; - existing.kind = kind; - existing.descriptors = descriptors; - self.tracks.insert(name.clone(), existing); - } - None => { - let source = ExportSource::for_stream(&self.source, name, self.delay)?; - self.insert_track(name, source, pid, kind, descriptors, Reserve::default()); - } - } + let mut track = match old.remove(name) { + Some(track) => track, + None => Track::new(ExportSource::for_stream(&self.source, name, self.delay)?, kind.clone()), + }; + track.kind = kind; + self.refresh(name, track, pids[name], entry.descriptors.clone()); } Ok(()) } + /// Keep `track` under `name` with this snapshot's PID and descriptors. + fn refresh(&mut self, name: &str, mut track: Track, pid: u16, descriptors: Vec) { + track.pid = pid; + track.descriptors = descriptors; + self.tracks.insert(name.to_string(), track); + } + /// Point each stale track this snapshot lists at the returned broadcast. /// /// The PIDs and PMT stay as announced. A track the returned catalog does not list stays @@ -1232,34 +1241,6 @@ impl Export { Ok(()) } - /// Insert a freshly created export track. - fn insert_track( - &mut self, - name: &str, - source: ExportSource, - pid: u16, - kind: Kind, - descriptors: Vec, - reserve: Reserve, - ) { - self.tracks.insert( - name.to_string(), - Track { - source, - pending: None, - empty: web_async::time::Instant::now(), - discontinuity: 0, - finished: false, - pid, - kind, - descriptors, - clock: DecodeClock::default(), - timeline: None, - reserve, - }, - ); - } - /// How many frames were dropped for missing their deadline. #[cfg(test)] pub(super) fn dropped(&self) -> u64 { @@ -1641,7 +1622,9 @@ impl Export { self.write_pes(&mut out, &unit, &es_payload)?; } } - self.schedule.push(decode.as_nanos(), out, keyframe); + let drain = self.tracks.get(name).map_or(Duration::ZERO, |track| track.drain); + self.schedule + .push(decode.as_nanos().saturating_sub(drain.as_nanos()), out, keyframe); Ok(()) } @@ -1700,7 +1683,7 @@ impl Export { /// The clock packet for grid slot `index`. /// /// The value runs one slot behind the boundary, so the bytes of slot `index` are timed - /// up to its boundary: a unit due in that slot decodes no earlier ([`schedule::due`]). + /// up to its boundary: a unit due in that slot decodes no earlier ([`schedule::slot`]). /// Back off through the 33-bit wrap rather than saturating: a timeline that starts in /// its first slot would otherwise clamp to zero and break the uniform step. The wire /// field is a circular clock, so the masked wrapped value is the correct mod-2^33 @@ -2018,6 +2001,94 @@ fn video_kind(config: &VideoConfig, name: &str) -> anyhow::Result { } } +/// How long a receiver takes to pass one transport packet on at `rate` bits per second. +/// +/// A packet's bytes reach a decoder buffer only once it has drained through the T-STD +/// buffers ahead of it (ISO 13818-1 2.4.2), so a unit's last packet has to arrive at least +/// this long before its decode time. Without it a unit whose DTS sits on a PCR slot +/// boundary, as a source's own 25 fps timestamps put every fifth frame, arrives complete +/// only after it decodes. +fn drain(rate: u64) -> Duration { + Duration::from_nanos(TsPacket::SIZE as u64 * 8 * 1_000_000_000 / rate.max(1)) +} + +/// The slowest rate on a video stream's T-STD path, in bits per second: the MB to EB leak, +/// Rbx = CpbBrNalFactor * MaxBR for its level (H.222.0 2.14.3.1, 2.17.2; H.264 Table A-1, +/// H.265 Table A.8). The transport buffer's Rx is never slower. A level the tables don't list +/// takes the lowest, which only sends its units earlier. +fn video_rate(config: &VideoConfig, name: &str) -> u64 { + let rate = match &config.codec { + VideoCodec::H264(h264) => { + // level_idc 11 with constraint_set3_flag is level 1b for Baseline, Main and Extended. + let level_1b = h264.level == 11 && h264.constraints & 0x10 != 0 && matches!(h264.profile, 66 | 77 | 88); + let max_br = match h264.level { + _ if level_1b => Some(128), + 9 => Some(128), + 10 => Some(64), + 11 => Some(192), + 12 => Some(384), + 13 => Some(768), + 20 => Some(2_000), + 21 | 22 => Some(4_000), + 30 => Some(10_000), + 31 => Some(14_000), + 32 | 40 => Some(20_000), + 41 | 42 => Some(50_000), + 50 => Some(135_000), + 51 | 52 | 60 => Some(240_000), + 61 => Some(480_000), + 62 => Some(800_000), + _ => None, + }; + max_br.map(|max_br| 1_200 * max_br) + } + VideoCodec::H265(h265) => { + let max_br = match (h265.level_idc, h265.tier_flag) { + (30, false) => Some(128), + (60, false) => Some(1_500), + (63, false) => Some(3_000), + (90, false) => Some(6_000), + (93, false) => Some(10_000), + (120, false) => Some(12_000), + (120, true) => Some(30_000), + (123, false) => Some(20_000), + (123, true) => Some(50_000), + (150, false) => Some(25_000), + (150, true) => Some(100_000), + (153, false) => Some(40_000), + (153, true) => Some(160_000), + (156 | 180, false) => Some(60_000), + (156 | 180, true) => Some(240_000), + (183, false) => Some(120_000), + (183, true) => Some(480_000), + (186, false) => Some(240_000), + (186, true) => Some(800_000), + _ => None, + }; + max_br.map(|max_br| 1_100 * max_br) + } + _ => None, + }; + rate.unwrap_or_else(|| { + tracing::warn!(track = %name, codec = %config.codec, "no T-STD rate for this level; sending its units early"); + 1_200 * 64 + }) +} + +/// The rate an audio stream's transport buffer drains at, in bits per second (H.222.0 +/// 2.4.2.3): by channel count for ADTS AAC, 2 Mb/s for every other audio. +fn audio_rate(config: &AudioConfig) -> u64 { + if !matches!(config.codec, AudioCodec::AAC(_)) { + return 2_000_000; + } + match config.channel_count { + 0..=2 => 2_000_000, + 3..=8 => 5_529_600, + 9..=12 => 8_294_400, + _ => 33_177_600, + } +} + /// Build the Annex-B elementary-stream payload for one video frame: rewrite the /// length-prefixed NALs to start-code-delimited NALs, prepending the parameter /// sets (SPS/PPS, plus VPS for H.265) from the avcC/hvcC on keyframes so a diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index db3df8990bd..b9986ddf2fd 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -4695,6 +4695,115 @@ async fn export_cbr_video() -> Vec { drain_frames(&mut exporter).await } +/// A frame whose DTS sits on a PCR slot boundary still arrives in time to decode. A source +/// publishing its own 25 fps timestamps puts every fifth frame there, and an unpadded slot's +/// last packet is timed right up to the boundary. A receiver still has to drain that packet +/// into the decoder buffer, at the level's Rbx (16.8 Mb/s for level 3.1), so the frame has to +/// finish one packet's drain before it decodes. +#[tokio::test(start_paused = true)] +async fn a_frame_on_a_slot_boundary_finishes_before_it_decodes() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let track = broadcast + .create_track( + broadcast.unique_name(".h264"), + hang::container::track_info(hang::catalog::PRIORITY.video), + ) + .unwrap(); + { + let mut cfg = VideoConfig::new(H264 { + profile: 0x42, + constraints: 0xc0, + level: 0x1f, + inline: true, + }); + cfg.container = Container::Legacy; + // A whole number of frames of reserve, so each DTS is an earlier frame's PTS. + cfg.jitter = Some(Duration::from_millis(80)); + catalog + .modify() + .unwrap() + .video + .renditions + .insert(track.name().to_string(), cfg); + } + let mut video = Producer::new(track, HangContainer::Legacy(crate::container::Kind::Data)); + for i in 0..50u64 { + let keyframe = i % 25 == 0; + let mut nal = vec![if keyframe { 0x65u8 } else { 0x41 }]; + nal.extend(std::iter::repeat_n(0xAB, 2_000)); + if keyframe && i > 0 { + video.cut(None).unwrap(); + } + video + .write(Frame { + // ffmpeg's 1.4 s start, on a slot boundary like every fifth frame after it. + timestamp: Timestamp::from_micros(1_400_000 + i * 40_000).unwrap(), + duration: None, + payload: if keyframe { + annexb(&[SPS, PPS, &nal]) + } else { + annexb(&[&nal]) + }, + keyframe, + }) + .unwrap(); + } + video.finish().unwrap(); + + let mut exporter = Export::new(crate::source::announced(&consumer)).await.unwrap(); + let frames = drain_frames(&mut exporter).await; + let ts: Vec = frames.iter().flat_map(|f| f.payload.iter().copied()).collect(); + let packets: Vec<&[u8]> = ts.chunks(188).collect(); + let pcrs: Vec<(usize, f64)> = collect_pcrs(&frames) + .into_iter() + .map(|(at, base, _)| (at, base as f64)) + .collect(); + // The time a receiver assigns the start of packet `at`, interpolated between PCRs. + let time = |at: usize| { + let k = pcrs.partition_point(|&(index, _)| index <= at).checked_sub(1)?; + let (&(a, ta), &(b, tb)) = (pcrs.get(k)?, pcrs.get(k + 1)?); + Some(ta + (tb - ta) * (at - a) as f64 / (b - a) as f64) + }; + + // Each video PES: its DTS (else PTS), and the packet it ends on. + let mut units: Vec<(u64, usize)> = Vec::new(); + let mut video_pid = None; + let mut reader = TsPacketReader::new(Cursor::new(ts.clone())); + let mut at = 0; + while let Some(packet) = reader.read_ts_packet().unwrap() { + let pid = packet.header.pid.as_u16(); + match packet.payload { + Some(TsPayload::Pmt(pmt)) => video_pid = pmt.es_info.first().map(|es| es.elementary_pid.as_u16()), + Some(TsPayload::PesStart(pes)) if Some(pid) == video_pid => { + let decode = pes.header.dts.or(pes.header.pts).unwrap().as_u64(); + units.push((decode, at)); + } + Some(TsPayload::PesContinuation(_)) if Some(pid) == video_pid => units.last_mut().unwrap().1 = at, + _ => {} + } + at += 1; + } + assert_eq!(packets.len(), at); + + // One packet at 16.8 Mb/s is 89.5 us, a little over 8 ticks of 90 kHz. + let drain = 188.0 * 8.0 / 16_800_000.0 * 90_000.0; + let mut on_boundary = 0; + for &(decode, last) in &units { + let Some(end) = time(last + 1) else { continue }; + on_boundary += usize::from(decode % 2_250 == 0); + assert!( + decode as f64 - end >= drain, + "the unit decoding at {decode} finishes at {end:.1}, less than a packet's drain before it" + ); + } + assert!( + on_boundary >= 5, + "too few units on a slot boundary to judge: {on_boundary}" + ); +} + /// #3334: the clock a receiver recovers from *byte position* has to agree with the /// values, because a byte stream carries no other timing. Emitting each media frame /// whole put every PCR between two frames instead of among the bytes it labels, so diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index 4d55a6eea51..2ec25e908ff 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -23,19 +23,9 @@ const PACKET: u64 = TsPacket::SIZE as u64 * 8 * SLOTS_PER_SECOND; /// a span that carried no bytes only stalls anything pacing on the asserted values. const BACKFILL: u128 = 40; -/// How long before its decode time a unit's last packet has to arrive. A receiver still -/// moves that packet through its transport and multiplex buffers, which drain no slower -/// than 2 Mb/s (ISO 13818-1 2.4.2.3), so 0.75 ms a packet. -const DRAIN: Duration = Duration::from_millis(1); - -/// The grid slot a unit decoding at `nanos` must be sent by: its bytes are timed up to the -/// slot's boundary, which has to fall a [`DRAIN`] before the decode time. -pub(super) fn due(nanos: u128) -> u128 { - slot(nanos.saturating_sub(DRAIN.as_nanos())) -} - -/// The grid slot `nanos` falls in. -fn slot(nanos: u128) -> u128 { +/// The grid slot a unit whose last byte has to arrive by `nanos` must be sent by: the +/// slot's bytes are timed up to its boundary. +pub(super) fn slot(nanos: u128) -> u128 { nanos / PCR_INTERVAL.as_nanos() } @@ -188,14 +178,15 @@ impl Schedule { } } - /// Queue an access unit (whole TS packets) that decodes at `decode` nanoseconds. + /// Queue an access unit (whole TS packets) whose last byte has to arrive by `by` + /// nanoseconds: its decode time, less what the receiver needs to pass the last packet on. /// /// Units go out in the order they are pushed, which keeps the program tables ahead of /// what they describe. The jitter buffer hands them over in decode order; a unit that /// decodes before one pushed ahead of it (frames read in arrival order under a zero /// delay) is due with it. - pub fn push(&mut self, decode: u128, packets: Vec, keyframe: bool) { - let due = due(decode); + pub fn push(&mut self, by: u128, packets: Vec, keyframe: bool) { + let due = slot(by); for unit in self.units.iter_mut().rev() { if unit.due <= due { break; @@ -345,11 +336,6 @@ mod tests { ms * 1_000_000 } - /// A decode time `ms` in, with the drain on top, so the unit is due in `ms`'s slot. - fn at(ms: u128) -> u128 { - self::ms(ms) + DRAIN.as_nanos() - } - /// 40 packets per slot, the clock packet included. const RATE: u64 = 40 * PACKET; @@ -357,7 +343,7 @@ mod tests { fn a_unit_goes_out_in_its_due_slot() { let mut schedule = Schedule::new(Duration::from_millis(100)); schedule.set_rate(Some(RATE)); - schedule.push(at(1_000), unit(1, 3), true); + schedule.push(ms(1_000), unit(1, 3), true); // Nothing past the window is settled until a later unit shows up. let mut slots = Vec::new(); while let Some(slot) = schedule.next(Some(slot(ms(1_200)))).unwrap() { @@ -385,7 +371,7 @@ mod tests { fn a_burst_spreads_over_the_slots_before_it() { let mut schedule = Schedule::new(Duration::from_millis(100)); schedule.set_rate(Some(RATE)); - schedule.push(at(1_000), unit(1, 100), true); + schedule.push(ms(1_000), unit(1, 100), true); let mut sent = Vec::new(); while let Some(slot) = schedule.next(None).unwrap() { sent.push((slot.index, slot.packets.len() / TsPacket::SIZE)); @@ -398,7 +384,7 @@ mod tests { fn a_burst_too_big_for_the_window_fails() { let mut schedule = Schedule::new(Duration::from_millis(50)); schedule.set_rate(Some(RATE)); - schedule.push(at(1_000), unit(1, 200), true); + schedule.push(ms(1_000), unit(1, 200), true); let err = std::iter::from_fn(|| schedule.next(None).transpose()).find_map(Result::err); assert!(err.is_some(), "a burst past the window must fail the export"); } @@ -408,17 +394,17 @@ mod tests { #[test] fn a_late_rate_overruns_for_one_window() { let mut schedule = Schedule::new(Duration::from_millis(100)); - schedule.push(at(1_000), unit(1, 4), true); + schedule.push(ms(1_000), unit(1, 4), true); while schedule.next(Some(slot(ms(1_050)))).unwrap().is_some() {} schedule.set_rate(Some(RATE)); - schedule.push(at(1_100), unit(1, 150), true); + schedule.push(ms(1_100), unit(1, 150), true); let mut sent = Vec::new(); while let Some(slot) = schedule.next(None).unwrap() { sent.push((slot.index, slot.packets.len() / TsPacket::SIZE, slot.nulls)); } assert_eq!(sent, [(42, 72, 0), (43, 39, 0), (44, 39, 0)]); - schedule.push(at(3_000), unit(1, 2_000), true); + schedule.push(ms(3_000), unit(1, 2_000), true); let err = std::iter::from_fn(|| schedule.next(None).transpose()).find_map(Result::err); assert!(err.is_some(), "past the window, a burst that does not fit fails"); } @@ -426,8 +412,8 @@ mod tests { #[test] fn unpadded_units_spread_up_to_their_decode_time() { let mut schedule = Schedule::new(Duration::ZERO); - schedule.push(at(1_000), unit(1, 4), true); - schedule.push(at(1_100), unit(1, 4), false); + schedule.push(ms(1_000), unit(1, 4), true); + schedule.push(ms(1_100), unit(1, 4), false); let mut sent = Vec::new(); while let Some(slot) = schedule.next(None).unwrap() { sent.push((slot.index, slot.packets.len() / TsPacket::SIZE, slot.nulls)); From 4822c071402f00a1da4ad80efd354391b6c4c743 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 10:02:43 -0700 Subject: [PATCH 15/40] fix(moq-mux): keep every TS track on one jitter clock A consumer skipping a stale group at join counted a discontinuity, and the video opened a generation of its own while audio and tables stayed on the first, so the tracks ran up to a second apart. A discontinuity before anything went out now keeps the clock, and once one opens a generation every track's next frame runs on it. Generations go out in turn, so an older one can no longer follow a rewind. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/import.rs | 58 +++--- rs/moq-mux/src/jitter.rs | 264 +++++++++++++++----------- 2 files changed, 189 insertions(+), 133 deletions(-) diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 184a670919a..201adbdfb60 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -6023,34 +6023,44 @@ pub(super) mod test { } } - /// Import `data`, re-export the broadcast to MPEG-TS, and count the exported packets - /// whose adaptation field sets `discontinuity_indicator`. - async fn export_discontinuities(data: &[u8]) -> usize { + /// Import `before`, re-export the broadcast to MPEG-TS until it has gone out, then import + /// `after` and count the exported packets whose adaptation field sets + /// `discontinuity_indicator`. + async fn export_discontinuities(before: &[u8], after: &[u8]) -> usize { let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); let mut import = super::Import::new(broadcast, catalog.reserve()); - import.decode(&bytes::BytesMut::from(data)).unwrap(); - import.finish().unwrap(); + import.decode(&bytes::BytesMut::from(before)).unwrap(); - // `import` and `catalog` stay alive so the exporter can subscribe to the finished, - // retained tracks. A generous delay, since the default of zero would shed the - // earlier groups and the boundary with them. The paused clock runs out the - // delay and every gap in the media without a real wait. + // `import` and `catalog` stay alive so the exporter can subscribe to the retained + // tracks. A generous delay, since the default of zero would shed the earlier groups + // and the boundary with them. The paused clock runs out the delay and every gap in + // the media without a real wait. let delay = std::time::Duration::from_secs(3600); let mut exporter = crate::container::ts::Export::new(crate::source::announced(&consumer)) .await .unwrap() .with_delay(delay); - let mut flagged = 0; - while let Ok(Ok(Some(frame))) = tokio::time::timeout(2 * delay, exporter.next()).await { - flagged += frame + let flags = |frame: crate::container::Frame| { + frame .payload .as_chunks::<188>() .0 .iter() .filter(|p| p[3] & 0x20 != 0 && p[4] > 0 && p[5] & 0x80 != 0) - .count(); + .count() + }; + // A discontinuity before anything went out moves no clock, so the media before the + // boundary goes out first, as it would live. + let mut flagged = 0; + while let Ok(Ok(Some(frame))) = tokio::time::timeout(2 * delay, exporter.next()).await { + flagged += flags(frame); + } + import.decode(&bytes::BytesMut::from(after)).unwrap(); + import.finish().unwrap(); + while let Ok(Ok(Some(frame))) = tokio::time::timeout(2 * delay, exporter.next()).await { + flagged += flags(frame); } flagged } @@ -6075,21 +6085,21 @@ pub(super) mod test { }; let mut cc = 0; - let mut signalled = synth_pmt(&[(StreamType::Mpeg1Audio, PID)], false); - media(&mut signalled, &mut cc, 90_000); - let mut unsignalled = signalled.clone(); - signalled.extend_from_slice(&clock_break_packet(PID)); + let mut before = synth_pmt(&[(StreamType::Mpeg1Audio, PID)], false); + media(&mut before, &mut cc, 90_000); + let mut signalled = clock_break_packet(PID); + let mut unsignalled = Vec::new(); let mut peer_cc = cc; media(&mut signalled, &mut cc, 31 * 90_000); media(&mut unsignalled, &mut peer_cc, 31 * 90_000); assert_eq!( - export_discontinuities(&signalled).await, + export_discontinuities(&before, &signalled).await, 1, "the declared reset never reached the exported clock" ); assert_eq!( - export_discontinuities(&unsignalled).await, + export_discontinuities(&before, &unsignalled).await, 0, "a leap the source did not declare must not be flagged" ); @@ -6116,9 +6126,13 @@ pub(super) mod test { let mut cc = 0; media(&mut stimulus, &mut cc, 90_000); - stimulus.extend_from_slice(&clock_break_packet(PCR_PID)); - media(&mut stimulus, &mut cc, 31 * 90_000); + let mut after = clock_break_packet(PCR_PID); + media(&mut after, &mut cc, 31 * 90_000); - assert_eq!(export_discontinuities(&stimulus).await, 1, "one flag per program break"); + assert_eq!( + export_discontinuities(&stimulus, &after).await, + 1, + "one flag per program break" + ); } } diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index 10d2ac53615..5577adbd9f8 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -13,7 +13,7 @@ use web_async::time::{Instant, Sleep}; pub(crate) struct Arrival { /// When the frame was read from its source. pub arrived: Instant, - /// When the frame decodes, nondecreasing within a track between discontinuities. + /// When the frame decodes, nondecreasing within a track. pub decode: Timestamp, /// The source's discontinuity counter when the frame was read. pub discontinuity: u64, @@ -25,8 +25,8 @@ pub(crate) struct Arrival { /// A frame [`Buffer::poll_next`] let go. pub(crate) struct Ready { pub track: K, - /// Counts up each time a track's discontinuity started a new anchor. Tracks that - /// cross the same discontinuity share one. + /// Counts up each time a discontinuity moved the timeline, and the clock with it. Every + /// frame of one generation goes out before any frame of the next. pub generation: u64, pub item: T, } @@ -42,37 +42,45 @@ pub(crate) enum Push { } /// Holds each track's frames until a fixed delay past their decode time, like an SRT -/// receiver's TSBPD, in `(deadline, track)` order across tracks. +/// receiver's TSBPD, in decode order across tracks. /// -/// The clock is anchored at the first frame's arrival. Within one anchor the deadline -/// order is the decode order, so two buffers that saw the same frames arrive with +/// Every track runs on one clock, anchored at the first frame's arrival. Under it the +/// deadline order is the decode order, so two buffers that saw the same frames arrive with /// different skew, each within its deadline, emit them in the same order. A frame that /// arrives past its deadline would break that order, so it is dropped and counted. /// -/// A track whose source reports a discontinuity may have restarted its timeline, so it -/// moves to a new generation with its own anchor: the newest one when another track got -/// there first and the frame lands on its clock, otherwise one anchored at this frame, -/// though never ahead of a deadline already given out. A backlog read in one go still -/// releases each generation after the one before it, and a live stream keeps its delay. +/// A source's discontinuity (a skipped group, a declared marker) may have moved its +/// timeline, so once frames have gone out it opens a new generation with its own clock, +/// anchored at that frame though never ahead of a deadline already given out. Every +/// track's next frame runs on it: one crossing the same discontinuity joins it when the +/// frame lands on its clock, neither late nor held more than a delay past everything +/// queued, and opens another otherwise. Before anything has gone out there is nothing to +/// keep in step with, so a discontinuity there (a consumer skipping a stale group at join) +/// keeps the clock. No two tracks are ever on different clocks. /// /// A zero delay holds nothing and drops nothing: each frame goes out as soon as it is /// read, ordered only among the frames read together. pub(crate) struct Buffer { delay: Duration, - /// Each generation's anchor: the first frame's arrival and decode time. - anchors: BTreeMap, + /// The clock every frame is pushed under: its generation, and the arrival and decode + /// time of the frame that anchored it. + clock: Option<(u64, Instant, Timestamp)>, tracks: BTreeMap>, /// The latest deadline given out. horizon: Option, + /// Whether a frame has gone out since the clock started. + released: bool, timer: Option>>, dropped: u64, } struct Track { - /// Frames in arrival order, each with its deadline and generation. - queue: VecDeque<(Instant, u64, T)>, - generation: u64, + /// Frames in arrival order, each with its generation and deadline. + queue: VecDeque<(u64, Instant, T)>, + /// The source's discontinuity counter at the last frame. discontinuity: u64, + /// The generation of the last frame. + generation: u64, /// A frame was dropped, so later ones are too until the next sync frame. waiting: bool, } @@ -81,9 +89,10 @@ impl Buffer { pub fn new(delay: Duration) -> Self { Self { delay, - anchors: BTreeMap::new(), + clock: None, tracks: BTreeMap::new(), horizon: None, + released: false, timer: None, dropped: 0, } @@ -91,36 +100,35 @@ impl Buffer { /// Queue a frame, or drop it if it cannot make its deadline. pub fn push(&mut self, key: K, arrival: Arrival) -> Push { - // A new track counts from the start, as if it had always been there. - let (mut generation, discontinuity) = self - .tracks - .get(&key) - .map_or((0, 0), |track| (track.generation, track.discontinuity)); - if discontinuity != arrival.discontinuity { - generation = self.rejoin(generation, &arrival); - } - let anchor = *self.anchors.entry(generation).or_insert_with(|| { - let after = self.horizon.and_then(|horizon| horizon.checked_sub(self.delay)); - ( - after.map_or(arrival.arrived, |after| after.max(arrival.arrived)), - arrival.decode, - ) + let (mut generation, anchor, base) = *self.clock.get_or_insert((0, arrival.arrived, arrival.decode)); + let mut deadline = self.deadline(anchor, base, arrival.decode); + // A track still on an older generation may be crossing the discontinuity another + // track opened the newest one for. + let jumped = self.tracks.get(&key).is_some_and(|track| { + track.discontinuity != arrival.discontinuity + && (track.generation == generation || !self.lands(deadline, arrival.arrived)) }); - let deadline = self.deadline(anchor, arrival.decode); + if jumped && self.released { + generation += 1; + let after = self.horizon.and_then(|horizon| horizon.checked_sub(self.delay)); + let anchor = after.map_or(arrival.arrived, |after| after.max(arrival.arrived)); + self.clock = Some((generation, anchor, arrival.decode)); + deadline = self.deadline(anchor, arrival.decode, arrival.decode); + } let track = self.tracks.entry(key).or_insert_with(|| Track { queue: VecDeque::new(), + discontinuity: arrival.discontinuity, generation, - discontinuity: 0, waiting: false, }); - track.generation = generation; track.discontinuity = arrival.discontinuity; + track.generation = generation; let push = match deadline { _ if track.waiting && !arrival.sync => Push::Waiting, Some(deadline) if arrival.arrived <= deadline || self.delay.is_zero() => { track.waiting = false; - track.queue.push_back((deadline, generation, arrival.item)); + track.queue.push_back((generation, deadline, arrival.item)); self.horizon = self.horizon.max(Some(deadline)); Push::Queued } @@ -133,36 +141,22 @@ impl Buffer { if push != Push::Queued { self.dropped += 1; } - - // Generation 0 stays, since a track that has yet to show up starts there. - let oldest = self.tracks.values().map(|track| track.generation).min(); - self.anchors - .retain(|generation, _| *generation == 0 || oldest.is_none_or(|oldest| *generation >= oldest)); push } - /// The generation a track at `generation` moves to on a discontinuity: the newest, if - /// another track opened it and this frame lands on its clock, or else a new one. - /// - /// Landing on the clock means the frame is neither late nor held past both an on-time - /// frame and everything already queued, which a different timeline would be. - fn rejoin(&self, generation: u64, arrival: &Arrival) -> u64 { - let Some((&latest, &anchor)) = self.anchors.last_key_value() else { - return generation + 1; - }; - let lands = self.deadline(anchor, arrival.decode).is_some_and(|deadline| { - let bound = (arrival.arrived + self.delay).max(self.horizon.unwrap_or(arrival.arrived)); - arrival.arrived <= deadline && deadline <= bound - }); - match latest > generation && lands { - true => latest, - false => latest.max(generation) + 1, - } + /// Whether a frame arriving at `arrived` and due at `deadline` is on the clock: not late, + /// and held no more than a delay past both its arrival and everything already queued. + fn lands(&self, deadline: Option, arrived: Instant) -> bool { + deadline.is_some_and(|deadline| { + let bound = arrived.max(self.horizon.unwrap_or(arrived)) + self.delay; + arrived <= deadline && deadline <= bound + }) } - /// When a frame decoding at `decode` goes out under `anchor`, if any instant holds it. - fn deadline(&self, (anchor, base): (Instant, Timestamp), decode: Timestamp) -> Option { - // Signed, since a frame can decode before the one that anchored its generation. + /// When a frame decoding at `decode` goes out on the clock anchored at `anchor` and + /// `base`, if any instant holds it. + fn deadline(&self, anchor: Instant, base: Timestamp, decode: Timestamp) -> Option { + // Signed, since a frame can decode before the one that anchored the clock. let offset = decode.as_nanos() as i128 - base.as_nanos() as i128 + self.delay.as_nanos() as i128; let nanos = u64::try_from(offset.unsigned_abs()).ok()?; match offset >= 0 { @@ -171,17 +165,25 @@ impl Buffer { } } - /// The next frame whose deadline has come, earliest deadline first, ties by track. - pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll> { - let Some((deadline, key)) = self - .tracks + /// The generation, deadline and track of the frame that goes out next: generation by + /// generation, earliest deadline first, ties by track. + fn front(&self) -> Option<(u64, Instant, &K)> { + self.tracks .iter() - .filter_map(|(key, track)| track.queue.front().map(|(deadline, ..)| (*deadline, key))) + .filter_map(|(key, track)| { + let (generation, deadline, _) = track.queue.front()?; + Some((*generation, *deadline, key)) + }) .min() - else { + } + + /// The next frame whose deadline has come, in [`Self::front`] order. + pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll> { + let Some((_, deadline, key)) = self.front() else { self.timer = None; return Poll::Pending; }; + let key = key.clone(); if !self.delay.is_zero() && Instant::now() < deadline { let timer = self .timer @@ -193,10 +195,10 @@ impl Buffer { return Poll::Pending; } } - let track = key.clone(); - let (_, generation, item) = self.tracks.get_mut(&track).and_then(|t| t.queue.pop_front()).unwrap(); + let (generation, _, item) = self.tracks.get_mut(&key).and_then(|t| t.queue.pop_front()).unwrap(); + self.released = true; Poll::Ready(Ready { - track, + track: key, generation, item, }) @@ -205,10 +207,7 @@ impl Buffer { /// When the next queued frame is due. #[cfg(test)] pub fn next_deadline(&self) -> Option { - self.tracks - .values() - .filter_map(|track| track.queue.front().map(|(deadline, ..)| *deadline)) - .min() + self.front().map(|(_, deadline, _)| deadline) } /// Whether no frame is waiting. @@ -221,11 +220,12 @@ impl Buffer { self.dropped } - /// Drop every queued frame and anchor, so the next frame starts the clock afresh. + /// Drop every queued frame and the clock, so the next frame starts it afresh. pub fn clear(&mut self) { - self.anchors.clear(); + self.clock = None; self.tracks.clear(); self.horizon = None; + self.released = false; self.timer = None; } } @@ -250,16 +250,29 @@ mod tests { } } - /// Every frame due by now, in order. - fn due(buffer: &mut Buffer) -> Vec<&'static str> { + /// The same frame, read after its source's discontinuity counter moved to `discontinuity`. + fn after(discontinuity: u64, arrival: Arrival<&'static str>) -> Arrival<&'static str> { + Arrival { + discontinuity, + ..arrival + } + } + + /// Every frame due by now, in order, with its generation. + fn released(buffer: &mut Buffer) -> Vec<(&'static str, u64)> { let waiter = kio::Waiter::noop(); let mut out = Vec::new(); while let Poll::Ready(ready) = buffer.poll_next(&waiter) { - out.push(ready.item); + out.push((ready.item, ready.generation)); } out } + /// Every frame due by now, in order. + fn due(buffer: &mut Buffer) -> Vec<&'static str> { + released(buffer).into_iter().map(|(item, _)| item).collect() + } + #[tokio::test(start_paused = true)] async fn releases_at_the_delay_in_decode_order() { let start = Instant::now(); @@ -339,49 +352,78 @@ mod tests { assert_eq!(due(&mut buffer), ["v0", "a0", "a180", "v240"]); } - /// A discontinuity re-anchors the track, and a track reaching the same discontinuity - /// later joins that anchor rather than making its own. + /// A consumer that skips a stale group at join counts a discontinuity before anything + /// went out. The skip keeps the clock, so the tracks stay on one. #[tokio::test(start_paused = true)] - async fn a_discontinuity_re_anchors() { + async fn a_skip_before_the_first_release_keeps_one_clock() { let start = Instant::now(); let mut buffer = Buffer::new(DELAY); - buffer.push(1, arrival(start, 5_000, "v5000")); - buffer.push(2, arrival(start, 5_000, "a5000")); - - // The publisher restarts its timeline at zero, a second in. - let restart = start + Duration::from_secs(1); - let reset = |arrived, decode, item| Arrival { - discontinuity: 1, - ..arrival(arrived, decode, item) - }; - assert_eq!(buffer.push(1, reset(restart, 0, "v0")), Push::Queued); - // Audio still carries its old timeline for a moment, then restarts too. - assert_eq!(buffer.push(2, arrival(restart, 6_000, "a6000")), Push::Queued); - let later = restart + Duration::from_millis(10); - assert_eq!(buffer.push(2, reset(later, 0, "a0")), Push::Queued); - - tokio::time::advance(Duration::from_secs(1) + DELAY).await; - assert_eq!(due(&mut buffer), ["v5000", "a5000", "v0", "a6000", "a0"]); + // The video's first group is stale and anchors the clock; the audio is live. + buffer.push(1, arrival(start, 0, "v0")); + buffer.push(2, arrival(start, 900, "a900")); + // The video consumer skips to the live group. + let skip = start + Duration::from_millis(5); + assert_eq!(buffer.push(1, after(1, arrival(skip, 900, "v900"))), Push::Queued); + assert_eq!(buffer.push(1, after(1, arrival(skip, 940, "v940"))), Push::Queued); + assert_eq!(buffer.push(2, arrival(skip, 940, "a940")), Push::Queued); + + tokio::time::advance(Duration::from_secs(2)).await; + assert_eq!( + released(&mut buffer), + [("v0", 0), ("v900", 0), ("a900", 0), ("v940", 0), ("a940", 0)] + ); } - /// A backlog read in one go releases a new generation after the one before it. + /// A discontinuity after frames went out opens a new generation, and every track's next + /// frame runs on it, whether or not that track saw the discontinuity. #[tokio::test(start_paused = true)] - async fn a_backlog_releases_generations_in_turn() { + async fn every_track_joins_a_new_generation_at_its_next_frame() { let start = Instant::now(); let mut buffer = Buffer::new(DELAY); - buffer.push(1, arrival(start, 5_000, "v5000")); - buffer.push(1, arrival(start, 5_040, "v5040")); - let reset = Arrival { - discontinuity: 1, - ..arrival(start, 0, "v0") - }; - buffer.push(1, reset); - buffer.push(2, arrival(start, 5_020, "a5020")); + buffer.push(1, arrival(start, 0, "v0")); + buffer.push(2, arrival(start, 0, "a0")); + buffer.push(3, arrival(start, 0, "d0")); + tokio::time::advance(DELAY).await; + assert_eq!(due(&mut buffer), ["v0", "a0", "d0"]); + + // The publisher resumes a second later with its timeline five seconds on. + let resume = start + Duration::from_secs(1); + tokio::time::advance(resume - Instant::now()).await; + assert_eq!(buffer.push(1, after(1, arrival(resume, 5_000, "v5000"))), Push::Queued); + // The audio crosses no discontinuity of its own; the data crosses the same one. + let late = resume + Duration::from_millis(10); + assert_eq!(buffer.push(2, arrival(late, 5_000, "a5000")), Push::Queued); + assert_eq!(buffer.push(3, after(1, arrival(late, 5_000, "d5000"))), Push::Queued); + assert_eq!(buffer.push(2, arrival(late, 5_020, "a5020")), Push::Queued); tokio::time::advance(DELAY).await; - assert_eq!(due(&mut buffer), ["v5000"]); - tokio::time::advance(Duration::from_millis(40)).await; - assert_eq!(due(&mut buffer), ["a5020", "v5040", "v0"]); + assert_eq!(released(&mut buffer), [("v5000", 1), ("a5000", 1), ("d5000", 1)]); + tokio::time::advance(Duration::from_millis(20)).await; + assert_eq!(released(&mut buffer), [("a5020", 1)]); + } + + /// Every frame of a generation goes out before the next generation's, even one the new + /// clock puts earlier. + #[tokio::test(start_paused = true)] + async fn generations_go_out_in_turn() { + let start = Instant::now(); + let mut buffer = Buffer::new(DELAY); + buffer.push(1, arrival(start, 0, "v0")); + buffer.push(1, arrival(start, 40, "v40")); + buffer.push(2, arrival(start, 20, "a20")); + tokio::time::advance(DELAY).await; + assert_eq!(due(&mut buffer), ["v0"]); + + // A backlog across a jump in the timeline, read in one go. + let now = Instant::now(); + buffer.push(1, after(1, arrival(now, 5_000, "v5000"))); + buffer.push(2, arrival(now, 4_900, "a4900")); + buffer.push(2, arrival(now, 5_020, "a5020")); + tokio::time::advance(Duration::from_secs(1)).await; + assert_eq!( + released(&mut buffer), + [("a20", 0), ("v40", 0), ("a4900", 1), ("v5000", 1), ("a5020", 1)] + ); } /// Zero holds nothing: frames read together go out at once in decode order, and a From a1f3f5fa72f8634122bdec27d35fc23393b27de2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 10:19:12 -0700 Subject: [PATCH 16/40] fix(moq-mux): author TS video DTS in time, not picture counts The decode clock held back a fixed count of pictures, so a field-coded passage decoded half as early as a frame-coded one and an open GOP's leading pictures bunched one tick apart. Each frame now decodes in the slot of the next presentation time, a reorder delay early, once no later frame can be presented before it: the catalog jitter, the SPS depth, or the reordering seen bound that lookahead. On the Kyrion 1080i capture the authored DTS is the encoder's, frame for frame. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export.rs | 505 ++++++++++----------- rs/moq-mux/src/container/ts/export_test.rs | 173 ++++--- 2 files changed, 357 insertions(+), 321 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 7d77064eb07..07cd32c9fbd 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -197,14 +197,10 @@ struct Track { /// PMT ES-level descriptors to re-announce, captured verbatim on import (language, /// registration, ...). Empty for non-TS sources; AC-3/E-AC-3 then synthesize one. descriptors: Vec, - /// Authors each video frame's DTS from its PTS. - clock: DecodeClock, - /// High-water mark of the timestamps read within this rendition, which sizes the - /// video [`Reserve`] from how far a reordered frame lands below it. - timeline: Option, - /// Decode-clock reserve: how far ahead of its PTS each frame decodes. Only video sizes it; - /// every other kind holds [`DEFAULT_DTS_RESERVE`]. - reserve: Reserve, + /// Authors each video frame's DTS, holding the frames until it can. + clock: DecodeClock<(Pending, Option)>, + /// What the catalog and the SPS declare about the video's reordering. + timing: Timing, /// How long a receiver takes to pass one of its packets on ([`drain`]), so its last /// packet has to arrive that long before it decodes. drain: Duration, @@ -223,55 +219,58 @@ impl Track { kind, descriptors: Vec::new(), clock: DecodeClock::default(), - timeline: None, - reserve: Reserve::default(), + timing: Timing::default(), drain: Duration::ZERO, } } - /// Author the frame's decode time and queue it in the jitter buffer. + /// Queue a frame read from the source in the jitter buffer, once its decode time is known. /// - /// The decode clock runs here, as frames are read, because the deadline is - /// on decode time: reordered video arrives in decode order with PTS 0, 120, 40, 80, - /// so a PTS deadline would strand a B-frame behind its reference or send it first. + /// The decode clock runs here, as frames are read, because the deadline is on decode time: + /// reordered video arrives in decode order with PTS 0, 120, 40, 80, so a PTS deadline would + /// strand a B-frame behind its reference or send it first. fn queue(&mut self, name: &str, pending: Pending, jitter: &mut jitter::Buffer) { - let Pending { - frame, - discontinuity, - arrived, - } = pending; - if discontinuity != self.discontinuity { + if pending.discontinuity != self.discontinuity { // The source may have restarted its timeline, so the decode clock restarts too. - self.discontinuity = discontinuity; + self.release(name, jitter, true); + self.discontinuity = pending.discontinuity; self.clock = DecodeClock::default(); - self.timeline = None; } - - let (dts, sync, description) = match self.kind { - Kind::Video(stream_type) => { - if frame.keyframe { - self.reserve.describe(stream_type, self.source.description(), name); - } - if let Some(timeline) = self.timeline - && frame.timestamp < timeline - { - let gap = to_ticks(timeline) - to_ticks(frame.timestamp); - self.reserve.observe(gap, name); - } else { - self.reserve.peak(); - } - let pts = to_ticks(frame.timestamp); - let dts = self.clock.author(pts, self.reserve.ticks, self.reserve.period()); - ( - (dts != pts).then_some(dts), - frame.keyframe, - self.source.description().cloned(), - ) - } - _ => (None, true, None), + let Kind::Video(stream_type) = self.kind else { + return self.push(name, pending, None, None, jitter); }; - self.timeline = Some(self.timeline.map_or(frame.timestamp, |last| last.max(frame.timestamp))); + if pending.frame.keyframe { + self.timing.describe(stream_type, self.source.description(), name); + } + let pts = to_ticks(pending.frame.timestamp); + self.clock.push((pending, self.source.description().cloned()), pts); + self.release(name, jitter, false); + } + /// Queue every held video frame whose DTS is settled, or all of them when `flush`. + fn release(&mut self, name: &str, jitter: &mut jitter::Buffer, flush: bool) { + let (delay, lookahead) = self.timing.reorder(); + let lookahead = lookahead.max(self.timing.jitter); + while let Some(((pending, description), dts)) = self.clock.pop(lookahead, delay, flush) { + self.push(name, pending, Some(dts), description, jitter); + } + } + + /// Queue a frame decoding at `dts` (90 kHz ticks), else at its PTS. + fn push( + &mut self, + name: &str, + pending: Pending, + dts: Option, + description: Option, + jitter: &mut jitter::Buffer, + ) { + let Pending { + frame, + discontinuity, + arrived, + } = pending; + let dts = dts.filter(|&dts| dts != to_ticks(frame.timestamp)); let decode = dts .and_then(|ticks| Timestamp::from_scale(ticks, 90_000).ok()) .unwrap_or(frame.timestamp); @@ -279,7 +278,7 @@ impl Track { arrived, decode, discontinuity, - sync, + sync: frame.keyframe || !matches!(self.kind, Kind::Video(_)), item: Queued { frame, decode, @@ -293,56 +292,24 @@ impl Track { } } -/// A video rendition's decode-clock reserve in 90 kHz ticks: how far each frame's DTS runs -/// behind its PTS. It must exceed the rendition's reordering for `DTS <= PTS` to hold. -/// -/// The catalog `jitter` (the importer's max PTS - DTS) sizes it when published. Otherwise the -/// reorder depth the SPS declares sizes it, at the SPS's fixed picture rate or the catalog -/// `framerate`, so two exporters agree on it from their first keyframe. Otherwise it holds -/// [`DEFAULT_DTS_RESERVE`]. Reordering deeper than all of those raises it further -/// ([`Self::observe`]). -/// -/// It never shrinks: that would step the decode clock forward past frames already read. -struct Reserve { - /// The catalog `jitter`, in ticks. - jitter: Option, +/// What a video rendition declares about its reordering, which bounds its [`DecodeClock`]. +#[derive(Default)] +struct Timing { + /// The catalog `jitter` in 90 kHz ticks: the most any frame decodes before it is presented. + jitter: u64, /// The catalog `framerate`: the picture period for an SPS that declares no fixed rate. framerate: Option, /// The avcC/hvcC [`Self::declared`] was read from. description: Option, declared: Option, - /// The reserve covering the deepest reordering muxed so far. - observed: u64, - /// Frames muxed since the one that set the rendition's high-water mark. - since_peak: u64, - /// The reserve in effect. - ticks: u64, } -impl Default for Reserve { - fn default() -> Self { - Self { - jitter: None, - framerate: None, - description: None, - declared: None, - observed: 0, - since_peak: 0, - ticks: DEFAULT_DTS_RESERVE, - } - } -} - -impl Reserve { +impl Timing { /// Take the rendition's timing from a catalog snapshot, before or after the program /// tables are written. - fn configure(&mut self, config: &VideoConfig, name: &str) { - self.jitter = config - .jitter - .map(|t| (t.as_micros() * 90_000 / 1_000_000) as u64) - .filter(|&ticks| ticks > 0); + fn configure(&mut self, config: &VideoConfig) { + self.jitter = config.jitter.map_or(0, |t| (t.as_micros() * 90_000 / 1_000_000) as u64); self.framerate = config.framerate.filter(|fps| fps.is_finite() && *fps > 0.0); - self.settle(name, "catalog"); } /// Read the reorder depth from the codec config a keyframe is carried with. @@ -352,70 +319,27 @@ impl Reserve { } self.description = description.cloned(); self.declared = description.and_then(|d| declared_reorder(stream_type, d)); - self.settle(name, "sps"); + tracing::debug!(track = %name, reorder = ?self.reorder(), "video reordering declared"); } - /// Account for a frame muxed `gap` ticks below the rendition's high-water mark. + /// The least reorder delay and lookahead the SPS declares, in ticks, so two exporters agree + /// on them from their first keyframe whatever reordering each has seen. /// - /// This is the one path where the reserve depends on when an exporter joined rather than on - /// what the stream declares: a joiner adopts the depth only once it has muxed the deepest - /// reordering itself, and until then its DTS and PCR differ from an exporter that has. It - /// fires only when the catalog and the SPS both undercount the stream's reordering (or - /// declare none), and each growth is logged. - /// - /// The decode clock steps at least one tick per frame, so a frame decoded `k` frames after - /// the one that set the high-water mark needs `k` ticks beyond its gap. - fn observe(&mut self, gap: u64, name: &str) { - self.since_peak += 1; - let covering = gap.saturating_add(self.since_peak).min(MAX_DTS_RESERVE); - if covering > self.observed { - self.observed = covering; - self.settle(name, "observed"); - } - } - - /// Account for a frame at or above the rendition's high-water mark. - fn peak(&mut self) { - self.since_peak = 0; - } - - /// The picture period in ticks the SPS or the catalog `framerate` declares. - fn period(&self) -> Option { - match self.declared?.period { - Some((units, scale)) => Some(units.saturating_mul(90_000).div_ceil(scale)), - None => Some((90_000.0 / self.framerate?).ceil() as u64), - } - .filter(|&period| period > 0) - } - - fn settle(&mut self, name: &str, source: &'static str) { - // As in [`Self::observe`], a reordered frame needs a tick for each frame decoded between - // it and the one above it; the declared depth stands in for that count. - let period = self.period(); - let declared = self.declared.and_then(|reorder| { - let period = period?; - let depth = u64::from(reorder.depth); - let ticks = depth.saturating_mul(period).saturating_add(depth.max(1)); - Some(ticks.min(MAX_DTS_RESERVE)) - }); - let target = self - .jitter - .or(declared) - .unwrap_or(DEFAULT_DTS_RESERVE) - .max(self.observed); - if target > self.ticks { - if source == "observed" { - tracing::warn!( - track = %name, - from = self.ticks, - to = target, - "reordering deeper than the catalog or SPS declares; raising the video DTS reserve" - ); - } else { - tracing::info!(track = %name, from = self.ticks, to = target, source, "raising the video DTS reserve"); - } - self.ticks = target; - } + /// The delay is the SPS's depth in pictures, at its picture period or the catalog + /// `framerate`'s. A frame can be presented up to `2^depth - 1` pictures below one decoded + /// before it (a B-pyramid that deep), which bounds the lookahead the catalog `jitter` + /// gives when it is not published. + fn reorder(&self) -> (u64, u64) { + let Some(declared) = self.declared else { + return (0, 0); + }; + let period = match declared.period { + Some((units, scale)) => units.saturating_mul(90_000).div_ceil(scale), + None => self.framerate.map_or(0, |fps| (90_000.0 / fps).ceil() as u64), + }; + let delay = u64::from(declared.depth).saturating_mul(period); + let lookahead = 1u64.checked_shl(declared.depth).unwrap_or(u64::MAX).saturating_mul(period); + (delay.min(MAX_REORDER), lookahead.min(MAX_REORDER)) } } @@ -1011,6 +935,10 @@ impl Export { } } } + // Nothing more can come in below the frames still held for their DTS. + if track.finished && !waiting_for_header { + track.release(name, &mut self.jitter, true); + } } Ok(()) } @@ -1122,7 +1050,7 @@ impl Export { // after the tables still sizes the decode clock. for (name, config) in catalog.video.renditions.iter() { if let Some(track) = self.tracks.get_mut(name) { - track.reserve.configure(config, name); + track.timing.configure(config); } } return Ok(()); @@ -1170,7 +1098,7 @@ impl Export { }; track.kind = kind; track.drain = drain(video_rate(config, name)); - track.reserve.configure(config, name); + track.timing.configure(config); self.refresh(name, track, pids[name], track_descriptors(&mpegts, name)); } for (name, config) in catalog.audio.renditions.iter() { @@ -1286,7 +1214,6 @@ impl Export { track.pending = None; track.discontinuity = 0; track.clock = DecodeClock::default(); - track.timeline = None; self.stale.insert(name.clone()); } // The replacement's clock starts afresh at its own first frame. @@ -1886,16 +1813,11 @@ fn pcr_packet(pid: u16, ticks: u64) -> anyhow::Result> { const PES_OPTIONAL_LEN: usize = 3 + 5; /// Extra bytes when the optional region also carries a DTS (5 DTS bytes). const PES_DTS_LEN: usize = 5; -/// Fallback decode-clock reserve in 90 kHz ticks when neither the catalog `jitter` nor the SPS -/// sizes one. At 16 ticks (~0.18 ms) it is just a strict-monotonic nudge: it keeps DTS strictly -/// increasing across reordered (B-frame) decode order (the `ffplay -fflags +igndts` fix) but -/// does not keep `DTS <= PTS` until reordering raises it. See [`author_dts`] and [`Reserve`]. -const DEFAULT_DTS_RESERVE: u64 = 16; - -/// Largest reserve an SPS or observed reordering can raise a track to (2 s): far past any real -/// reorder depth, so a corrupt SPS or a timestamp stepping back within a timeline cannot drag -/// the PCR back without bound. -const MAX_DTS_RESERVE: u64 = 2 * 90_000; +/// Largest reorder delay or lookahead an SPS or observed reordering can raise a video decode +/// clock to (2 s, in 90 kHz ticks): far past any real reorder depth, so a corrupt SPS or a +/// timestamp stepping back within a timeline cannot hold frames or drag the DTS back without +/// bound. +const MAX_REORDER: u64 = 2 * 90_000; /// Whether `timestamp` has crossed into a later repetition slot than `last`. /// @@ -2234,65 +2156,95 @@ fn ensure_raw(container: &Container, kind: &str, name: &str) -> anyhow::Result<( /// A video rendition's decode clock: a decode timestamp (DTS) for each frame from its PTS. /// /// [`Frame`] carries only a presentation timestamp and frames reach the muxer in decode order, -/// so a B-frame stream arrives with non-monotonic PTS and no decode time. The clock hands out -/// the presentation times in display order, `depth` frames late, the way a decoder's reorder -/// buffer of that depth would (ffmpeg's `pts_buffer`): with depth 2, decode-order PTS 0, 120, -/// 40, 80 decode at -80, -40, 0, 40, one frame period apart. `depth` is how many frame periods -/// fit in the track's [`Reserve`]; the remainder of the reserve backs every DTS off a little -/// further. So a reference frame decodes a whole reserve before it is presented and a B-frame -/// keeps its own slot rather than bunching one tick behind its reference. +/// so a B-frame stream arrives with non-monotonic PTS and no decode time. An encoder decodes +/// the n-th frame in decode order at the n-th presentation time, a fixed reorder delay early, +/// so that is the DTS authored here: with decode-order PTS 0, 120, 40, 80 and a delay of one +/// 40 ms frame, the frames decode at -40, 0, 40, 80. It keeps the encoder's own spacing, a +/// frame's or a field's, through any switch between frame and field coding, and the delay is +/// a time, the least that keeps every DTS at or before its PTS (or the SPS's declared depth, +/// when that is more). On a source muxed that way this is its DTS, frame for frame. /// -/// The period is the one the stream declares, else the smallest step seen between consecutive -/// PTS. Without either the clock runs a reserve behind the PTS. A DTS that would not advance -/// (reordering deeper than the reserve) is nudged one tick past the last, so the clock stays -/// strictly increasing either way. Ticks are continuous (unwrapped) 90 kHz, so the clock never -/// wraps mid-stream; the 33-bit wire wrap happens once at emission. -#[derive(Default)] -struct DecodeClock { - /// Presentation times read but not yet handed out, ascending. - window: Vec, +/// A frame's slot is settled once no frame still to come can be presented before it. A later +/// frame decodes after this one, and is presented at most the catalog `jitter` after it +/// decodes, so the slot is settled once the highest PTS read is that far past it. Frames wait +/// here until then, as long as the reordering spans and well inside the jitter buffer's delay. +/// Without a `jitter`, the SPS's depth bounds it, and so does how far a frame was presented +/// below the highest PTS read before it. A frame that still comes in below a settled slot +/// (reordering deeper than all of those) is nudged one tick past the last DTS, so the clock +/// stays strictly increasing. +/// +/// Ticks are continuous (unwrapped) 90 kHz, so the clock never wraps mid-stream; the 33-bit +/// wire wrap happens once at emission. +struct DecodeClock { + /// Frames read but not yet given a DTS, in decode order, with their PTS. + held: VecDeque<(T, u64)>, + /// The held frames' PTS, ascending: the slots they decode in, in turn. + slots: Vec, + /// The highest PTS read. + high: Option, + /// The furthest a frame was presented below the highest PTS read before it, plus a tick. + reach: u64, + /// The reorder delay in effect. It never shrinks: that would step the clock back. + delay: u64, /// The last DTS handed out. last: Option, - /// The previous frame's PTS, to measure the frame period. - previous: Option, - /// The smallest positive step seen between consecutive PTS. - period: Option, } -impl DecodeClock { - /// The DTS of a frame presented at `pts`, the next in decode order. - fn author(&mut self, pts: u64, reserve: u64, declared: Option) -> u64 { - if let Some(step) = self - .previous - .map(|previous| pts.abs_diff(previous)) - .filter(|&step| step > 0) +impl Default for DecodeClock { + fn default() -> Self { + Self { + held: VecDeque::new(), + slots: Vec::new(), + high: None, + reach: 0, + delay: 0, + last: None, + } + } +} + +impl DecodeClock { + /// Hold the next frame in decode order, presented at `pts`. + fn push(&mut self, item: T, pts: u64) { + if let Some(high) = self.high + && pts < high { - self.period = Some(self.period.map_or(step, |period| period.min(step))); - } - self.previous = Some(pts); - - let dts = match declared.or(self.period) { - Some(period) => { - let depth = (reserve / period) as usize; - let at = self.window.partition_point(|&p| p < pts); - self.window.insert(at, pts); - // Until the window holds `depth + 1` frames, the earliest is still to come, so - // count back from the earliest seen as a decoder that ran all along would have. - let held = self.window.len(); - let earliest = match held > depth { - true => self.window.drain(..held - depth).next_back().unwrap_or(pts), - false => self.window[0].saturating_sub((depth + 1 - held) as u64 * period), - }; - earliest.saturating_sub(reserve % period) + self.reach = self.reach.max(high - pts + 1).min(MAX_REORDER); + } + self.high = Some(self.high.map_or(pts, |high| high.max(pts))); + let at = self.slots.partition_point(|&slot| slot <= pts); + self.slots.insert(at, pts); + self.held.push_back((item, pts)); + } + + /// The next frame and its DTS, once its slot is settled `lookahead` past (the catalog + /// `jitter`), or at once when `flush`. `floor` is the least reorder delay, in ticks. + fn pop(&mut self, lookahead: u64, floor: u64, flush: bool) -> Option<(T, u64)> { + let high = self.high?; + let lookahead = lookahead.max(self.reach); + let settled = |slot: u64| flush || high >= slot.saturating_add(lookahead); + if !settled(*self.slots.first()?) { + return None; + } + // Every settled frame bounds the delay, so one decoding later never steps it back. + let mut delay = self.delay.max(floor); + for (&slot, (_, pts)) in self.slots.iter().zip(&self.held) { + if !settled(slot) { + break; } - None => pts.saturating_sub(reserve), - }; + delay = delay.max(slot.saturating_sub(*pts)); + } + self.delay = delay.min(MAX_REORDER); + + let slot = self.slots.remove(0); + let (item, _) = self.held.pop_front()?; + let dts = slot.saturating_sub(self.delay); let dts = match self.last { Some(last) if dts <= last => last + 1, _ => dts, }; self.last = Some(dts); - dts + Some((item, dts)) } } @@ -2315,20 +2267,28 @@ mod tests { use moq_net::Timestamp; - use super::{ - DEFAULT_DTS_RESERVE, DecodeClock, PCR_INTERVAL, PSI_INTERVAL, due, is_complete_section, si_due, slot, - slot_ticks, - }; + use super::{DecodeClock, PCR_INTERVAL, PSI_INTERVAL, due, is_complete_section, si_due, slot, slot_ticks}; fn ms(value: u64) -> Timestamp { Timestamp::from_millis(value).unwrap() } - /// Push a decode-order PTS stream (90 kHz, 25 fps declared) through the decode clock with a - /// given reserve and return the DTS per frame. - fn run_clock(pts: &[u64], reserve: u64) -> Vec { + /// Push a decode-order PTS stream (90 kHz) through the decode clock under a catalog + /// `jitter` and a declared reorder `floor`, flushing at the end, and return the DTS per + /// frame. + fn run_clock(pts: &[u64], jitter: u64, floor: u64) -> Vec { let mut clock = DecodeClock::default(); - pts.iter().map(|&p| clock.author(p, reserve, Some(3_600))).collect() + let mut dts = vec![0; pts.len()]; + for (i, &p) in pts.iter().enumerate() { + clock.push(i, p); + while let Some((i, d)) = clock.pop(jitter, floor, false) { + dts[i] = d; + } + } + while let Some((i, d)) = clock.pop(jitter, floor, true) { + dts[i] = d; + } + dts } /// Decode-order PTS for a constant-frame-rate display timeline with `b` B-frames between @@ -2349,36 +2309,30 @@ mod tests { out } + /// How an encoder times `pts`: the n-th frame decodes at the n-th presentation time, a + /// fixed `delay` early. Returns the DTS and the catalog `jitter` an importer would publish. + fn source_dts(pts: &[u64], delay: u64) -> (Vec, u64) { + let mut slots = pts.to_vec(); + slots.sort(); + let dts: Vec = slots.iter().map(|slot| slot - delay).collect(); + let jitter = pts.iter().zip(&dts).map(|(p, d)| p - d).max().unwrap(); + (dts, jitter) + } + #[test] fn dts_is_monotonic_across_reorder() { - // 25 fps, 10 s offset. Even with the tiny fallback reserve the decode timeline is - // strictly increasing (the `+igndts` fix); it just may sit above PTS for B-frames. + // 25 fps, 10 s offset. Even with nothing declared, the decode timeline is strictly + // increasing (the `+igndts` fix), and past the first reordered frames it stays at or + // before the PTS. for b in [1, 3, 5] { let pts = decode_order(40, b, 3_600, 10_000_000); - let dts = run_clock(&pts, DEFAULT_DTS_RESERVE); - - // The fixture genuinely reorders (PTS dips in decode order). assert!(pts.windows(2).any(|w| w[1] < w[0]), "b={b}: stream must reorder PTS"); + let dts = run_clock(&pts, 0, 0); for (i, win) in dts.windows(2).enumerate() { assert!(win[1] > win[0], "b={b}: DTS not strictly increasing at {i}: {win:?}"); } - } - } - - #[test] - fn sufficient_reserve_keeps_dts_under_pts() { - // With a reserve covering the reorder span (the catalog `jitter` carries it), the decode - // timeline is both strictly increasing and never after the PTS. - let dur = 3_600; - for b in [1, 3, 5] { - let reserve = (b as u64 + 1) * dur; // one frame past the b-frame run - let pts = decode_order(40, b, dur, 10_000_000); - let dts = run_clock(&pts, reserve); - - for (i, win) in dts.windows(2).enumerate() { - assert!(win[1] > win[0], "b={b}: DTS not strictly increasing at {i}: {win:?}"); - } - for (i, (&d, &p)) in dts.iter().zip(pts.iter()).enumerate() { + let settled = 2 * (b + 1); + for (i, (&d, &p)) in dts.iter().zip(pts.iter()).enumerate().skip(settled) { assert!(d <= p, "b={b}: DTS {d} after PTS {p} at {i}"); } } @@ -2391,7 +2345,7 @@ mod tests { // only at emission, so here the authored DTS keeps climbing past 1 << 33. let wrap = 1u64 << 33; let pts = decode_order(40, 3, 3_600, wrap - 20 * 3_600); - let dts = run_clock(&pts, DEFAULT_DTS_RESERVE); + let dts = run_clock(&pts, 0, 0); assert!(pts.iter().any(|&p| p >= wrap), "test must cross the wrap boundary"); for (i, win) in dts.windows(2).enumerate() { @@ -2403,45 +2357,68 @@ mod tests { } #[test] - fn dts_without_reorder_trails_pts_by_the_reserve() { - // A monotonic (no-B) stream stays strictly increasing and one reserve under its PTS. + fn dts_without_reorder_is_the_pts() { let pts: Vec = (0..40).map(|i| 10_000_000 + i * 3_600).collect(); - let dts = run_clock(&pts, DEFAULT_DTS_RESERVE); - - for (i, win) in dts.windows(2).enumerate() { - assert!(win[1] > win[0], "DTS not strictly increasing at {i}: {win:?}"); - } - for (i, (&d, &p)) in dts.iter().zip(pts.iter()).enumerate() { - assert_eq!(d, p - DEFAULT_DTS_RESERVE, "DTS should trail PTS by the reserve at {i}"); - } + assert_eq!(run_clock(&pts, 0, 0), pts); } - /// B-frames decode a frame period apart, not one tick behind their reference. + /// Given the catalog `jitter` and the SPS's reorder depth, a reordered stream decodes + /// exactly as its encoder timed it, a frame period apart. #[test] - fn reordered_dts_is_frame_spaced() { + fn reordered_dts_is_the_encoders() { let dur = 3_600; for b in [1, 2, 3] { - let reserve = b as u64 * dur + 1; let pts = decode_order(40, b, dur, 10_000_000); - let dts = run_clock(&pts, reserve); - for (i, win) in dts.windows(2).enumerate() { - assert_eq!(win[1] - win[0], dur, "b={b}: DTS not one frame apart at {i}: {dts:?}"); - } - for (i, (&d, &p)) in dts.iter().zip(pts.iter()).enumerate() { - assert!(d <= p, "b={b}: DTS {d} after PTS {p} at {i}"); - } + let (source, jitter) = source_dts(&pts, dur); + assert_eq!(run_clock(&pts, jitter, dur), source, "b={b}"); + } + } + + /// A field-coded passage decodes a field apart and a frame-coded one a frame apart: the + /// reorder delay is a time, so a switch between them does not squeeze the timeline. + #[test] + fn a_switch_to_field_coding_keeps_the_encoders_spacing() { + let frame = 3_600; + let mut pts = decode_order(10, 2, frame, 10_000_000); + // Then each picture is two fields, a reference pair ahead of the B pairs it predicts. + let start = pts.iter().max().unwrap() + frame; + for g in 0..10u64 { + let reference = start + (g * 3 + 2) * frame; + let pair = |at: u64| [at, at + frame / 2]; + pts.extend(pair(reference)); + pts.extend(pair(reference - 2 * frame)); + pts.extend(pair(reference - frame)); + } + let (source, jitter) = source_dts(&pts, frame); + assert_eq!(run_clock(&pts, jitter, frame), source); + } + + /// An open GOP's leading pictures, read after the keyframe they precede in presentation, + /// decode a slot apart rather than one tick behind it. + #[test] + fn leading_pictures_decode_a_slot_apart() { + let frame = 3_600; + let base = 10_000_000; + // The keyframe is presented after three leading B-frames. + let mut pts = vec![base + 3 * frame, base, base + frame, base + 2 * frame]; + pts.extend(decode_order(10, 2, frame, base + 3 * frame).into_iter().skip(1)); + let (source, jitter) = source_dts(&pts, frame); + let dts = run_clock(&pts, jitter, frame); + assert_eq!(dts, source); + for (i, win) in dts.windows(2).enumerate() { + assert!(win[1] - win[0] >= frame, "DTS less than a frame apart at {i}: {dts:?}"); } } #[test] - fn author_dts_is_join_independent_at_a_peak() { + fn dts_is_join_independent_at_a_peak() { // An exporter that has been running and one that just joined author the same decode // timeline from any frame whose PTS leads everything decoded before it. A keyframe is - // exactly that (export only ever tunes in on one), so the monotonic bump cannot fire - // there and the state carried across the join stops mattering. + // exactly that (export only ever tunes in on one). for b in [1, 3, 5] { let pts = decode_order(40, b, 3_600, 10_000_000); - let running = run_clock(&pts, DEFAULT_DTS_RESERVE); + let (_, jitter) = source_dts(&pts, 3_600); + let running = run_clock(&pts, jitter, 3_600); let mut peaks = 0; for k in 1..pts.len() { @@ -2449,7 +2426,7 @@ mod tests { continue; } peaks += 1; - let fresh = run_clock(&pts[k..], DEFAULT_DTS_RESERVE); + let fresh = run_clock(&pts[k..], jitter, 3_600); assert_eq!( &running[k..], &fresh[..], diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index b9986ddf2fd..d586950499b 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -615,9 +615,9 @@ async fn export_import_h265_keeps_suffix_sei_on_its_picture() { /// A real broadcast contribution feed (Ateme Kyrion, H.264 1080i with ~86 B-frames) /// must come out of the exporter with an authored decode timeline. The importer publishes -/// the reorder depth as the catalog `jitter`, and the exporter sizes its decode-clock reserve -/// from it, so the video PES carry a DTS that is both strictly increasing and never after the -/// PTS in decode order. Also assert the reorder was real (non-monotonic PTS in the source). +/// the reorder depth as the catalog `jitter`, and the exporter authors a decode timeline from it, +/// so the video PES carry a DTS that is both strictly increasing and never after the PTS in +/// decode order. Also assert the reorder was real (non-monotonic PTS in the source). #[tokio::test(start_paused = true)] async fn export_bframe_video_authors_dts() { let data = include_bytes!("test_data/scte35/kyrion_dirtystart.ts"); @@ -671,7 +671,7 @@ async fn export_bframe_video_authors_dts() { // The exporter authored a decode timeline (the decode clock trails the PTS). assert!(authored > 0, "no DTS authored for a B-frame stream"); // Strictly increasing (removes the `+igndts` requirement) and never after presentation - // (the catalog jitter sized the reserve to the reorder depth). + // (the catalog jitter bounds how far ahead the decode clock looks). for (i, win) in effective.windows(2).enumerate() { assert!(win[1] > win[0], "DTS not strictly increasing at frame {i}: {win:?}"); } @@ -680,6 +680,65 @@ async fn export_bframe_video_authors_dts() { } } +/// `(PTS, DTS)` of every video PES start, in transport (decode) order. Read raw, so a capture +/// that starts before its PMT still counts. +fn video_pes_timing(ts: &[u8]) -> Vec<(u64, u64)> { + let stamp = |b: &[u8]| { + u64::from((b[0] >> 1) & 7) << 30 + | u64::from(b[1]) << 22 + | u64::from(b[2] >> 1) << 15 + | u64::from(b[3]) << 7 + | u64::from(b[4] >> 1) + }; + let mut out = Vec::new(); + for packet in ts.as_chunks::<188>().0 { + if packet[1] & 0x40 == 0 || packet[3] & 0x10 == 0 { + continue; + } + let start = 4 + if packet[3] & 0x20 != 0 { usize::from(packet[4]) + 1 } else { 0 }; + let Some(pes) = packet.get(start..).filter(|pes| pes.len() >= 19) else { + continue; + }; + if pes[..3] != [0, 0, 1] || !(0xe0..=0xef).contains(&pes[3]) || pes[7] & 0x80 == 0 { + continue; + } + let pts = stamp(&pes[9..14]); + let dts = if pes[7] & 0x40 != 0 { stamp(&pes[14..19]) } else { pts }; + out.push((pts, dts)); + } + out +} + +/// The export decodes a broadcast feed exactly where its encoder did. The Kyrion capture is +/// field-coded 1080i, two fields a frame, with B-frames: the decode clock steps a field at a +/// time, so a held-back picture count would decode it half as early as a frame-coded stream, +/// and its leading pictures would bunch one tick apart. +#[tokio::test(start_paused = true)] +async fn export_decodes_where_the_source_did() { + let data = include_bytes!("test_data/scte35/kyrion_dirtystart.ts"); + let source: std::collections::BTreeMap = video_pes_timing(data).into_iter().collect(); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut import = crate::container::ts::Import::new(broadcast, catalog.reserve()); + import.decode(&BytesMut::from(&data[..])).unwrap(); + import.finish().unwrap(); + let ts = drain(consumer).await; + + let exported = video_pes_timing(&ts); + assert!(exported.len() > 100, "expected the full feed, got {} frames", exported.len()); + // The capture ends mid-reorder, so the last frame's slot is one the encoder filled with a + // picture past the end. Timestamps cross MoQ in microseconds, so each is within a tick. + for (i, &(pts, dts)) in exported[..exported.len() - 1].iter().enumerate() { + let (_, &want) = source + .range(pts - 1..=pts + 1) + .next() + .unwrap_or_else(|| panic!("frame {i} presented at {pts} is not in the source")); + assert!(dts.abs_diff(want) <= 2, "frame {i} decodes at {dts}, the source at {want}"); + } +} + /// #2937: the PCR must be a uniform bounded-interval ramp, not a sample of the /// per-unit decode clock. On a reordered (B-frame) capture the authored DTS is a /// saw (reference frames leap a reorder span, B-frames nudge one tick), so a PCR @@ -832,11 +891,10 @@ async fn export_pcr_wraps_below_zero_at_start() { } } -/// Every rendition decodes after the clock, not just the PCR track: a second -/// rendition with a deeper reorder (catalog `jitter`) authors its DTS further -/// behind the PTS, and the spans close on those decode times. +/// Every rendition decodes after the clock, not just the PCR track, whatever `jitter` each +/// declares. #[tokio::test(start_paused = true)] -async fn export_pcr_respects_every_renditions_reserve() { +async fn export_pcr_respects_every_rendition() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); @@ -858,8 +916,7 @@ async fn export_pcr_respects_every_renditions_reserve() { catalog.modify().unwrap().video.renditions.insert(name.to_string(), cfg); Producer::new(track, HangContainer::Legacy(crate::container::Kind::Data)) }; - // "a" gets the lowest PID and so carries the PCR, with the tiny default - // reserve; "b" declares a 100 ms reorder depth. + // "a" gets the lowest PID and so carries the PCR; "b" declares a 100 ms `jitter`. let mut a = make("a.avc1", None); let mut b = make("b.avc1", Some(Duration::from_millis(100))); @@ -2708,8 +2765,8 @@ async fn reordered_video_keeps_the_table_cadence() { assert_eq!(export.discontinuity(), 0, "a reorder is not a rewind"); assert_eq!(count_pid(&out, 0x0000), 10, "PAT once per 500ms slot, not per reorder"); assert_eq!(count_pid(&out, 0x0011), 3, "SDT once per 2s slot, not per reorder"); - // The SPS declares no reorder depth, so the first B-frame grows the reserve. It is read - // a delay ahead of the first PCR, so the clock starts on the grown reserve and never + // The SPS declares no reorder depth, so the first B-frame grows the reorder delay. It is + // read a delay ahead of the first PCR, so the clock starts on the grown delay and never // steps back. assert_eq!(count_discontinuity(&out), 0, "a reorder does not restart the clock"); } @@ -4428,14 +4485,15 @@ async fn si_reordered_frames_earn_no_emission_credit() { .unwrap(); assert_eq!(rig.media(2_400, 0x0011).await, 0, "a reordered frame earns no credit"); - // The floor measures from the 2.5s anchor: not due at 3.4s, due at 3.5s. - assert_eq!(rig.media(3_400, 0x0011).await, 0, "not due within the floor"); - assert_eq!( - rig.media(3_500, 0x0011).await, - 1, - "the deferred revision rides the frame at the floor" - ); - assert_eq!(rig.media(3_600, 0x0011).await, 0, "and only that one"); + // The floor measures from the 2.5s anchor: not due at 3.4s, due at 3.5s. Since the reorder, + // each frame waits for one read past its reorder span before its DTS is known, so count + // across the reads that let each frame out. + let early = rig.media(3_400, 0x0011).await + rig.media(3_500, 0x0011).await + rig.media(3_600, 0x0011).await; + assert_eq!(early, 0, "not due within the floor"); + let floor = rig.media(3_700, 0x0011).await + rig.media(3_800, 0x0011).await; + assert_eq!(floor, 1, "the deferred revision rides the frame at the floor"); + let later = rig.media(3_900, 0x0011).await + rig.media(4_000, 0x0011).await; + assert_eq!(later, 0, "and only that one"); assert_eq!(rig.exporter.discontinuity(), 0, "B-frame reordering is not a rewind"); } @@ -4461,9 +4519,9 @@ async fn si_anchor_survives_a_zero_interval_reorder() { "the 1s emission, and the anchor advancing to 3s" ); // A reordered frame steps back behind the anchor, into the previous 1s slot; - // zero interval still emits. It decodes just after the 3s frame, so it shares that - // frame's last slot and waits with it. + // zero interval still emits. It waits for a read past its reorder span, which lets it out. assert_eq!(rig.media(2_900, 0x0011).await, 0, "the reordered frame waits"); + let reordered = rig.media(3_100, 0x0011).await; // The catalog raises the interval to 1s. The grid slot must be the 3s anchor's, // not the reordered 2.9s emission's. @@ -4480,9 +4538,12 @@ async fn si_anchor_survives_a_zero_interval_reorder() { .interval = Some(Duration::from_secs(1)); // The decode clock catches up on the reorder over the next frames, so count the // emissions across them rather than per frame. - let held = rig.media(3_400, 0x0011).await + rig.media(3_900, 0x0011).await; + let held = reordered + rig.media(3_400, 0x0011).await + rig.media(3_900, 0x0011).await; assert_eq!(held, 1, "only the reordered emission, still in the anchor's slot"); - let due = rig.media(4_000, 0x0011).await + rig.media(4_100, 0x0011).await + rig.media(4_200, 0x0011).await; + let due = rig.media(4_000, 0x0011).await + + rig.media(4_100, 0x0011).await + + rig.media(4_200, 0x0011).await + + rig.media(4_300, 0x0011).await; assert_eq!(due, 1, "the 4s table, once"); assert_eq!(rig.exporter.discontinuity(), 0, "B-frame reordering is not a rewind"); } @@ -4719,7 +4780,7 @@ async fn a_frame_on_a_slot_boundary_finishes_before_it_decodes() { inline: true, }); cfg.container = Container::Legacy; - // A whole number of frames of reserve, so each DTS is an earlier frame's PTS. + // Each DTS is an earlier frame's PTS. cfg.jitter = Some(Duration::from_millis(80)); catalog .modify() @@ -5170,9 +5231,9 @@ async fn export_stuffing_keeps_the_fractional_remainder() { assert!(clocked.nulls > 0, "no null stuffing was emitted"); } -// The decode clock follows the stream's reordering: its reserve comes from the catalog `jitter`, -// else the depth the SPS declares, else the reordering muxed so far, and it keeps following all -// three after the program tables are written. +// The decode clock follows the stream's reordering: its delay comes from the depth the SPS +// declares and the reordering muxed so far, its lookahead also from the catalog `jitter`, and it +// keeps following all three after the program tables are written. /// Display offsets within a group, in decode order: a closed GOP with one B-frame per reference. /// A B-frame lands one frame below the high-water mark. @@ -5421,9 +5482,9 @@ fn assert_decodes_after_the_clock(frames: &[Frame]) { } /// With no `jitter`, an SPS that declares its reorder depth at a fixed frame rate sizes the -/// reserve from the first keyframe: every B-frame decodes at or before it is presented from the -/// start, and an exporter whose B-frames arrive behind the audio covering them renders the same -/// bytes as one that sees each tick whole. +/// reorder delay from the first keyframe: every B-frame decodes at or before it is presented +/// from the start, and an exporter whose B-frames arrive behind the audio covering them renders +/// the same bytes as one that sees each tick whole. #[tokio::test(start_paused = true)] async fn declared_reorder_sizes_the_decode_clock_from_the_first_frame() { let mut rig = Reordered::new( @@ -5453,8 +5514,8 @@ async fn declared_reorder_sizes_the_decode_clock_from_the_first_frame() { let timing = video_timing(&out_late, ..); assert_decodes_before_presenting(&timing); - // Depth 1 at 25 fps, plus one tick of reserve: every frame decodes one period after the - // last, the keyframe a whole reserve before it is presented. + // Depth 1 at 25 fps: every frame decodes one period after the last, in the slot of the + // next presentation time a period early. let (pts, dts) = timing[1]; assert_eq!( pts, @@ -5463,8 +5524,8 @@ async fn declared_reorder_sizes_the_decode_clock_from_the_first_frame() { ); assert_eq!( dts, - Some(REORDERED_BASE * FRAME_TICKS - 1), - "the first P-frame already runs on the declared reserve" + Some(REORDERED_BASE * FRAME_TICKS), + "the first P-frame already runs on the declared depth" ); let decode: Vec = timing.iter().map(|&(pts, dts)| dts.unwrap_or(pts)).collect(); for (i, step) in decode.windows(2).map(|w| w[1] - w[0]).enumerate() { @@ -5502,11 +5563,11 @@ async fn reordered_video_loses_nothing_on_a_clean_path() { "every video frame went out" ); // The SPS declares no reorder depth, but the first pyramid is read a delay ahead of - // the first PCR, so the reserve it grows never steps the clock back. + // the first PCR, so the reorder delay it grows never steps the clock back. assert_eq!( count_discontinuity(&out), 0, - "the reserve settled before the clock started" + "the reorder delay settled before the clock started" ); let decode = pes_decode_in_order(&out); assert!( @@ -5515,14 +5576,14 @@ async fn reordered_video_loses_nothing_on_a_clean_path() { ); } -/// With nothing declared, the reserve grows to the deepest reordering muxed so far. This is the -/// one path where it depends on when an exporter joined: a joiner that has muxed only shallow -/// groups runs a shallower clock than a runner that saw a deep one before the join, until the -/// deep structure recurs. From then on the two render the same bytes. +/// With nothing declared, the reorder delay grows to the deepest reordering muxed so far. This +/// is the one path where it depends on when an exporter joined: a joiner that has muxed only +/// shallow groups runs a shallower clock than a runner that saw a deep one before the join, +/// until the deep structure recurs. From then on the two render the same bytes. #[tokio::test(start_paused = true)] async fn undeclared_reorder_converges_after_its_deepest_reorder() { // The runner sees a pyramid in group 2; the joiner, arriving at group 5, first in group 7. - // Group 9 repeats it on the settled reserve. + // Group 9 repeats it on the settled delay. let (runner, joiner) = export_reordered(SPS, PPS, &[2, 7, 9], None).await; let keyframes: Vec = joiner.iter().filter(|f| f.keyframe).map(|f| f.timestamp).collect(); let deepest = reordered_at(8 * GOP); @@ -5545,27 +5606,25 @@ async fn undeclared_reorder_converges_after_its_deepest_reorder() { } } -/// A `jitter` first published after the program tables sizes the decode clock from then on, -/// so the running exporter renders what a joiner that found it in its first catalog renders. +/// A `jitter` bounds how far ahead the decode clock looks, not how early it decodes: published +/// after the program tables, it leaves the encoder's spacing alone, and the running exporter +/// renders what a joiner that found it in its first catalog renders. #[tokio::test(start_paused = true)] -async fn jitter_published_after_the_tables_raises_the_reserve() { +async fn jitter_published_after_the_tables_keeps_the_encoders_clock() { let jitter = Duration::from_millis(80); let (runner, joiner) = export_reordered(SPS, PPS, &[], Some((2 * GOP, jitter))).await; let keyframes: Vec = joiner.iter().filter(|f| f.keyframe).map(|f| f.timestamp).collect(); assert_only_continuity_differs(&runner, &joiner, keyframes[1]); - // Every keyframe after the publish decodes a full `jitter` ahead of its presentation. - let ticks = 80 * 90; - let keyframe_timing: Vec<(u64, Option)> = video_timing(&runner, reordered_at(3 * GOP)..) - .into_iter() - .filter(|(pts, _)| (pts / FRAME_TICKS).is_multiple_of(GOP)) - .collect(); - assert!( - keyframe_timing.len() > 3, - "too few keyframes to judge: {keyframe_timing:?}" - ); - for (pts, dts) in keyframe_timing { - assert_eq!(dts, Some(pts - ticks), "keyframe at {pts} ignores the published jitter"); + // One B-frame per reference: every frame decodes a period after the last, a period early. + let timing = video_timing(&runner, reordered_at(3 * GOP)..); + assert_decodes_before_presenting(&timing); + let decode: Vec = timing.iter().map(|&(pts, dts)| dts.unwrap_or(pts)).collect(); + for (i, step) in decode.windows(2).map(|w| w[1] - w[0]).enumerate() { + assert_eq!(step, FRAME_TICKS, "frame {i} does not decode one period after the last"); + } + for (pts, dts) in timing.into_iter().filter(|(pts, _)| (pts / FRAME_TICKS).is_multiple_of(GOP)) { + assert_eq!(dts, Some(pts - FRAME_TICKS), "a keyframe decodes a period early"); } assert_decodes_after_the_clock(&runner); } From 9e08202bda823ec9a21de3dcc81a321b02ee7bcb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 11:17:55 -0700 Subject: [PATCH 17/40] fix(moq-mux): admit TS packets per PID against the receiver's buffers The schedule sent each unit as late as the rate allowed and saw one window ahead, so the slots it padded before a heavy passage were lost and a stretch outrunning the rate for about a window failed the export. Each slot now takes packets earliest deadline first across PIDs, each unit as soon as its PID's T-STD buffers admit it: no more packets per slot than its transport buffer drains, no more bytes than its decoder buffer holds until the units in it decode. Video buffers come from the SPS HRD or the level, audio from 13818-1 per codec. The floor and push-order logic are gone, a rate that turns up mid-stream takes over from the next slot, and slots are laid on the jitter buffer's clock so the output keeps its pace between frames. Each PCR is now the time of its own byte position at the multiplex rate, so it no longer strays up to a packet from where its bytes sit. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 25 +- rs/moq-mux/src/codec/h264/mod.rs | 40 +- rs/moq-mux/src/codec/h265/mod.rs | 18 + rs/moq-mux/src/codec/video.rs | 11 + rs/moq-mux/src/container/ts/export.rs | 412 +++++++++------ rs/moq-mux/src/container/ts/export_test.rs | 314 ++++++++--- rs/moq-mux/src/container/ts/schedule.rs | 580 +++++++++++++++------ rs/moq-mux/src/jitter.rs | 6 + 8 files changed, 1010 insertions(+), 396 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index cbd2c4b674a..0a944ec20cf 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -110,15 +110,18 @@ A constant-rate MPEG-TS source records its multiplex rate in the catalog (`mpegts.muxRate`, measured off the PCR clock, null stuffing included), and `export ts` pads its output with null packets back to that rate on a constant-rate schedule, so an IRD or groomer recovering its clock from packet arrival can lock: -every PCR interval carries the bytes the rate implies. `--mux-rate 5000000` pads to -an explicit rate instead, including for a broadcast that recorded none. A frame -goes out as late as the rate lets it while still arriving by its decode time, and a -keyframe bigger than one 25 ms interval spreads over the intervals before it, back -as far as `--delay`. That window adds up to the delay again to the output's -latency, so a padded export trails the source by up to twice the delay. A burst -that does not fit the window, or a source that sustains more than the rate, fails -the export with an error naming both knobs. A VBR source records no rate, so -export without one stays unpadded and adds no window. +every PCR is the time of its own byte position at the rate. `--mux-rate 5000000` +pads to an explicit rate instead, including for a broadcast that recorded none. + +Each frame goes out as early as a receiver's buffers for its PID admit (the +ISO 13818-1 T-STD: no more of a PID's packets per interval than its transport +buffer passes on, and no more bytes than its decoder buffer holds, sized from the +SPS's HRD or level for video and per codec for audio), up to `--delay` ahead of its +decode time, earliest decode time first. So a heavy passage rides the intervals +before it, and the output trails the source by twice the delay. A frame that cannot +arrive by its decode time at the rate fails the export with an error naming both +knobs; a broadcast-sized decoder buffer (a CPB of a second or more) needs a delay to +match. A VBR source records no rate, so export without one stays unpadded. fMP4 export writes one fragment per publisher group on each track. Audio follows the publisher's cuts; video normally follows GOPs. Closing a group flushes it @@ -359,9 +362,9 @@ groups fetchable, which the [HLS gateway](/bin/hls) depends on. `export --max-ag stalled group before skipping. Raising the first never delays playback. `export ts` takes `--delay` (default 500 ms) instead, and works like an SRT -receiver's latency. Every frame is written that long after its decode time, +receiver's latency. Every frame is muxed that long after its decode time, with the clock started when the first frame arrived, so the output keeps the -source's pace and interleaves all tracks in decode order whatever their arrival +source's pace and muxes all tracks in decode order whatever their arrival skew: two exporters of one broadcast emit them in one order. A frame that arrives later than its deadline is dropped, and video then resumes at its next keyframe. The delay is also how stale a group may get before it is skipped. diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index bddbe199633..800ff31d414 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -156,6 +156,11 @@ pub(crate) fn sps_reorder(nal: &[u8]) -> Option { }) } +/// The NAL HRD an SPS NAL unit's VUI declares, when it carries one. +pub(crate) fn sps_hrd(nal: &[u8]) -> Option { + sps_vui(nal)?.hrd +} + /// The frame rate an SPS NAL unit fixes with `fixed_frame_rate_flag`. /// /// Without the flag the VUI tick only bounds the rate from above, so it is left to measurement. @@ -178,6 +183,8 @@ struct Vui { period: Option<(u64, u64)>, /// `max_num_reorder_frames`, when the bitstream restriction is present. depth: Option, + /// The NAL HRD, when present. + hrd: Option, } /// `None` when the SPS carries no VUI or fails to parse. @@ -293,12 +300,13 @@ fn vui(rbsp: &[u8]) -> h264_parser::Result> { } } let nal_hrd = r.read_flag()?; - if nal_hrd { - skip_hrd(&mut r)?; - } + let hrd = match nal_hrd { + true => Some(read_hrd(&mut r)?), + false => None, + }; let vcl_hrd = r.read_flag()?; if vcl_hrd { - skip_hrd(&mut r)?; + read_hrd(&mut r)?; } if nal_hrd || vcl_hrd { r.skip_bits(1)?; // low_delay_hrd_flag @@ -309,6 +317,7 @@ fn vui(rbsp: &[u8]) -> h264_parser::Result> { frame_only, period, depth: None, + hrd, })); } r.skip_bits(1)?; // motion_vectors_over_pic_boundaries_flag @@ -320,24 +329,35 @@ fn vui(rbsp: &[u8]) -> h264_parser::Result> { frame_only, period, depth: Some(depth), + hrd, })) } -/// Skip `hrd_parameters()` (ITU-T H.264 E.1.2). -fn skip_hrd(r: &mut h264_parser::bitreader::BitReader) -> h264_parser::Result<()> { +/// Read `hrd_parameters()` (ITU-T H.264 E.1.2) down to its last schedule (E.2.2). +fn read_hrd(r: &mut h264_parser::bitreader::BitReader) -> h264_parser::Result { use h264_parser::eg::read_ue; let cpb_cnt_minus1 = read_ue(r)?; if cpb_cnt_minus1 > 31 { return Err(h264_parser::Error::MalformedSps("cpb_cnt_minus1 out of range".into())); } - r.skip_bits(8)?; // bit_rate_scale, cpb_size_scale + let bit_rate_scale = r.read_bits(4)?; + let cpb_size_scale = r.read_bits(4)?; + let mut hrd = crate::codec::video::Hrd { + bit_rate: 0, + cpb_size: 0, + }; for _ in 0..=cpb_cnt_minus1 { - read_ue(r)?; // bit_rate_value_minus1 - read_ue(r)?; // cpb_size_value_minus1 + let bit_rate = u64::from(read_ue(r)?) + 1; + let cpb_size = u64::from(read_ue(r)?) + 1; r.skip_bits(1)?; // cbr_flag + hrd = crate::codec::video::Hrd { + bit_rate: bit_rate << (6 + bit_rate_scale), + cpb_size: cpb_size << (4 + cpb_size_scale), + }; } - r.skip_bits(20) // four delay/offset lengths + r.skip_bits(20)?; // four delay/offset lengths + Ok(hrd) } /// Parsed AVCDecoderConfigurationRecord (ISO/IEC 14496-15 §5.3.3.1.2). diff --git a/rs/moq-mux/src/codec/h265/mod.rs b/rs/moq-mux/src/codec/h265/mod.rs index d0f420e2745..fe34194cf3b 100644 --- a/rs/moq-mux/src/codec/h265/mod.rs +++ b/rs/moq-mux/src/codec/h265/mod.rs @@ -231,6 +231,24 @@ pub(crate) fn sps_reorder(nal: &[u8]) -> Option { }) } +/// The HRD an SPS NAL unit's VUI declares at its highest sub-layer, when it carries one: the +/// NAL HRD's last schedule, or the VCL HRD's when that is all there is. +pub(crate) fn sps_hrd(nal: &[u8]) -> Option { + let sps = SpsNALUnit::parse(&mut &nal[..]).ok()?; + let highest = sps.rbsp.sps_max_sub_layers_minus1 as usize; + let hrd = sps.rbsp.vui_parameters.as_ref()?.vui_timing_info.as_ref()?.hrd_parameters.as_ref()?; + let layer = hrd.sub_layers.get(highest).or(hrd.sub_layers.last())?; + // The NAL HRD's schedules come first when both are present. + let schedules = usize::try_from(layer.cpb_cnt_minus1).ok()? + 1; + let last = layer.sub_layer_parameters.get(..schedules)?.last()?; + let bit_rate_scale = u32::from(hrd.common_inf.bit_rate_scale?); + let cpb_size_scale = u32::from(hrd.common_inf.cpb_size_scale?); + Some(crate::codec::video::Hrd { + bit_rate: (u64::from(last.bit_rate_value_minus1) + 1) << (6 + bit_rate_scale), + cpb_size: (u64::from(last.cpb_size_value_minus1) + 1) << (4 + cpb_size_scale), + }) +} + /// One picture's duration at the highest sub-layer as `(units, scale)`, when the VUI's HRD /// parameters fix the picture rate within the CVS (ITU-T H.265 E.3.2). Without that the VUI tick /// only bounds the rate from above. diff --git a/rs/moq-mux/src/codec/video.rs b/rs/moq-mux/src/codec/video.rs index 219da9d09d5..3553188621b 100644 --- a/rs/moq-mux/src/codec/video.rs +++ b/rs/moq-mux/src/codec/video.rs @@ -27,6 +27,17 @@ pub(crate) struct Reorder { pub period: Option<(u64, u64)>, } +/// The hypothetical reference decoder a video sequence header declares: its NAL HRD's last +/// schedule, the one the transport stream's decoder buffers are sized from (ISO 13818-1 +/// 2.14.3.1, 2.17.2). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Hrd { + /// The CPB's input rate in bits per second. + pub bit_rate: u64, + /// The CPB's size in bits. + pub cpb_size: u64, +} + /// The catalog-publishing state a video importer overlays onto every config it resolves. /// /// Holds the caller's hint, the last config published (to dedupe re-publishes), and the diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 07cd32c9fbd..1793320c5b4 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -40,14 +40,14 @@ use moq_net::Timestamp; use crate::catalog::hang::Catalog; use crate::catalog::{CatalogFormat, Stream}; -use crate::codec::video::Reorder; +use crate::codec::video::{Hrd, Reorder}; use crate::codec::{aac, annexb}; use crate::container::{ExportSource, Frame}; use crate::jitter::{self, Arrival, Push}; use super::adts; use super::catalog; -use super::schedule::{self, Schedule}; +use super::schedule::{self, Buffer, Schedule}; /// PID of the single program's PMT. const PMT_PID: u16 = 0x1000; @@ -141,6 +141,8 @@ pub struct Export { pcr_discontinuity: bool, /// Lays the muxed packets onto the PCR grid. schedule: Schedule, + /// Wakes the export when the next grid slot is due on the jitter buffer's clock. + slot_timer: Option>>, /// Output frames ready to hand out, one per grid slot. queue: VecDeque, /// The rate to pad the output to with null packets, in bits per second: the @@ -201,9 +203,8 @@ struct Track { clock: DecodeClock<(Pending, Option)>, /// What the catalog and the SPS declare about the video's reordering. timing: Timing, - /// How long a receiver takes to pass one of its packets on ([`drain`]), so its last - /// packet has to arrive that long before it decodes. - drain: Duration, + /// The receiver's buffers for an audio or verbatim track; video's come from [`Timing`]. + buffer: Option, } impl Track { @@ -220,7 +221,7 @@ impl Track { descriptors: Vec::new(), clock: DecodeClock::default(), timing: Timing::default(), - drain: Duration::ZERO, + buffer: None, } } @@ -247,6 +248,24 @@ impl Track { self.release(name, jitter, false); } + /// The receiver's buffers for the track's PID, if the T-STD gives it any. + fn buffer(&self) -> Option { + match self.kind { + Kind::Video(_) => self.timing.buffer(), + _ => self.buffer, + } + } + + /// How long a receiver takes to pass one of the track's packets on, so its last packet has + /// to arrive that long before it decodes. A packet's bytes reach a decoder buffer only once + /// they have drained through the T-STD buffers ahead of it (ISO 13818-1 2.4.2); without it + /// a unit whose DTS sits on a PCR slot boundary, as a source's own 25 fps timestamps put + /// every fifth frame, arrives complete only after it decodes. + fn drain(&self) -> Duration { + let rate = self.buffer().map_or(Buffer::SYSTEM.rate, |buffer| buffer.rate); + Duration::from_nanos(TsPacket::SIZE as u64 * 8 * 1_000_000_000 / rate.max(1)) + } + /// Queue every held video frame whose DTS is settled, or all of them when `flush`. fn release(&mut self, name: &str, jitter: &mut jitter::Buffer, flush: bool) { let (delay, lookahead) = self.timing.reorder(); @@ -299,17 +318,22 @@ struct Timing { jitter: u64, /// The catalog `framerate`: the picture period for an SPS that declares no fixed rate. framerate: Option, - /// The avcC/hvcC [`Self::declared`] was read from. + /// The avcC/hvcC [`Self::declared`] and [`Self::hrd`] were read from. description: Option, declared: Option, + /// The stream's level limits, from the catalog codec. + level: Option, + /// The HRD its SPS declares. + hrd: Option, } impl Timing { /// Take the rendition's timing from a catalog snapshot, before or after the program /// tables are written. - fn configure(&mut self, config: &VideoConfig) { + fn configure(&mut self, config: &VideoConfig, name: &str) { self.jitter = config.jitter.map_or(0, |t| (t.as_micros() * 90_000 / 1_000_000) as u64); self.framerate = config.framerate.filter(|fps| fps.is_finite() && *fps > 0.0); + self.level = Some(video_level(config, name)); } /// Read the reorder depth from the codec config a keyframe is carried with. @@ -319,9 +343,25 @@ impl Timing { } self.description = description.cloned(); self.declared = description.and_then(|d| declared_reorder(stream_type, d)); + self.hrd = description.and_then(|d| declared_hrd(stream_type, d)); tracing::debug!(track = %name, reorder = ?self.reorder(), "video reordering declared"); } + /// The receiver's buffers (H.222.0 2.14.3.1, 2.17.2): EB is the HRD's CPB, else the + /// level's, and packets go no faster than the slower of the transport buffer's Rx and the + /// leak from MB into EB, so neither MB nor TB fills. + fn buffer(&self) -> Option { + let level = self.level?; + let (rate, cpb) = match self.hrd { + Some(hrd) => (hrd.bit_rate * level.factor / 1_000, hrd.cpb_size), + None => (level.leak, level.cpb), + }; + Some(Buffer { + rate: rate.min(level.leak), + size: Some(usize::try_from(cpb / 8).unwrap_or(usize::MAX)), + }) + } + /// The least reorder delay and lookahead the SPS declares, in ticks, so two exporters agree /// on them from their first keyframe whatever reordering each has seen. /// @@ -639,6 +679,7 @@ impl Export { emitted_epoch: 0, pcr_discontinuity: false, schedule: Schedule::new(Duration::ZERO), + slot_timer: None, queue: VecDeque::new(), video_start: None, mux_rate: None, @@ -665,16 +706,16 @@ impl Export { /// Mux each frame this long after its decode time, like an SRT receiver's TSBPD. /// /// The clock starts at the first frame's arrival, so the output keeps the source's - /// pace and interleaves every track in `(DTS, PID)` order whatever the arrival skew - /// between them. A frame that arrives after its deadline is dropped, and a video - /// track that dropped one resumes at its next keyframe. The delay is also each - /// source's staleness budget (see [`Consumer`](crate::container::Consumer)). + /// pace and muxes every track in `(DTS, PID)` order whatever the arrival skew between + /// them. A frame that arrives after its deadline is dropped, and a video track that + /// dropped one resumes at its next keyframe. The delay is also each source's staleness + /// budget (see [`Consumer`](crate::container::Consumer)). /// - /// With a multiplex rate it is also how far ahead of its decode time a frame may go - /// out: a keyframe bigger than one PCR interval at the rate spreads over the intervals - /// before it, back as far as the delay, so the output trails the source by up to twice - /// the delay. A frame that does not fit there fails the export. Defaults to - /// [`Duration::ZERO`], which pads no further ahead than one interval. + /// It is also how far ahead of its decode time a frame may go out, as early as the + /// receiver's buffers for its PID admit, so a heavy passage rides the slots before it + /// and the output trails the source by twice the delay. With a multiplex rate, a frame + /// that cannot arrive by its decode time fails the export. Defaults to + /// [`Duration::ZERO`], which holds nothing and sends each frame by its decode time. pub fn with_delay(mut self, delay: Duration) -> Self { self.delay = delay; self.jitter = jitter::Buffer::new(delay); @@ -781,9 +822,7 @@ impl Export { } // 4. Mux each frame the jitter buffer lets go (the first carries the buffered - // PAT/PMT) and lay out every grid slot it settles. A slot is settled once every - // frame that could ride it has been muxed: the jitter buffer hands frames over in - // decode order, so a frame settles every slot whose window ends before its own. + // PAT/PMT), then lay out every grid slot whose time has come ([`Self::lay_due`]). loop { if let Some(out) = self.queue.pop_front() { self.emitted_epoch = self.epoch; @@ -816,10 +855,18 @@ impl Export { self.last_timestamp = Some(ready.item.frame.timestamp); let decode = ready.item.decode.as_nanos(); self.mux(&name, ready.item)?; - // A later frame decodes no earlier, so it can be due no earlier than this, - // less the longest drain of any track. - let drain = self.tracks.values().map(|track| track.drain).max().unwrap_or_default(); - self.lay(Some(schedule::slot(decode.saturating_sub(drain.as_nanos()))))?; + if self.delay.is_zero() { + // Without a clock to lay slots on, a frame settles every slot before its own: a + // later frame decodes no earlier, so it can be due no earlier than this, less the + // longest drain of any track. + let drain = self.max_drain(); + self.lay(Some(schedule::slot(decode.saturating_sub(drain.as_nanos()))))?; + } + } + self.lay_due(waiter)?; + if let Some(out) = self.queue.pop_front() { + self.emitted_epoch = self.epoch; + return Poll::Ready(Ok(Some(out))); } // 5. Once every track has drained, nothing more can ride the slots still open, so @@ -1050,7 +1097,7 @@ impl Export { // after the tables still sizes the decode clock. for (name, config) in catalog.video.renditions.iter() { if let Some(track) = self.tracks.get_mut(name) { - track.timing.configure(config); + track.timing.configure(config, name); } } return Ok(()); @@ -1097,8 +1144,7 @@ impl Export { }, }; track.kind = kind; - track.drain = drain(video_rate(config, name)); - track.timing.configure(config); + track.timing.configure(config, name); self.refresh(name, track, pids[name], track_descriptors(&mpegts, name)); } for (name, config) in catalog.audio.renditions.iter() { @@ -1110,8 +1156,8 @@ impl Export { None => continue, }, }; + track.buffer = Some(audio_buffer(config, &kind)); track.kind = kind; - track.drain = drain(audio_rate(config)); self.refresh(name, track, pids[name], track_descriptors(&mpegts, name)); } for (name, entry) in mpegts.tracks.iter() { @@ -1127,6 +1173,7 @@ impl Export { Some(track) => track, None => Track::new(ExportSource::for_stream(&self.source, name, self.delay)?, kind.clone()), }; + track.buffer = verbatim_buffer(&kind, &entry.descriptors); track.kind = kind; self.refresh(name, track, pids[name], entry.descriptors.clone()); } @@ -1175,10 +1222,11 @@ impl Export { self.jitter.dropped() } - /// When the next queued frame is due. + /// When the next queued frame, or grid slot while media is queued, is due. #[cfg(test)] pub(super) fn next_due(&self) -> Option { - self.jitter.next_deadline() + let slot = self.slot_due().filter(|_| self.schedule.queued()).map(|(_, at)| at); + self.jitter.next_deadline().into_iter().chain(slot).min() } /// The discontinuity counter of the most recently returned output frame. @@ -1411,6 +1459,8 @@ impl Export { es_info, }; + self.schedule.set_buffer(0, Buffer::SYSTEM); + self.schedule.set_buffer(pmt_pid, Buffer::SYSTEM); self.psi = Some(Psi { pat, pmt, @@ -1519,6 +1569,7 @@ impl Export { }) .collect(); for (pid, sections) in pending { + self.schedule.set_buffer(pid, Buffer::SYSTEM); for section in §ions { self.write_section(&mut out, pid, section)?; } @@ -1549,9 +1600,51 @@ impl Export { self.write_pes(&mut out, &unit, &es_payload)?; } } - let drain = self.tracks.get(name).map_or(Duration::ZERO, |track| track.drain); - self.schedule - .push(decode.as_nanos().saturating_sub(drain.as_nanos()), out, keyframe); + let track = self.tracks.get(name).context("missing track")?; + if let Some(buffer) = track.buffer() { + self.schedule.set_buffer(pid, buffer); + } + let by = decode.as_nanos().saturating_sub(track.drain().as_nanos()); + self.schedule.push(pid, by, out, keyframe); + Ok(()) + } + + /// The longest any track's receiver takes to pass one of its packets on. + fn max_drain(&self) -> Duration { + self.tracks.values().map(Track::drain).max().unwrap_or_default() + } + + /// When the next grid slot is due on the jitter buffer's clock: once every frame that could + /// ride it has gone out of the jitter buffer, so its contents depend on the frames alone, + /// never on when they arrived, and the output keeps the clock's pace while media is queued + /// or still to decode. `None` without a delay, where frames settle the slots instead. + fn slot_due(&self) -> Option<(u128, web_async::time::Instant)> { + if self.delay.is_zero() || self.psi.is_none() || self.schedule.is_empty() { + return None; + } + let (_, known) = self.schedule.upcoming()?; + let settled = known * PCR_INTERVAL.as_nanos() + self.max_drain().as_nanos(); + let settled = Timestamp::from_micros(u64::try_from(settled / 1_000).ok()?).ok()?; + Some((known, self.jitter.at(settled)?)) + } + + /// Lay out every grid slot whose time has come, and wake when the next one is due. + fn lay_due(&mut self, waiter: &kio::Waiter) -> anyhow::Result<()> { + while let Some((known, at)) = self.slot_due() { + if web_async::time::Instant::now() < at { + let timer = self + .slot_timer + .get_or_insert_with(|| Box::pin(web_async::time::sleep_until(at))); + if timer.deadline() != at { + timer.as_mut().reset(at); + } + if waiter.poll_future(timer.as_mut()).is_pending() { + return Ok(()); + } + } + self.lay(Some(known))?; + } + self.slot_timer = None; Ok(()) } @@ -1559,24 +1652,27 @@ impl Export { /// the first slot a frame still to be muxed could be due in, or `None` when nothing /// more is coming. /// - /// Each frame opens with its slot's clock packet, so the packet count between - /// consecutive PCRs is what the interval between their values carries, and a consumer - /// holding only the byte stream (which is every MPEG-TS tool) recovers the same clock - /// from byte position that the values assert. Each frame is stamped at its slot - /// boundary, so a pacing caller releases the clock at the instant it asserts. + /// Each frame opens with its slot's clock packet. At a multiplex rate its value is the + /// time of its own byte position at the rate, so a consumer holding only the byte stream + /// (which is every MPEG-TS tool) recovers the same clock the values assert. Each frame is + /// stamped at its slot boundary, so a pacing caller releases the clock at the instant it + /// asserts. /// /// The PES units cannot carry the clock themselves: a PCR sampled from the DTS would /// step with the frame cadence, and no downstream CBR stage can repair that, because a - /// groomer can only place the clock samples it receives. So the PCR asserts its own - /// uniform ramp instead: absolute grid slots on the media timeline, shared by every - /// exporter of the broadcast, like [`due`]. + /// groomer can only place the clock samples it receives. So the PCR follows its own + /// grid instead: slots on the media timeline, shared by every exporter of the + /// broadcast, like [`due`]. fn lay(&mut self, known: Option) -> anyhow::Result<()> { let Some(psi) = self.psi.as_ref() else { return Ok(()); }; let (pcr_pid, pmt_pid) = (psi.pcr_pid, psi.pmt_pid); while let Some(slot) = self.schedule.next(known)? { - let mut payload = self.pcr_at(slot.index, pcr_pid)?; + let mut payload = pcr_packet(pcr_pid, slot.pcr)?; + if std::mem::take(&mut self.pcr_discontinuity) { + payload[5] |= 0x80; + } payload.extend_from_slice(&slot.layout(pmt_pid, &NULL_PACKET)); self.number(&mut payload); self.queue.push_back(Frame { @@ -1607,23 +1703,6 @@ impl Export { } } - /// The clock packet for grid slot `index`. - /// - /// The value runs one slot behind the boundary, so the bytes of slot `index` are timed - /// up to its boundary: a unit due in that slot decodes no earlier ([`schedule::slot`]). - /// Back off through the 33-bit wrap rather than saturating: a timeline that starts in - /// its first slot would otherwise clamp to zero and break the uniform step. The wire - /// field is a circular clock, so the masked wrapped value is the correct mod-2^33 - /// back-off. - fn pcr_at(&mut self, index: u128, pcr_pid: u16) -> anyhow::Result> { - let ticks = slot_ticks(index, PCR_INTERVAL).wrapping_sub(slot_ticks(1, PCR_INTERVAL)); - let mut packet = pcr_packet(pcr_pid, ticks)?; - if std::mem::take(&mut self.pcr_discontinuity) { - packet[5] |= 0x80; - } - Ok(packet) - } - /// Packetize a PES payload into 188-byte TS packets. fn write_pes(&mut self, out: &mut Vec, unit: &PesUnit, payload: &[u8]) -> anyhow::Result<()> { let pts = to_ts_timestamp(unit.timestamp)?; @@ -1784,9 +1863,9 @@ impl Export { /// /// There is no payload, so the field's stuffing fills the packet; its continuity counter /// is numbered when it goes out ([`Export::number`]). -fn pcr_packet(pid: u16, ticks: u64) -> anyhow::Result> { +fn pcr_packet(pid: u16, pcr: u64) -> anyhow::Result> { anyhow::ensure!(pid <= 0x1FFF, "PID out of range: {pid}"); - let base = ticks & TS_TIMESTAMP_MASK; + let (base, extension) = ((pcr / 300) & TS_TIMESTAMP_MASK, pcr % 300); let mut p = Vec::with_capacity(TsPacket::SIZE); p.push(0x47); p.push((pid >> 8) as u8); @@ -1797,14 +1876,13 @@ fn pcr_packet(pid: u16, ticks: u64) -> anyhow::Result> { p.push(183); // PCR_flag alone. p.push(0x10); - // program_clock_reference_base (33 bits), 6 reserved '1' bits, and a zero - // 9-bit extension (the grid is 90 kHz-exact, so there is nothing sub-tick). + // program_clock_reference_base (33 bits), 6 reserved '1' bits, and the 9-bit extension. p.push((base >> 25) as u8); p.push((base >> 17) as u8); p.push((base >> 9) as u8); p.push((base >> 1) as u8); - p.push(((base as u8) << 7) | 0x7e); - p.push(0x00); + p.push(((base as u8) << 7) | 0x7e | (extension >> 8) as u8); + p.push(extension as u8); p.resize(TsPacket::SIZE, 0xff); Ok(p) } @@ -1885,11 +1963,6 @@ fn stamp(nanos: u128) -> anyhow::Result { Ok(Timestamp::from_micros(micros)?) } -/// A repetition slot's boundary (`index * interval`) in 90 kHz ticks. -fn slot_ticks(index: u128, interval: Duration) -> u64 { - (index * interval.as_nanos() * 90_000 / 1_000_000_000) as u64 -} - /// External byte size of an adaptation field (manual mirror of the crate's /// private `external_size`); only PCR is ever set. fn adaptation_size(af: &AdaptationField) -> usize { @@ -1923,92 +1996,130 @@ fn video_kind(config: &VideoConfig, name: &str) -> anyhow::Result { } } -/// How long a receiver takes to pass one transport packet on at `rate` bits per second. -/// -/// A packet's bytes reach a decoder buffer only once it has drained through the T-STD -/// buffers ahead of it (ISO 13818-1 2.4.2), so a unit's last packet has to arrive at least -/// this long before its decode time. Without it a unit whose DTS sits on a PCR slot -/// boundary, as a source's own 25 fps timestamps put every fifth frame, arrives complete -/// only after it decodes. -fn drain(rate: u64) -> Duration { - Duration::from_nanos(TsPacket::SIZE as u64 * 8 * 1_000_000_000 / rate.max(1)) +/// A video level's T-STD limits (H.222.0 2.14.3.1, 2.17.2), in bits per second and bits. +#[derive(Clone, Copy, Debug)] +struct Level { + /// The leak from MB into EB, Rbx: CpbBrNalFactor * MaxBR. + leak: u64, + /// EB without an HRD: CpbBrNalFactor * MaxCPB. + cpb: u64, + /// Rx over an HRD's BitRate, in thousandths: 1.2 for H.264, CpbBrNalFactor / + /// CpbBrVclFactor for H.265. + factor: u64, } -/// The slowest rate on a video stream's T-STD path, in bits per second: the MB to EB leak, -/// Rbx = CpbBrNalFactor * MaxBR for its level (H.222.0 2.14.3.1, 2.17.2; H.264 Table A-1, -/// H.265 Table A.8). The transport buffer's Rx is never slower. A level the tables don't list -/// takes the lowest, which only sends its units earlier. -fn video_rate(config: &VideoConfig, name: &str) -> u64 { - let rate = match &config.codec { +/// A video rendition's level limits from its catalog codec (H.264 Table A-1, H.265 Table +/// A.8). A level the tables don't list takes the lowest, which only sends its units earlier. +fn video_level(config: &VideoConfig, name: &str) -> Level { + let level = match &config.codec { VideoCodec::H264(h264) => { // level_idc 11 with constraint_set3_flag is level 1b for Baseline, Main and Extended. let level_1b = h264.level == 11 && h264.constraints & 0x10 != 0 && matches!(h264.profile, 66 | 77 | 88); - let max_br = match h264.level { - _ if level_1b => Some(128), - 9 => Some(128), - 10 => Some(64), - 11 => Some(192), - 12 => Some(384), - 13 => Some(768), - 20 => Some(2_000), - 21 | 22 => Some(4_000), - 30 => Some(10_000), - 31 => Some(14_000), - 32 | 40 => Some(20_000), - 41 | 42 => Some(50_000), - 50 => Some(135_000), - 51 | 52 | 60 => Some(240_000), - 61 => Some(480_000), - 62 => Some(800_000), + let limits = match h264.level { + _ if level_1b => Some((128, 350)), + 9 => Some((128, 350)), + 10 => Some((64, 175)), + 11 => Some((192, 500)), + 12 => Some((384, 1_000)), + 13 => Some((768, 2_000)), + 20 => Some((2_000, 2_000)), + 21 | 22 => Some((4_000, 4_000)), + 30 => Some((10_000, 10_000)), + 31 => Some((14_000, 14_000)), + 32 => Some((20_000, 20_000)), + 40 => Some((20_000, 25_000)), + 41 | 42 => Some((50_000, 62_500)), + 50 => Some((135_000, 135_000)), + 51 | 52 | 60 => Some((240_000, 240_000)), + 61 => Some((480_000, 480_000)), + 62 => Some((800_000, 800_000)), _ => None, }; - max_br.map(|max_br| 1_200 * max_br) + limits.map(|(max_br, max_cpb)| Level { + leak: 1_200 * max_br, + cpb: 1_200 * max_cpb, + factor: 1_200, + }) } VideoCodec::H265(h265) => { - let max_br = match (h265.level_idc, h265.tier_flag) { - (30, false) => Some(128), - (60, false) => Some(1_500), - (63, false) => Some(3_000), - (90, false) => Some(6_000), - (93, false) => Some(10_000), - (120, false) => Some(12_000), - (120, true) => Some(30_000), - (123, false) => Some(20_000), - (123, true) => Some(50_000), - (150, false) => Some(25_000), - (150, true) => Some(100_000), - (153, false) => Some(40_000), - (153, true) => Some(160_000), - (156 | 180, false) => Some(60_000), - (156 | 180, true) => Some(240_000), - (183, false) => Some(120_000), - (183, true) => Some(480_000), - (186, false) => Some(240_000), - (186, true) => Some(800_000), + let limits = match (h265.level_idc, h265.tier_flag) { + (30, false) => Some((128, 350)), + (60, false) => Some((1_500, 1_500)), + (63, false) => Some((3_000, 3_000)), + (90, false) => Some((6_000, 6_000)), + (93, false) => Some((10_000, 10_000)), + (120, false) => Some((12_000, 12_000)), + (120, true) => Some((30_000, 30_000)), + (123, false) => Some((20_000, 20_000)), + (123, true) => Some((50_000, 50_000)), + (150, false) => Some((25_000, 25_000)), + (150, true) => Some((100_000, 100_000)), + (153, false) => Some((40_000, 40_000)), + (153, true) => Some((160_000, 160_000)), + (156 | 180, false) => Some((60_000, 60_000)), + (156 | 180, true) => Some((240_000, 240_000)), + (183, false) => Some((120_000, 120_000)), + (183, true) => Some((480_000, 480_000)), + (186, false) => Some((240_000, 240_000)), + (186, true) => Some((800_000, 800_000)), _ => None, }; - max_br.map(|max_br| 1_100 * max_br) + limits.map(|(max_br, max_cpb)| Level { + leak: 1_100 * max_br, + cpb: 1_100 * max_cpb, + factor: 1_100, + }) } _ => None, }; - rate.unwrap_or_else(|| { - tracing::warn!(track = %name, codec = %config.codec, "no T-STD rate for this level; sending its units early"); - 1_200 * 64 + level.unwrap_or_else(|| { + tracing::warn!(track = %name, codec = %config.codec, "no T-STD limits for this level; sending its units early"); + Level { + leak: 1_200 * 64, + cpb: 1_200 * 175, + factor: 1_200, + } }) } -/// The rate an audio stream's transport buffer drains at, in bits per second (H.222.0 -/// 2.4.2.3): by channel count for ADTS AAC, 2 Mb/s for every other audio. -fn audio_rate(config: &AudioConfig) -> u64 { - if !matches!(config.codec, AudioCodec::AAC(_)) { - return 2_000_000; - } - match config.channel_count { - 0..=2 => 2_000_000, - 3..=8 => 5_529_600, - 9..=12 => 8_294_400, - _ => 33_177_600, - } +/// An audio stream's buffers in a receiver (H.222.0 2.4.2.3): by channel count for ADTS +/// AAC (and Opus, which borrows them), 2 Mb/s and the codec's main buffer for the rest. +fn audio_buffer(config: &AudioConfig, kind: &Kind) -> Buffer { + let by_channels = || match config.channel_count { + 0..=2 => (2_000_000, 3_584), + 3..=8 => (5_529_600, 8_976), + 9..=12 => (8_294_400, 12_804), + _ => (33_177_600, 51_216), + }; + let (rate, size) = match kind { + Kind::Aac(_) | Kind::Opus { .. } => by_channels(), + Kind::Mp2 { .. } => (2_000_000, 3_584), + // ATSC A/53 Part 5 5.7, and A/52 Annex G 3.6.1 for E-AC-3. + Kind::Ac3 => (2_000_000, 2_592), + Kind::Eac3 => (2_000_000, 12_896), + _ => (2_000_000, 3_584), + }; + Buffer { rate, size: Some(size) } +} + +/// A verbatim stream's buffers, for the kinds its descriptors identify: AC-3 and E-AC-3 as +/// DVB private data (ATSC A/52 Annex A 5.4). Any other stream has none the schedule knows. +fn verbatim_buffer(kind: &Kind, descriptors: &[catalog::Descriptor]) -> Option { + let Kind::Verbatim { + stream_type: 0x06, + framing: catalog::Framing::Pes, + .. + } = kind + else { + return None; + }; + descriptors + .iter() + .any(|descriptor| matches!(descriptor.tag, 0x6a | 0x7a)) + .then_some(Buffer { + rate: 2_000_000, + size: Some(5_696), + }) } /// Build the Annex-B elementary-stream payload for one video frame: rewrite the @@ -2248,6 +2359,15 @@ impl DecodeClock { } } +/// The HRD declared by the first SPS in a video rendition's avcC/hvcC. +fn declared_hrd(stream_type: StreamType, description: &[u8]) -> Option { + match stream_type { + StreamType::H264 => crate::codec::h264::sps_hrd(crate::codec::h264::Avcc::parse(description).ok()?.sps.first()?), + StreamType::H265 => crate::codec::h265::sps_hrd(crate::codec::h265::Hvcc::parse(description).ok()?.sps.first()?), + _ => None, + } +} + /// The reordering declared by the first SPS in a video rendition's avcC/hvcC. fn declared_reorder(stream_type: StreamType, description: &[u8]) -> Option { match stream_type { @@ -2267,7 +2387,7 @@ mod tests { use moq_net::Timestamp; - use super::{DecodeClock, PCR_INTERVAL, PSI_INTERVAL, due, is_complete_section, si_due, slot, slot_ticks}; + use super::{DecodeClock, PSI_INTERVAL, due, is_complete_section, si_due, slot}; fn ms(value: u64) -> Timestamp { Timestamp::from_millis(value).unwrap() @@ -2541,18 +2661,6 @@ mod tests { } } - #[test] - fn pcr_slots_are_exact_ticks() { - // 25 ms is exactly 2250 ticks at 90 kHz: consecutive slot boundaries differ by - // exactly one grid step with no rounding drift, however far the timeline runs. - // That is what makes the emitted PCR intervals uniform rather than merely bounded. - let step = slot_ticks(1, PCR_INTERVAL); - assert_eq!(step, 2250); - for index in [0u128, 1, 7, 1_000_000, u32::MAX as u128] { - assert_eq!(slot_ticks(index, PCR_INTERVAL), index as u64 * step); - } - } - #[test] fn section_validation() { // section_length 27 (0x1b) -> 30 bytes total. diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index d586950499b..9dfdf950a6d 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -953,8 +953,10 @@ async fn export_pcr_respects_every_rendition() { units += 1; let pts = pes.header.pts.expect("PES carried no PTS").as_u64(); let decode = pes.header.dts.map(|t| t.as_u64()).unwrap_or(pts); + // The clock starts a delay ahead of the timeline, so it may sit below zero. if let Some(pcr) = last_pcr { - assert!(decode >= pcr, "unit decodes at {decode}, before the clock at {pcr}"); + let ahead = decode.wrapping_sub(pcr) & ((1 << 33) - 1); + assert!(ahead < 1 << 32, "unit decodes at {decode}, before the clock at {pcr}"); } } } @@ -1027,7 +1029,8 @@ async fn export_pcr_backfills_a_coarse_cadence() { pcrs.len() ); for (i, w) in pcrs.windows(2).enumerate() { - assert_eq!(w[1] - w[0], 2250, "PCR interval off the grid at {i}: {w:?}"); + let step = w[1].wrapping_sub(w[0]) & ((1 << 33) - 1); + assert_eq!(step, 2250, "PCR interval off the grid at {i}: {w:?}"); } } @@ -2818,12 +2821,14 @@ async fn discontinuity_flags_the_break_once_across_tracks() { let mut video = Producer::new(video_track, HangContainer::Legacy(crate::container::Kind::Video)); let mut audio = Producer::new(audio_track, HangContainer::Legacy(crate::container::Kind::Audio)); - // One keyframe-led second per group on video, 100ms audio frames alongside it. + // One keyframe-led second per group on video, 100ms audio frames alongside it, a minute + // in so a frame can go out the whole recording delay ahead of it. + const MINUTE: u64 = 60_000_000; let write = |video: &mut Producer, audio: &mut Producer, start: u64, seconds: u64| { for sec in start..start + seconds { video .write(Frame { - timestamp: Timestamp::from_micros(sec * 1_000_000).unwrap(), + timestamp: Timestamp::from_micros(MINUTE + sec * 1_000_000).unwrap(), duration: None, payload: length_prefixed(&[idr.as_slice()]), keyframe: true, @@ -2833,7 +2838,7 @@ async fn discontinuity_flags_the_break_once_across_tracks() { for tenth in 0..10u64 { audio .write(Frame { - timestamp: Timestamp::from_micros(sec * 1_000_000 + tenth * 100_000).unwrap(), + timestamp: Timestamp::from_micros(MINUTE + sec * 1_000_000 + tenth * 100_000).unwrap(), duration: None, payload: Bytes::from_iter((0..180u16).map(|b| (b ^ tenth as u16) as u8)), keyframe: tenth == 0, @@ -2857,7 +2862,7 @@ async fn discontinuity_flags_the_break_once_across_tracks() { } video .write(Frame { - timestamp: Timestamp::from_micros(4_000_000).unwrap(), + timestamp: Timestamp::from_micros(MINUTE + 4_000_000).unwrap(), duration: None, payload: length_prefixed(&[idr.as_slice()]), keyframe: true, @@ -2866,7 +2871,7 @@ async fn discontinuity_flags_the_break_once_across_tracks() { video.cut(None).unwrap(); video .write(Frame { - timestamp: Timestamp::from_micros(4_100_000).unwrap(), + timestamp: Timestamp::from_micros(MINUTE + 4_100_000).unwrap(), duration: None, payload: length_prefixed(&[idr.as_slice()]), keyframe: true, @@ -2888,6 +2893,9 @@ async fn discontinuity_flags_the_break_once_across_tracks() { video.discontinuity().unwrap(); audio.discontinuity().unwrap(); write(&mut video, &mut audio, 5, 2); + // #3533: a content join on a continuous timeline, audio stepping only a sub-frame + // (already re-anchored forward), arriving live behind the break. + write(&mut video, &mut audio, 7, 2); let after = drain_frames(&mut export).await; assert_eq!( @@ -2933,17 +2941,16 @@ async fn discontinuity_flags_the_break_once_across_tracks() { } } assert!(video_frames > 0); - let video_pid = video_pid.expect("video PID in PMT"); + video_pid.expect("video PID in PMT"); - // #3533: a content join on a continuous timeline, audio stepping only a - // sub-frame (already re-anchored forward). Both tracks keep emitting; no fence. - write(&mut video, &mut audio, 7, 2); - let again = drain_frames(&mut export).await; - assert!(!again.is_empty(), "both tracks kept emitting across the join"); - assert!(count_pid(&again, video_pid) > 0, "video was not fenced"); - assert_eq!(count_discontinuity(&again), 0, "a forward join is not a PCR break"); + // Both tracks keep emitting across the join; no fence. + let joined = Timestamp::from_micros(MINUTE + 7_000_000).unwrap(); + assert!( + video_timing(&after, joined..).len() >= 2, + "video was not fenced across the join" + ); let mut counters = std::collections::HashMap::new(); - for frame in before.iter().chain(&marked).chain(&after).chain(&again) { + for frame in before.iter().chain(&marked).chain(&after) { for packet in frame.payload.as_chunks::<188>().0.iter() { let pid = u16::from(packet[1] & 0x1f) << 8 | u16::from(packet[2]); let cc = packet[3] & 15; @@ -3886,9 +3893,70 @@ fn in_decode_order(decode: &[u64]) -> bool { }) } +/// Every PES unit, each PID in its own decode order, finishes arriving before it decodes, on +/// the clock a receiver recovers from the PCRs. Returns how many units were judged. +fn assert_on_time(frames: &[Frame]) -> usize { + const WRAP: i128 = 1 << 33; + let ts: Vec = frames.iter().flat_map(|f| f.payload.iter().copied()).collect(); + // PCRs unwrapped from the first, which a start a window before the timeline may wrap. + let mut pcrs: Vec<(usize, f64)> = Vec::new(); + for (at, base, _) in collect_pcrs(frames) { + let base = base as i128; + let value = match pcrs.last() { + Some(&(_, last)) => last + ((base - last as i128).rem_euclid(WRAP)) as f64, + None => base as f64, + }; + pcrs.push((at, value)); + } + let first = pcrs.first().map_or(0.0, |&(_, value)| value); + let time = |at: usize| { + let k = pcrs.partition_point(|&(index, _)| index <= at).checked_sub(1)?; + let (&(a, ta), &(b, tb)) = (pcrs.get(k)?, pcrs.get(k + 1)?); + Some(ta + (tb - ta) * (at - a) as f64 / (b - a) as f64) + }; + // Each PES: its PID, DTS (else PTS) on the PCRs' unwrapped scale, and its last packet. + let mut units: Vec<(u16, f64, usize)> = Vec::new(); + for (at, packet) in ts.chunks(188).enumerate() { + let pid = (u16::from(packet[1] & 0x1f) << 8) | u16::from(packet[2]); + if pid == 0x1fff || packet[3] & 0x10 == 0 { + continue; + } + let start = 4 + if packet[3] & 0x20 != 0 { usize::from(packet[4]) + 1 } else { 0 }; + let pes = &packet[start.min(188)..]; + if packet[1] & 0x40 != 0 && pes.len() >= 14 && pes[..3] == [0, 0, 1] && pes[7] & 0x80 != 0 { + let stamp = |b: &[u8]| { + i128::from((b[0] >> 1) & 7) << 30 + | i128::from(b[1]) << 22 + | i128::from(b[2] >> 1) << 15 + | i128::from(b[3]) << 7 + | i128::from(b[4] >> 1) + }; + let decode = if pes[7] & 0x40 != 0 { stamp(&pes[14..19]) } else { stamp(&pes[9..14]) }; + let decode = first + ((decode - first as i128).rem_euclid(WRAP)) as f64; + units.push((pid, decode, at)); + } else if let Some(unit) = units.iter_mut().rev().find(|unit| unit.0 == pid) { + unit.2 = at; + } + } + let mut last: std::collections::HashMap = std::collections::HashMap::new(); + let mut judged = 0; + for &(pid, decode, end) in &units { + if let Some(previous) = last.insert(pid, decode) { + assert!(decode >= previous, "PID {pid} decodes at {decode} after a unit at {previous}"); + } + let Some(arrived) = time(end + 1) else { continue }; + assert!( + arrived <= decode, + "PID {pid}: the unit decoding at {decode} finishes arriving at {arrived:.1}" + ); + judged += 1; + } + judged +} + /// Audio lagging video by up to a frame, on a live clock: each video frame lands before /// the audio that precedes it. Returns the decode order the exporter emitted. -async fn lagging_audio(delay: Duration) -> Vec { +async fn lagging_audio(delay: Duration) -> Vec { let mut rig = Interleave::new(); let mut export = rig.export(delay).await; let mut out = Vec::new(); @@ -3901,23 +3969,19 @@ async fn lagging_audio(delay: Duration) -> Vec { rig.finish(); out.extend(drain_frames(&mut export).await); assert_eq!(export.dropped(), 0, "every frame arrived inside its deadline"); - pes_decode_in_order(&out) + out } #[tokio::test(start_paused = true)] -async fn late_audio_still_leads_the_interleave() { - let decode = lagging_audio(Duration::from_millis(500)).await; - assert!(decode.len() > 100, "too little output to judge: {}", decode.len()); - assert!( - in_decode_order(&decode), - "a frame went out ahead of an earlier one: {decode:?}" - ); +async fn late_audio_still_arrives_on_time() { + let frames = lagging_audio(Duration::from_millis(500)).await; + assert!(assert_on_time(&frames) > 100, "too little output to judge"); } /// A zero delay holds nothing, so the interleave stays in arrival order. #[tokio::test(start_paused = true)] async fn zero_delay_keeps_arrival_order() { - let decode = lagging_audio(Duration::ZERO).await; + let decode = pes_decode_in_order(&lagging_audio(Duration::ZERO).await); assert!( !in_decode_order(&decode), "video should have led the audio that arrived after it" @@ -3951,7 +4015,7 @@ async fn arrival_order_does_not_change_the_output() { } /// Audio that arrives past its deadline is dropped and counted, and everything else still -/// goes out in decode order. +/// goes out on time. #[tokio::test(start_paused = true)] async fn a_late_frame_is_dropped_and_the_rest_keep_their_order() { let delay = Duration::from_millis(100); @@ -3978,10 +4042,7 @@ async fn a_late_frame_is_dropped_and_the_rest_keep_their_order() { written - export.dropped(), "only the late frames were dropped" ); - assert!( - in_decode_order(&decode), - "a frame went out ahead of an earlier one: {decode:?}" - ); + assert_on_time(&out); } /// Audio dropped for leading the first keyframe does not stall the interleave. @@ -4094,7 +4155,8 @@ async fn repointed_si_entry_resubscribes() { let write_key = |producer: &mut Producer, sec: u64| { producer .write(Frame { - timestamp: Timestamp::from_micros(sec * 1_000_000).unwrap(), + // A minute in, so a frame can go out the whole recording delay ahead of it. + timestamp: Timestamp::from_micros((60 + sec) * 1_000_000).unwrap(), duration: None, payload: length_prefixed(&[&idr]), keyframe: true, @@ -4678,6 +4740,24 @@ async fn debounce_opens_without_a_media_clock() { /// Every PCR packet in transport order: its packet index in the stream, its value /// (90 kHz), and the media timestamp of the frame carrying it. +/// [`collect_pcrs`] up to the last packet that carries media: past it, the clock runs on alone +/// to the last decode time, the media having gone out up to a delay ahead of it. +fn media_pcrs(frames: &[Frame]) -> Vec<(usize, u64, u128)> { + let last = frames + .iter() + .flat_map(|frame| frame.payload.chunks(188)) + .enumerate() + .filter(|(_, packet)| packet[3] & 0x10 != 0 && (packet[1] & 0x1f, packet[2]) != (0x1f, 0xff)) + .map(|(at, _)| at) + .last() + .unwrap_or_default(); + let mut pcrs = collect_pcrs(frames); + // The clock packet after the last media closes its slot. + let end = pcrs.partition_point(|&(at, ..)| at < last) + 1; + pcrs.truncate(end); + pcrs +} + fn collect_pcrs(frames: &[Frame]) -> Vec<(usize, u64, u128)> { let mut at = 0; let mut pcrs = Vec::new(); @@ -4736,7 +4816,8 @@ async fn export_cbr_video() -> Vec { } video .write(Frame { - timestamp: Timestamp::from_micros(i * 40_000).unwrap(), + // A minute in, so a frame can go out the whole delay ahead of it. + timestamp: Timestamp::from_micros(60_000_000 + i * 40_000).unwrap(), duration: None, payload: if keyframe { annexb(&[SPS, PPS, &nal]) @@ -4752,6 +4833,7 @@ async fn export_cbr_video() -> Vec { let mut exporter = Export::new(crate::source::announced(&consumer)) .await .unwrap() + .with_mux_rate(20_000_000) .with_delay(RECORDING_MAX_AGE); drain_frames(&mut exporter).await } @@ -4875,14 +4957,16 @@ async fn a_frame_on_a_slot_boundary_finishes_before_it_decodes() { #[tokio::test(start_paused = true)] async fn pcr_rides_the_bytes_it_labels() { let frames = export_cbr_video().await; - let pcrs = collect_pcrs(&frames); + let pcrs = media_pcrs(&frames); assert!(pcrs.len() > 100, "expected the full feed, got {} PCRs", pcrs.len()); - // Every step is one grid slot, so every gap should carry the same span of media - // and therefore a comparable number of packets. + // Every step is one grid slot, to within the packet a PCR's byte position rounds it by + // (75 us at 20 Mb/s), so every gap should carry the same span of media and therefore a + // comparable number of packets. let step = PCR_INTERVAL.as_micros() as u64 * 90 / 1_000; for (i, w) in pcrs.windows(2).enumerate() { - assert_eq!(w[1].1.wrapping_sub(w[0].1) & ((1 << 33) - 1), step, "value step at {i}"); + let value = w[1].1.wrapping_sub(w[0].1) & ((1 << 33) - 1); + assert!(value.abs_diff(step) <= 7, "value step at {i}: {value}"); } let gaps: Vec = pcrs.windows(2).map(|w| w[1].0 - w[0].0).collect(); @@ -4925,9 +5009,7 @@ async fn pcr_stamps_step_by_the_grid() { /// The same positional property on a real reordered (B-frame) capture with a second /// rendition, where the exporter has no uniform cadence to lean on: frames arrive in /// decode order, and the two tracks advance the media clock at different rates. The -/// clock still lands among the bytes it labels rather than clustering, though the -/// gaps are not uniform: without a multiplex rate each frame spreads only over the -/// slots since the frame before it. +/// clock still lands among the bytes it labels rather than clustering. #[tokio::test(start_paused = true)] async fn pcr_stays_among_the_bytes_across_reordered_tracks() { let data = include_bytes!("test_data/scte35/kyrion_dirtystart.ts"); @@ -4941,9 +5023,10 @@ async fn pcr_stays_among_the_bytes_across_reordered_tracks() { let mut exporter = Export::new(crate::source::announced(&consumer)) .await .unwrap() + .with_mux_rate(20_000_000) .with_delay(RECORDING_MAX_AGE); let frames = drain_frames(&mut exporter).await; - let pcrs = collect_pcrs(&frames); + let pcrs = media_pcrs(&frames); assert!(pcrs.len() > 50, "expected the full feed, got {} PCRs", pcrs.len()); let gaps: Vec = pcrs.windows(2).map(|w| w[1].0 - w[0].0).collect(); @@ -4983,6 +5066,10 @@ async fn payload_less_clock_packets_repeat_the_counter() { let pid = u16::from(packet[1] & 0x1f) << 8 | u16::from(packet[2]); let cc = packet[3] & 0x0f; let payload = packet[3] & 0x10 != 0; + // A null packet's counter is undefined. + if pid == 0x1fff { + continue; + } if let Some(&prev) = last.get(&pid) { if payload && cc != (prev + 1) & 0x0f { discontinuities += 1; @@ -5001,6 +5088,89 @@ async fn payload_less_clock_packets_repeat_the_counter() { /// What a constant-rate receiver would measure of `ts`: the packets between its /// first and last clock packet over the PCR ticks they span, how much of that was /// null stuffing, and the widest PCR gap. +/// A heavy passage whose frames outrun the multiplex rate for longer than the delay, after a +/// light one with room to spare, fits when its units go out as early as the receiver's +/// buffers admit. Sent as late as the rate allows, the light passage's spare slots go out +/// empty and the heavy one misses its deadlines. +#[tokio::test(start_paused = true)] +async fn a_heavy_passage_fills_the_slots_before_it() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let track = broadcast + .create_track( + broadcast.unique_name(".avc1"), + hang::container::track_info(hang::catalog::PRIORITY.video), + ) + .unwrap(); + { + // Level 4.0: a 25 Mbit CPB, room for the whole passage sent early. + let mut cfg = VideoConfig::new(H264 { + profile: 0x42, + constraints: 0xc0, + level: 40, + inline: false, + }); + cfg.container = Container::Legacy; + cfg.description = + Some(crate::codec::h264::build_avcc(&[Bytes::from_static(SPS)], &[Bytes::from_static(PPS)]).unwrap()); + catalog + .modify() + .unwrap() + .video + .renditions + .insert(track.name().to_string(), cfg); + } + let mut producer = Producer::new(track, HangContainer::Legacy(crate::container::Kind::Data)); + // 2 Mb/s carries 32 media packets a 25 ms slot, 51 a 40 ms frame. Two seconds of light + // frames, then 0.8 s of frames half as big again as the rate carries, then light again. + for i in 0..100u64 { + let size = match i { + 50..70 => 77 * 184, + _ => 1_000, + }; + producer + .write(Frame { + timestamp: Timestamp::from_millis(10_000 + i * 40).unwrap(), + duration: None, + payload: length_prefixed(&[&vec![if i == 0 { 0x65 } else { 0x41 }; size]]), + keyframe: i == 0, + }) + .unwrap(); + } + producer.finish().unwrap(); + + let mut export = Export::new(crate::source::announced(&consumer)) + .await + .unwrap() + .with_mux_rate(2_000_000) + .with_delay(Duration::from_millis(500)); + let frames = drain_frames(&mut export).await; + let ts: Vec = frames.iter().flat_map(|f| f.payload.iter().copied()).collect(); + assert_eq!(video_pes_timing(&ts).len(), 100, "every frame went out"); +} + +/// Every PCR in `ts`: its packet index and value in system clock ticks, unwrapped. +fn pcr_positions(ts: &[u8]) -> Vec<(usize, u64)> { + const WRAP: u64 = (1 << 33) * 300; + let mut out: Vec<(usize, u64)> = Vec::new(); + for (index, packet) in ts.chunks(188).enumerate() { + if packet[3] & 0x20 != 0 && packet[4] >= 7 && packet[5] & 0x10 != 0 { + let base = (u64::from(packet[6]) << 25) + | (u64::from(packet[7]) << 17) + | (u64::from(packet[8]) << 9) + | (u64::from(packet[9]) << 1) + | (u64::from(packet[10]) >> 7); + let mut pcr = base * 300 + ((u64::from(packet[10] & 1) << 8) | u64::from(packet[11])); + if let Some(&(_, last)) = out.last() { + pcr = last + (pcr + WRAP - last % WRAP) % WRAP; + } + out.push((index, pcr)); + } + } + out +} + struct Clocked { packets: usize, ticks: u64, @@ -5176,12 +5346,13 @@ async fn export_without_a_mux_rate_is_unpadded() { assert_eq!(Clocked::of(&ts).nulls, 0, "null packets in an unpadded export"); } -/// 1 Mb/s is 16.62 packets per 25 ms slot. The count between the first and last -/// clock packet must be the floor of the exact allowance, which only holds when -/// the fractional remainder carries across slots instead of rounding each one. -/// Also the override supplying a rate to a catalog that has none. +/// 1 Mb/s is 16.62 packets per 25 ms slot. The fractional remainder carries across slots +/// instead of rounding each one, and each PCR is the time of its own byte at the rate, so +/// every PCR is within a system clock tick of its byte position: far inside the ±500 ns +/// PCR accuracy a TR 101 290 probe holds it to. Also the override supplying a rate to a +/// catalog that has none. #[tokio::test(start_paused = true)] -async fn export_stuffing_keeps_the_fractional_remainder() { +async fn export_pcr_is_its_byte_position_at_the_rate() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); @@ -5220,15 +5391,15 @@ async fn export_stuffing_keeps_the_fractional_remainder() { assert_packet_aligned(&ts); let clocked = Clocked::of(&ts); - let slots = clocked.ticks / (PCR_INTERVAL.as_nanos() as u64 * 27 / 1000); - assert!(slots > 300, "expected a long run of slots, got {slots}"); - // Bits allowed over the run, in whole packets. - let expected = (slots as u128 * 1_000_000 * PCR_INTERVAL.as_nanos() / (1_000_000_000 * 188 * 8)) as u64; - assert_eq!( - clocked.packets as u64, expected, - "{slots} slots at 1 Mb/s carry {expected} packets" - ); assert!(clocked.nulls > 0, "no null stuffing was emitted"); + let pcrs = pcr_positions(&ts); + assert!(pcrs.len() > 300, "expected a long run of slots, got {}", pcrs.len()); + let (first, base) = pcrs[0]; + for &(index, pcr) in &pcrs { + let at = (index - first) as u128 * 188 * 8 * 27_000_000 / 1_000_000; + let off = (u128::from(pcr - base)).abs_diff(at); + assert!(off <= 1, "the PCR at packet {index} is {off} ticks off its byte position"); + } } // The decode clock follows the stream's reordering: its delay comes from the depth the SPS @@ -5415,29 +5586,20 @@ async fn export_reordered( (out_a, out_b) } -/// `(PTS, DTS)` of every video PES start in the frames stamped within `range`, in transport -/// order. The whole stream is read so the program tables resolve every PID. +/// `(PTS, DTS)` of every video PES start presented within `range`, in transport order. fn video_timing(frames: &[Frame], range: impl std::ops::RangeBounds) -> Vec<(u64, Option)> { - let stamps: Vec = frames - .iter() - .flat_map(|f| std::iter::repeat_n(f.timestamp, f.payload.len() / 188)) - .collect(); let bytes: Vec = frames.iter().flat_map(|f| f.payload.iter().copied()).collect(); let mut reader = TsPacketReader::new(Cursor::new(bytes)); let mut out = Vec::new(); - let mut index = 0; while let Some(packet) = reader.read_ts_packet().unwrap() { - let stamp = stamps[index]; - index += 1; - if !range.contains(&stamp) { - continue; - } if let Some(TsPayload::PesStart(pes)) = packet.payload && (mpeg2ts::es::StreamId::VIDEO_MIN..=mpeg2ts::es::StreamId::VIDEO_MAX) .contains(&pes.header.stream_id.as_u8()) { let pts = pes.header.pts.expect("video PES carried no PTS").as_u64(); - out.push((pts, pes.header.dts.map(|t| t.as_u64()))); + if range.contains(&Timestamp::from_micros(pts * 1_000 / 90).unwrap()) { + out.push((pts, pes.header.dts.map(|t| t.as_u64()))); + } } } out @@ -5539,7 +5701,7 @@ async fn declared_reorder_sizes_the_decode_clock_from_the_first_frame() { /// Reordered video (each pyramid in decode order: PTS 0, 160, 80, 40, 120 ms) loses nothing on /// a clean path. The deadline is on decode time, so no B-frame is stranded behind its -/// reference or sent ahead of it, and every track interleaves in decode order. +/// reference or sent ahead of it, and every unit arrives before it decodes. #[tokio::test(start_paused = true)] async fn reordered_video_loses_nothing_on_a_clean_path() { let delay = Duration::from_millis(500); @@ -5569,11 +5731,7 @@ async fn reordered_video_loses_nothing_on_a_clean_path() { 0, "the reorder delay settled before the clock started" ); - let decode = pes_decode_in_order(&out); - assert!( - in_decode_order(&decode), - "a frame went out ahead of an earlier one: {decode:?}" - ); + assert_on_time(&out); } /// With nothing declared, the reorder delay grows to the deepest reordering muxed so far. This @@ -5648,7 +5806,8 @@ fn aac_rendition( fn write_aac(producer: &mut Producer, ms: u64) { producer .write(Frame { - timestamp: Timestamp::from_millis(ms).unwrap(), + // A minute in, so a frame can go out the whole recording delay ahead of it. + timestamp: Timestamp::from_millis(60_000 + ms).unwrap(), duration: None, payload: Bytes::from_static(&[0x21, 0x10, 0x04, 0x60]), keyframe: true, @@ -5674,7 +5833,7 @@ fn pes_count(frames: &[Frame]) -> usize { async fn drain_to_end(export: &mut Export) -> (Vec, crate::Result<()>) { let mut out = Vec::new(); loop { - match tokio::time::timeout(Duration::from_secs(5), export.next()) + match tokio::time::timeout(DRAIN, export.next()) .await .expect("the export ends") { @@ -5706,8 +5865,9 @@ async fn a_track_leaving_the_catalog_is_read_to_its_end() { write_aac(&mut kept, ms); write_aac(&mut leaving, ms); } - let mut frames = drain_frames(&mut export).await; - assert!(!frames.is_empty(), "the program started"); + // The program tables are built from the first frames, without waiting out the delay that + // would make the rest late. + let mut frames = poll_frames(&mut export); catalog.modify().unwrap().audio.renditions.remove("b.aac"); for ms in (200..300).step_by(20) { diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index 2ec25e908ff..e8a4ed0c992 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -18,10 +18,16 @@ const _: () = assert!( /// One packet in the fixed-point unit of the rate credit: one slot at `rate` bits per second /// is exactly `rate` units, so no slot rounds on its own and the remainder carries over. const PACKET: u64 = TsPacket::SIZE as u64 * 8 * SLOTS_PER_SECOND; +/// The most of a packet a decoder buffer can receive: everything after its 4-byte header. +const PAYLOAD: usize = TsPacket::SIZE - 4; /// How many empty slots to lay at most before the next packets: one second's worth. Past /// this the media had an outage rather than a coarse cadence, and a dense clock history for /// a span that carried no bytes only stalls anything pacing on the asserted values. const BACKFILL: u128 = 40; +/// The system clock in Hz, the unit of a PCR. +const SYSTEM_CLOCK: u128 = 27_000_000; +/// A PCR wraps at 2^33 ticks of 90 kHz, each 300 ticks of the system clock. +const PCR_WRAP: u128 = (1 << 33) * 300; /// The grid slot a unit whose last byte has to arrive by `nanos` must be sent by: the /// slot's bytes are timed up to its boundary. @@ -29,27 +35,88 @@ pub(super) fn slot(nanos: u128) -> u128 { nanos / PCR_INTERVAL.as_nanos() } +/// The PCR, in system clock ticks, that opens slot `index`: one slot behind its boundary, +/// so the slot's bytes are timed up to that boundary. It backs off through the 33-bit wrap +/// rather than saturating, since the wire field is a circular clock. +fn grid_pcr(index: u128) -> u128 { + let slot = PCR_INTERVAL.as_nanos() * SYSTEM_CLOCK / 1_000_000_000; + (index % PCR_WRAP * slot + PCR_WRAP - slot) % PCR_WRAP +} + +/// A PID's buffers in a receiver, as the schedule respects them (ISO 13818-1 2.4.2). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Buffer { + /// How fast the receiver takes the PID's packets on, in bits per second: its transport + /// buffer's Rx, or for video the slower leak from MB into EB. + pub rate: u64, + /// The decoder buffer (EB for video, B for audio) in bytes, which a unit fills from its + /// first packet until it decodes. `None` bounds nothing, so the PID's units go out only + /// in the slot before their own. + pub size: Option, +} + +impl Buffer { + /// The program tables' (ISO 13818-1 2.4.2.3, Rxsys). + pub const SYSTEM: Self = Self { + rate: 1_000_000, + size: None, + }; + + /// The packets one slot may carry on the PID: what its transport buffer passes on within + /// the slot, less one, since the layout spreads them across it. + fn per_slot(&self) -> usize { + let bits = u128::from(self.rate) * PCR_INTERVAL.as_nanos() / 1_000_000_000; + (bits / (TsPacket::SIZE as u128 * 8)).saturating_sub(1).max(1) as usize + } +} + /// One access unit's packets, with the tables muxed ahead of it, still to go out. struct Unit { + /// The elementary stream it carries, whose buffers bound it. + pid: u16, /// The last slot its packets may ride: the one that ends by its decode time. due: u128, - /// The first slot an unpadded stream spreads it from: the slot after the unit before it. + /// The slot after the unit pushed before it, which it may go out from even without a + /// window: nothing pushed later can be due sooner. start: u128, packets: Vec, /// Packets already laid out. sent: usize, keyframe: bool, + /// It was pushed after its due slot went out, so it goes out as soon as it can. + late: bool, + /// Its bytes in the PID's decoder buffer, until it decodes. + held: usize, } impl Unit { + fn count(&self) -> usize { + self.packets.len() / TsPacket::SIZE + } + fn remaining(&self) -> usize { - self.packets.len() / TsPacket::SIZE - self.sent + self.count() - self.sent + } + + fn pid_at(&self, packet: usize) -> u16 { + pid(&self.packets[packet * TsPacket::SIZE..]) + } + + /// How many of the packets in `range` are on `pid`. + fn packets_on(&self, range: std::ops::Range, pid: u16) -> usize { + range.filter(|&k| self.pid_at(k) == pid).count() } } +fn pid(packet: &[u8]) -> u16 { + u16::from(packet[1] & 0x1f) << 8 | u16::from(packet[2]) +} + /// One grid slot's worth of output. pub(super) struct Slot { pub index: u128, + /// The PCR that opens the slot, in system clock ticks. + pub pcr: u64, /// The media packets, in the order they were muxed. pub packets: Vec, /// How many null packets pad the slot to the multiplex rate, its clock packet included. @@ -77,7 +144,6 @@ impl Slot { .iter() .map(|p| p.as_slice()) .collect(); - let pid = |packet: &[u8]| u16::from(packet[1] & 0x1f) << 8 | u16::from(packet[2]); let mut counts: HashMap = HashMap::new(); for packet in &packets { *counts.entry(pid(packet)).or_default() += 1; @@ -127,16 +193,21 @@ impl Slot { /// Packs muxed access units onto the PCR grid. /// -/// With a multiplex rate each slot carries exactly what the rate allows, padded with null -/// packets, and each unit goes out as late as the rate lets it while still arriving by its -/// decode time. A unit bigger than one slot (a keyframe) spreads over the slots before its -/// decode time, back as far as the `window`; one that cannot fit there fails the export -/// rather than overrun the rate or arrive late. Sending no earlier than needed keeps a -/// receiver's buffers as empty as the schedule allows. +/// Each slot's packets go earliest deadline first across PIDs, each PID in its own order, and +/// each unit as soon as its PID's buffers in a receiver admit it: no more of a PID's packets +/// in a slot than its transport buffer passes on, and no more bytes than its decoder buffer +/// has room for until the units in it decode. A unit may go out up to the `window` before +/// its due slot, which keeps that far ahead of a heavy passage, so the output trails the +/// decode times by the window. /// -/// Without a rate the stream is unpadded, and each unit spreads evenly over the slots -/// between the unit before it and its own decode time. +/// With a multiplex rate each slot carries exactly what the rate allows, padded with null +/// packets, and a unit that is not complete by its due slot fails the export rather than +/// arrive late. Each slot's PCR is the time of its own first byte at the rate, so it matches +/// its byte position exactly. Without a rate the stream is unpadded, its PCRs on the grid, +/// and a unit still incomplete in its due slot goes out whole there. A rate that turns up +/// mid-stream takes over from the next slot. pub(super) struct Schedule { + /// In push order, which is each PID's decode order. units: VecDeque, /// Slots before its due slot a unit may go out: the export's delay. window: u128, @@ -146,10 +217,16 @@ pub(super) struct Schedule { credit: u64, /// The next slot to lay out. next: Option, + /// The latest due slot pushed: the clock runs at least that far. + horizon: Option, /// The due slot of the last unit pushed. last: Option, - /// Slots before this one may overrun the rate ([`Self::set_rate`]). - grace: Option, + /// Each PID's buffers, from [`Self::set_buffer`]. + buffers: HashMap, + /// Units sent into a decoder buffer: their PID, due slot and bytes held. + decoding: VecDeque<(u16, u128, usize)>, + /// The PCR of the first slot at the rate, and the packets laid out since. + clock: Option<(u128, u128)>, } impl Schedule { @@ -160,128 +237,153 @@ impl Schedule { rate: None, credit: 0, next: None, + horizon: None, last: None, - grace: None, + buffers: HashMap::new(), + decoding: VecDeque::new(), + clock: None, } } - /// Pad to `rate` bits per second from here on, or stop padding. - /// - /// A rate that turns up mid-stream (import records one only once it has measured it) - /// finds units already queued with less than a window of room before them, so for one - /// window those may overrun the rate rather than fail the export. + /// Pad to `rate` bits per second from the next slot on, or stop padding. pub fn set_rate(&mut self, rate: Option) { if self.rate != rate { self.rate = rate; self.credit = 0; - self.grace = self.next.map(|next| next + self.window); + self.clock = None; } } - /// Queue an access unit (whole TS packets) whose last byte has to arrive by `by` - /// nanoseconds: its decode time, less what the receiver needs to pass the last packet on. - /// - /// Units go out in the order they are pushed, which keeps the program tables ahead of - /// what they describe. The jitter buffer hands them over in decode order; a unit that - /// decodes before one pushed ahead of it (frames read in arrival order under a zero - /// delay) is due with it. - pub fn push(&mut self, by: u128, packets: Vec, keyframe: bool) { + /// Hold the packets on `pid` to `buffer`. + pub fn set_buffer(&mut self, pid: u16, buffer: Buffer) { + self.buffers.insert(pid, buffer); + } + + /// Queue an access unit on `pid` (whole TS packets, the tables muxed ahead of it first) + /// whose last byte has to arrive by `by` nanoseconds: its decode time, less what the + /// receiver needs to pass the last packet on. + pub fn push(&mut self, pid: u16, by: u128, packets: Vec, keyframe: bool) { let due = slot(by); - for unit in self.units.iter_mut().rev() { - if unit.due <= due { - break; - } - unit.due = due; - unit.start = unit.start.min(due); - } + self.horizon = self.horizon.max(Some(due)); let start = self.last.map_or(due, |last| (last + 1).min(due)); self.last = Some(due); self.units.push_back(Unit { + pid, due, start, packets, sent: 0, keyframe, + late: self.next.is_some_and(|next| due < next), + held: 0, }); } + /// Whether a unit is still to go out. + #[cfg(test)] + pub fn queued(&self) -> bool { + !self.units.is_empty() + } + + /// Whether nothing is left to lay out: every unit sent, and the clock past its decode time. pub fn is_empty(&self) -> bool { - self.units.is_empty() + self.units.is_empty() && self.horizon.is_none_or(|horizon| self.next.is_some_and(|next| next > horizon)) } /// Drop everything queued and start the grid afresh. pub fn clear(&mut self) { self.units.clear(); + self.decoding.clear(); self.next = None; + self.horizon = None; self.last = None; - self.grace = None; self.credit = 0; + self.clock = None; + } + + /// The first slot `unit` may go out in: a window ahead of its due slot, or only the slot + /// before it for a PID whose decoder buffer bounds nothing, and never before the slot + /// after the unit pushed ahead of it needs. + fn release(&self, unit: &Unit) -> u128 { + let ahead = match self.buffers.get(&unit.pid).and_then(|buffer| buffer.size) { + _ if unit.late => return 0, + Some(_) => self.window, + None => 1, + }; + unit.due.saturating_sub(ahead).min(unit.start) + } + + /// The slot [`Self::next`] lays out next, with `known` as it would be given. + fn index(&self, known: Option) -> Option { + let first = self.units.iter().map(|unit| self.release(unit)).min(); + match (self.next, first) { + (None, None) => None, + (None, Some(first)) => Some(first), + // Skip the empty slots of a long gap, keeping the second before what comes next. + (Some(next), first) => { + let target = first.or(known.map(|known| known.saturating_sub(self.window))); + Some(next.max(target.map_or(0, |target| target.saturating_sub(BACKFILL)))) + } + } + } + + /// The next slot to lay out, and the first slot a unit still to be pushed could be due + /// in once every unit that may ride it is known: the `known` that settles it. + pub fn upcoming(&self) -> Option<(u128, u128)> { + let index = self.index(None)?; + Some((index, index + self.window + 1)) } /// Lay out the next slot, if it is settled. /// /// `known` is the first slot a unit still to be pushed could be due in, which settles /// every slot whose window ends before it; `None` means nothing more is coming, so every - /// queued unit goes out. + /// queued unit goes out and the clock runs on to the last decode time. pub fn next(&mut self, known: Option) -> anyhow::Result> { - let first = match (&self.rate, self.units.front()) { - (Some(_), Some(unit)) => Some(unit.due.saturating_sub(self.window)), - (None, Some(_)) => self.units.iter().map(|unit| unit.start).min(), - (_, None) => None, - }; - let index = match (self.next, first) { - (None, None) => return Ok(None), - (None, Some(first)) => first, - // Skip the empty slots of a long gap, keeping the second before what comes next. - (Some(next), first) => { - let window = self.rate.map_or(0, |_| self.window); - let target = first.or(known.map(|known| known.saturating_sub(window))); - next.max(target.map_or(0, |target| target.saturating_sub(BACKFILL))) - } + if known.is_none() && self.is_empty() { + return Ok(None); + } + let Some(index) = self.index(known) else { + return Ok(None); }; - // An unpadded unit spreads only back to the one before it, so it needs no window. - let window = self.rate.map_or(0, |_| self.window); - match known { - Some(known) if index + window >= known => return Ok(None), - None if self.units.is_empty() => return Ok(None), - _ => {} + if known.is_some_and(|known| index + self.window >= known) { + return Ok(None); + } + if self.next != Some(index) { + // A skipped gap carried no bytes, so the clock starts over past it. + self.clock = None; } self.next = Some(index + 1); - let (take, nulls) = match self.rate { + let (take, nulls, pcr) = match self.rate { Some(rate) => { self.credit += rate; let allowed = (self.credit / PACKET).max(1); self.credit -= (self.credit / PACKET).min(allowed) * PACKET; - let media = self.needed(index, rate / PACKET); - let overrun = media >= allowed as usize; - anyhow::ensure!( - !overrun || self.grace.is_some_and(|grace| index < grace), - "MPEG-TS output needs {media} packets in a {}ms slot, more than {rate} b/s allows within the delay; raise the delay or the multiplex rate", - PCR_INTERVAL.as_millis() - ); - let mut left = media; - let take: Vec = self - .units - .iter() - .map(|unit| { - let take = unit.remaining().min(left); - left -= take; - take - }) - .collect(); - (take, (allowed as usize - 1).saturating_sub(media)) + let take = self.admit(index, allowed as usize - 1); + if let Some(unit) = self.missed(index, &take) { + anyhow::bail!( + "MPEG-TS output missed a decode deadline on PID {} at {rate} b/s; raise the delay or the multiplex rate", + unit.pid, + ); + } + let media: usize = take.iter().sum(); + let (base, laid) = self.clock.get_or_insert((grid_pcr(index), 0)); + let pcr = (*base + *laid * TsPacket::SIZE as u128 * 8 * SYSTEM_CLOCK / u128::from(rate)) % PCR_WRAP; + *laid += u128::from(allowed); + (take, allowed as usize - 1 - media, pcr) } None => { - let take = self - .units - .iter() - .map(|unit| match unit.start <= index { - true => unit.remaining().div_ceil((unit.due.max(index) - index + 1) as usize), - false => 0, - }) - .collect(); - (take, 0) + let mut take = self.admit(index, usize::MAX); + for (unit, take) in self.units.iter_mut().zip(take.iter_mut()) { + if unit.due <= index && *take < unit.remaining() { + if self.buffers.get(&unit.pid).is_some_and(|buffer| buffer.size.is_some()) { + unit.held += unit.packets_on(unit.sent + *take..unit.count(), unit.pid) * PAYLOAD; + } + *take = unit.remaining(); + } + } + (take, 0, grid_pcr(index)) } }; @@ -296,26 +398,95 @@ impl Schedule { packets.extend_from_slice(&unit.packets[from..from + take * TsPacket::SIZE]); unit.sent += take; } + for unit in self.units.iter().filter(|unit| unit.remaining() == 0 && unit.held > 0) { + self.decoding.push_back((unit.pid, unit.due, unit.held)); + } self.units.retain(|unit| unit.remaining() > 0); Ok(Some(Slot { index, + pcr: pcr as u64, packets, nulls, keyframe, })) } - /// The fewest packets slot `index` must carry so every queued unit still arrives by its - /// due slot, with `per_slot` packets of room in each slot after it. - fn needed(&self, index: u128, per_slot: u64) -> usize { - let mut queued = 0; - let mut needed = 0; + /// How many of each unit's packets slot `index` carries, at most `budget` in all. + /// + /// Earliest deadline first across PIDs. A PID whose unit stops short (its buffers full, + /// or not yet released) takes nothing more this slot, so each keeps its own order. + fn admit(&mut self, index: u128, mut budget: usize) -> Vec { + // A unit due in slot k decodes during slot k + 1, so its bytes leave the decoder + // buffer once that slot is past. + let mut occupancy: HashMap = HashMap::new(); + self.decoding.retain(|&(_, due, _)| due + 1 >= index); + for &(pid, _, held) in &self.decoding { + *occupancy.entry(pid).or_default() += held; + } for unit in &self.units { - queued += unit.remaining(); - let room = (unit.due.saturating_sub(index) as usize).saturating_mul(per_slot.saturating_sub(1) as usize); - needed = needed.max(queued.saturating_sub(room)); + *occupancy.entry(unit.pid).or_default() += unit.held; + } + + let mut order: Vec = (0..self.units.len()).collect(); + order.sort_by_key(|&i| self.units[i].due); + let mut take = vec![0; self.units.len()]; + let mut carried: HashMap = HashMap::new(); + let mut stopped: Vec = Vec::new(); + for i in order { + let unit = &self.units[i]; + if budget == 0 { + break; + } + if stopped.contains(&unit.pid) { + continue; + } + let size = self.buffers.get(&unit.pid).and_then(|buffer| buffer.size); + let mut n = 0; + if self.release(unit) <= index { + while n < unit.remaining() && budget > 0 { + let pid = unit.pid_at(unit.sent + n); + let cap = self.buffers.get(&pid).map_or(usize::MAX, Buffer::per_slot); + let count = carried.entry(pid).or_default(); + if *count >= cap { + break; + } + if pid == unit.pid + && let Some(size) = size + { + let held = occupancy.entry(pid).or_default(); + if *held + PAYLOAD > size { + break; + } + *held += PAYLOAD; + } + *count += 1; + budget -= 1; + n += 1; + } + } + if n < unit.remaining() { + stopped.push(unit.pid); + } + take[i] = n; } - needed + + // What a sized PID took stays in its decoder buffer until the unit decodes. + for (unit, &n) in self.units.iter_mut().zip(&take) { + if self.buffers.get(&unit.pid).is_some_and(|buffer| buffer.size.is_some()) { + unit.held += unit.packets_on(unit.sent..unit.sent + n, unit.pid) * PAYLOAD; + } + } + take + } + + /// The first unit due by slot `index` that `take` leaves incomplete, unless it was already + /// late when pushed. + fn missed(&self, index: u128, take: &[usize]) -> Option<&Unit> { + self.units + .iter() + .zip(take) + .find(|(unit, take)| !unit.late && unit.due <= index && unit.sent + **take < unit.count()) + .map(|(unit, _)| unit) } } @@ -338,86 +509,203 @@ mod tests { /// 40 packets per slot, the clock packet included. const RATE: u64 = 40 * PACKET; + /// A video-like buffer: far more than a slot passes on, and a large decoder buffer. + const VIDEO: Buffer = Buffer { + rate: 100_000_000, + size: Some(1_000_000), + }; + + /// Every slot until the schedule is empty: (index, packets per PID, nulls). + fn drain(schedule: &mut Schedule) -> Vec<(u128, HashMap, usize)> { + let mut slots = Vec::new(); + while let Some(slot) = schedule.next(None).unwrap() { + let mut per_pid = HashMap::new(); + for packet in slot.packets.as_chunks::<{ TsPacket::SIZE }>().0 { + *per_pid.entry(pid(packet)).or_default() += 1; + } + slots.push((slot.index, per_pid, slot.nulls)); + } + slots + } + /// A unit goes out a window ahead of its due slot, as soon as it is released. #[test] - fn a_unit_goes_out_in_its_due_slot() { + fn a_unit_goes_out_as_soon_as_it_is_released() { let mut schedule = Schedule::new(Duration::from_millis(100)); schedule.set_rate(Some(RATE)); - schedule.push(ms(1_000), unit(1, 3), true); - // Nothing past the window is settled until a later unit shows up. - let mut slots = Vec::new(); - while let Some(slot) = schedule.next(Some(slot(ms(1_200)))).unwrap() { - slots.push(( - slot.index, - slot.packets.len() / TsPacket::SIZE, - slot.nulls, - slot.keyframe, - )); - } - assert_eq!( - slots.first().map(|s| s.0), - Some(36), - "starts a window ahead of the unit" - ); - assert!( - slots.iter().all(|s| s.1 + s.2 == 39), - "every slot is padded to the rate" - ); - let sent: Vec<_> = slots.iter().filter(|s| s.1 > 0).collect(); - assert_eq!(sent, [&(40, 3, 36, true)], "as late as it can go: {slots:?}"); + schedule.set_buffer(1, VIDEO); + schedule.push(1, ms(1_000), unit(1, 3), true); + let slots = drain(&mut schedule); + assert_eq!(slots[0].0, 36, "a window ahead of its due slot"); + assert_eq!(slots[0].1[&1], 3); + assert_eq!(slots[0].2, 36, "padded to the rate"); + let clock: Vec = slots.iter().map(|slot| slot.0).collect(); + assert_eq!(clock, [36, 37, 38, 39, 40], "the clock runs on to its decode time"); } + /// A burst bigger than a slot fills the slots from its release on. #[test] - fn a_burst_spreads_over_the_slots_before_it() { + fn a_burst_fills_the_slots_from_its_release() { let mut schedule = Schedule::new(Duration::from_millis(100)); schedule.set_rate(Some(RATE)); - schedule.push(ms(1_000), unit(1, 100), true); - let mut sent = Vec::new(); - while let Some(slot) = schedule.next(None).unwrap() { - sent.push((slot.index, slot.packets.len() / TsPacket::SIZE)); - } - // 39 media packets fit each slot, so 100 packets take the last three before its due. - assert_eq!(sent, [(36, 0), (37, 0), (38, 22), (39, 39), (40, 39)]); + schedule.set_buffer(1, VIDEO); + schedule.push(1, ms(1_000), unit(1, 100), true); + let sent: Vec<_> = drain(&mut schedule) + .into_iter() + .map(|(index, per_pid, _)| (index, per_pid.get(&1).copied().unwrap_or(0))) + .collect(); + assert_eq!(sent, [(36, 39), (37, 39), (38, 22), (39, 0), (40, 0)]); } + /// A burst the window cannot carry by its due slot fails the export. #[test] fn a_burst_too_big_for_the_window_fails() { let mut schedule = Schedule::new(Duration::from_millis(50)); schedule.set_rate(Some(RATE)); - schedule.push(ms(1_000), unit(1, 200), true); + schedule.set_buffer(1, VIDEO); + schedule.push(1, ms(1_000), unit(1, 200), true); let err = std::iter::from_fn(|| schedule.next(None).transpose()).find_map(Result::err); assert!(err.is_some(), "a burst past the window must fail the export"); } - /// A rate that turns up mid-stream lets the units already queued overrun it for one - /// window, rather than fail the export. + /// The unit due first goes first, whatever order the PIDs were pushed in. #[test] - fn a_late_rate_overruns_for_one_window() { + fn the_earliest_deadline_goes_first() { let mut schedule = Schedule::new(Duration::from_millis(100)); - schedule.push(ms(1_000), unit(1, 4), true); - while schedule.next(Some(slot(ms(1_050)))).unwrap().is_some() {} schedule.set_rate(Some(RATE)); - schedule.push(ms(1_100), unit(1, 150), true); - let mut sent = Vec::new(); - while let Some(slot) = schedule.next(None).unwrap() { - sent.push((slot.index, slot.packets.len() / TsPacket::SIZE, slot.nulls)); + schedule.set_buffer(1, VIDEO); + schedule.set_buffer(2, VIDEO); + schedule.push(1, ms(1_100), unit(1, 39), true); + schedule.push(2, ms(1_000), unit(2, 39), false); + let sent: Vec<_> = drain(&mut schedule) + .into_iter() + .filter(|(_, per_pid, _)| !per_pid.is_empty()) + .map(|(index, per_pid, _)| (index, per_pid.get(&1).copied(), per_pid.get(&2).copied())) + .collect(); + assert_eq!(sent, [(36, None, Some(39)), (40, Some(39), None)]); + } + + /// A PID takes no more packets in a slot than its transport buffer passes on, so a + /// backlog of small audio units spreads out rather than going out in bulk. + #[test] + fn a_pid_takes_what_its_transport_buffer_passes_on() { + let mut schedule = Schedule::new(Duration::from_millis(100)); + schedule.set_rate(Some(RATE)); + // 2 Mb/s: 33 packets a slot, less one. + let audio = Buffer { + rate: 2_000_000, + size: Some(100_000), + }; + schedule.set_buffer(1, audio); + for at in 0..20 { + schedule.push(1, ms(1_000 + at), unit(1, 4), false); } - assert_eq!(sent, [(42, 72, 0), (43, 39, 0), (44, 39, 0)]); + for (index, per_pid, _) in drain(&mut schedule) { + assert!(per_pid.get(&1).copied().unwrap_or(0) <= 32, "slot {index}: {per_pid:?}"); + } + } - schedule.push(ms(3_000), unit(1, 2_000), true); - let err = std::iter::from_fn(|| schedule.next(None).transpose()).find_map(Result::err); - assert!(err.is_some(), "past the window, a burst that does not fit fails"); + /// A PID never holds more in its decoder buffer than it has: the next unit waits for the + /// one before it to decode. + #[test] + fn a_full_decoder_buffer_holds_the_next_unit() { + let mut schedule = Schedule::new(Duration::from_millis(200)); + schedule.set_rate(Some(RATE)); + // Room for ten packets. + let small = Buffer { + rate: 100_000_000, + size: Some(10 * PAYLOAD), + }; + schedule.set_buffer(1, small); + schedule.push(1, ms(1_000), unit(1, 8), true); + schedule.push(1, ms(1_100), unit(1, 8), false); + let sent: Vec<_> = drain(&mut schedule) + .into_iter() + .filter_map(|(index, per_pid, _)| per_pid.get(&1).map(|&n| (index, n))) + .collect(); + // The first unit goes at once; the second fills what is left, then the rest once the + // first decodes in slot 41. + assert_eq!(sent, [(32, 8), (36, 2), (42, 6)]); } + /// A PID with no decoder buffer to respect goes out only in the slot before its own. #[test] - fn unpadded_units_spread_up_to_their_decode_time() { + fn an_unbounded_pid_goes_just_before_it_is_due() { + let mut schedule = Schedule::new(Duration::from_millis(100)); + schedule.set_rate(Some(RATE)); + schedule.push(7, ms(1_000), unit(7, 2), false); + let sent: Vec<_> = drain(&mut schedule) + .into_iter() + .filter_map(|(index, per_pid, _)| per_pid.get(&7).map(|&n| (index, n))) + .collect(); + assert_eq!(sent, [(39, 2)]); + } + + /// Each PCR is the time of its slot's first byte at the rate, so a rate that does not + /// divide into whole packets per slot moves it off the grid by less than a packet. + #[test] + fn the_pcr_is_its_byte_position_at_the_rate() { let mut schedule = Schedule::new(Duration::ZERO); - schedule.push(ms(1_000), unit(1, 4), true); - schedule.push(ms(1_100), unit(1, 4), false); - let mut sent = Vec::new(); + // 10 Mb/s is 166.2 packets a slot. + let rate = 10_000_000; + schedule.set_rate(Some(rate)); + schedule.set_buffer(1, VIDEO); + for at in 0..40 { + schedule.push(1, ms(1_000 + 25 * at), unit(1, 1), false); + } + let mut laid = 0u128; + let mut first = None; while let Some(slot) = schedule.next(None).unwrap() { - sent.push((slot.index, slot.packets.len() / TsPacket::SIZE, slot.nulls)); + let base = *first.get_or_insert(u128::from(slot.pcr)); + let expected = base + laid * TsPacket::SIZE as u128 * 8 * SYSTEM_CLOCK / u128::from(rate); + assert_eq!(u128::from(slot.pcr), expected, "slot {}", slot.index); + let grid = grid_pcr(slot.index); + let packet = TsPacket::SIZE as u128 * 8 * SYSTEM_CLOCK / u128::from(rate); + assert!(grid - u128::from(slot.pcr) < packet, "slot {} strays a packet off the grid", slot.index); + laid += 1 + (slot.packets.len() / TsPacket::SIZE + slot.nulls) as u128; } - assert_eq!(sent, [(40, 4, 0), (41, 1, 0), (42, 1, 0), (43, 1, 0), (44, 1, 0)]); + } + + /// A rate that turns up mid-stream takes over the queue from the next slot. + #[test] + fn a_rate_takes_over_from_the_next_slot() { + let mut schedule = Schedule::new(Duration::from_millis(100)); + schedule.set_buffer(1, VIDEO); + schedule.push(1, ms(1_000), unit(1, 8), true); + schedule.push(1, ms(1_100), unit(1, 100), false); + // Unpadded, both go out as soon as they are released, a window ahead. + let first: Vec<_> = std::iter::from_fn(|| schedule.next(Some(slot(ms(1_100)))).unwrap()) + .map(|slot| (slot.index, slot.packets.len() / TsPacket::SIZE, slot.nulls)) + .collect(); + assert_eq!(first, [(36, 8, 0), (37, 0, 0), (38, 0, 0), (39, 0, 0)]); + schedule.set_rate(Some(RATE)); + let sent: Vec<_> = drain(&mut schedule) + .into_iter() + .map(|(index, per_pid, _)| (index, per_pid.get(&1).copied().unwrap_or(0))) + .collect(); + assert_eq!(sent, [(40, 39), (41, 39), (42, 22), (43, 0), (44, 0)]); + } + + /// Unpadded, a unit still incomplete in its due slot goes out whole there, a unit goes out + /// from the slot after the one before it even without a window, and the clock runs on to + /// the last decode time. + #[test] + fn unpadded_units_go_out_by_their_decode_time() { + let mut schedule = Schedule::new(Duration::ZERO); + // 2 Mb/s passes 32 packets a slot. + schedule.set_buffer( + 1, + Buffer { + rate: 2_000_000, + size: Some(100_000), + }, + ); + schedule.push(1, ms(1_000), unit(1, 50), true); + schedule.push(1, ms(1_100), unit(1, 4), false); + let sent: Vec<_> = drain(&mut schedule) + .into_iter() + .map(|(index, per_pid, nulls)| (index, per_pid.get(&1).copied().unwrap_or(0), nulls)) + .collect(); + assert_eq!(sent, [(40, 50, 0), (41, 4, 0), (42, 0, 0), (43, 0, 0), (44, 0, 0)]); } } diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index 5577adbd9f8..784f2578493 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -153,6 +153,12 @@ impl Buffer { }) } + /// When a frame decoding at `decode` goes out on the current clock, once there is one. + pub fn at(&self, decode: Timestamp) -> Option { + let (_, anchor, base) = self.clock?; + self.deadline(anchor, base, decode) + } + /// When a frame decoding at `decode` goes out on the clock anchored at `anchor` and /// `base`, if any instant holds it. fn deadline(&self, anchor: Instant, base: Timestamp, decode: Timestamp) -> Option { From aa82b100a3be5dc469acc864ebbd6822528e6157 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 11:20:55 -0700 Subject: [PATCH 18/40] fix(moq-mux): send passed-through AC-3 a sync frame a PES DVB AC-3 passed through as private data keeps the source's PES, which can carry nine sync frames, more than a receiver's 5,696-byte buffer holds at once. The export now splits such a PES into one PES a sync frame, each presented a frame after the last and due at its own decode time, and holds the PID to that buffer. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export.rs | 79 +++++++++++++++++--- rs/moq-mux/src/container/ts/export_test.rs | 87 ++++++++++++++++++++++ 2 files changed, 154 insertions(+), 12 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 1793320c5b4..749f6034f58 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -248,6 +248,19 @@ impl Track { self.release(name, jitter, false); } + /// Whether the track passes AC-3 through as DVB private data, whose PES may carry several + /// sync frames. + fn carries_ac3(&self) -> bool { + matches!( + self.kind, + Kind::Verbatim { + stream_type: 0x06, + framing: catalog::Framing::Pes, + .. + } + ) && self.descriptors.iter().any(|descriptor| descriptor.tag == AC3_DESCRIPTOR) + } + /// The receiver's buffers for the track's PID, if the T-STD gives it any. fn buffer(&self) -> Option { match self.kind { @@ -1589,17 +1602,40 @@ impl Export { Kind::Opus { .. } => Some(StreamId::PRIVATE_STREAM_1), _ => None, }; - let unit = PesUnit { - pid, - is_video, - keyframe: frame.keyframe, - timestamp: frame.timestamp, - dts, - stream_id, - }; - self.write_pes(&mut out, &unit, &es_payload)?; + // A passed-through PES of several AC-3 sync frames goes out a frame a PES, each + // due at its own decode time: a receiver's buffer holds and decodes each on its + // own, and the whole PES would overflow it. + let track = self.tracks.get(name).context("missing track")?; + let frames = track + .carries_ac3() + .then(|| ac3_frames(frame.timestamp, &es_payload)) + .flatten() + .unwrap_or_else(|| vec![(frame.timestamp, es_payload.as_slice())]); + let mut at = decode; + for (k, (timestamp, payload)) in frames.into_iter().enumerate() { + if k > 0 { + let unit = std::mem::take(&mut out); + self.push_unit(name, pid, at, unit, keyframe && k == 1)?; + at = timestamp; + } + let unit = PesUnit { + pid, + is_video, + keyframe: frame.keyframe && k == 0, + timestamp, + dts: dts.filter(|_| k == 0), + stream_id, + }; + self.write_pes(&mut out, &unit, payload)?; + } + return self.push_unit(name, pid, at, out, keyframe && at == decode); } } + self.push_unit(name, pid, decode, out, keyframe) + } + + /// Queue one unit of `name`'s packets on `pid`, decoding at `decode`, on the [`Schedule`]. + fn push_unit(&mut self, name: &str, pid: u16, decode: Timestamp, out: Vec, keyframe: bool) -> anyhow::Result<()> { let track = self.tracks.get(name).context("missing track")?; if let Some(buffer) = track.buffer() { self.schedule.set_buffer(pid, buffer); @@ -2102,8 +2138,11 @@ fn audio_buffer(config: &AudioConfig, kind: &Kind) -> Buffer { Buffer { rate, size: Some(size) } } -/// A verbatim stream's buffers, for the kinds its descriptors identify: AC-3 and E-AC-3 as -/// DVB private data (ATSC A/52 Annex A 5.4). Any other stream has none the schedule knows. +/// The DVB AC-3 descriptor's tag (ETSI EN 300 468 6.2.1), marking AC-3 as private data. +const AC3_DESCRIPTOR: u8 = 0x6a; + +/// A verbatim stream's buffers, for the kinds its descriptors identify: AC-3 as DVB private +/// data (ATSC A/52 Annex A 5.4). Any other stream has none the schedule knows. fn verbatim_buffer(kind: &Kind, descriptors: &[catalog::Descriptor]) -> Option { let Kind::Verbatim { stream_type: 0x06, @@ -2115,13 +2154,29 @@ fn verbatim_buffer(kind: &Kind, descriptors: &[catalog::Descriptor]) -> Option Option> { + let mut frames = Vec::new(); + let (mut rest, mut samples) = (payload, 0u64); + while !rest.is_empty() { + let header = (crate::codec::ac3::DESCRIPTOR.parse)(rest).ok()?; + let (frame, tail) = rest.split_at_checked(header.len)?; + let offset = Timestamp::from_micros(samples * 1_000_000 / u64::from(header.sample_rate.max(1))).ok()?; + frames.push((start.checked_add(offset).ok()?, frame)); + samples += header.samples; + rest = tail; + } + (frames.len() > 1).then_some(frames) +} + /// Build the Annex-B elementary-stream payload for one video frame: rewrite the /// length-prefixed NALs to start-code-delimited NALs, prepending the parameter /// sets (SPS/PPS, plus VPS for H.265) from the avcC/hvcC on keyframes so a diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 9dfdf950a6d..03c86ba9ad3 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -1034,6 +1034,93 @@ async fn export_pcr_backfills_a_coarse_cadence() { } } +/// AC-3 passed through as DVB private data (stream_type 0x06 with the AC-3 descriptor) can +/// carry several sync frames in one PES, more than a receiver's 5,696-byte buffer holds at +/// once. The export splits it a frame a PES, each presented a frame after the last, so each +/// is due and decoded on its own. +#[tokio::test(start_paused = true)] +async fn export_splits_a_multi_frame_ac3_pes() { + const AC3_PID: u16 = 0x104; + // A 32 kb/s 48 kHz 2/0 sync frame: 128 bytes, 1,536 samples (32 ms). + let mut sync = vec![0x0b, 0x77, 0x00, 0x00, 0x00, 0x40, 0x40]; + sync.resize(128, 0x00); + + let data = include_bytes!("test_data/bbb.ts"); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let ac3_track = broadcast + .unique_track(".ac3", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + { + let mut track = tscat::Track::new(AC3_PID); + track.verbatim = Some(tscat::Verbatim::new(0x06, tscat::Framing::Pes)); + track.descriptors = vec![tscat::Descriptor { + tag: 0x6a, + data: Bytes::from_static(&[0x00]), + }]; + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .insert(ac3_track.name().to_string(), track); + } + let mut ac3 = Producer::new(ac3_track, HangContainer::Legacy(crate::container::Kind::Data)); + // Just after bbb's first keyframe at 1.4 s, so it survives the tune-in alignment. + ac3.write(Frame { + timestamp: Timestamp::from_millis(1_410).unwrap(), + duration: None, + payload: Bytes::from(sync.repeat(3)), + keyframe: true, + }) + .unwrap(); + ac3.cut(None).unwrap(); + ac3.finish().unwrap(); + + let mut import = crate::container::ts::Import::new(broadcast, catalog.reserve()); + import.decode(&BytesMut::from(&data[..])).unwrap(); + import.finish().unwrap(); + let ts = drain_with( + Export::with_ts(crate::source::announced(&consumer), crate::catalog::CatalogFormat::Hang) + .await + .unwrap(), + ) + .await; + + // The PTS of every PES on the AC-3 PID, and how many bytes each carries. + let mut pes: Vec<(u64, usize)> = Vec::new(); + for packet in ts.as_chunks::<188>().0 { + let pid = (u16::from(packet[1] & 0x1f) << 8) | u16::from(packet[2]); + if pid != AC3_PID || packet[3] & 0x10 == 0 { + continue; + } + let start = 4 + if packet[3] & 0x20 != 0 { usize::from(packet[4]) + 1 } else { 0 }; + let body = &packet[start..]; + if packet[1] & 0x40 != 0 { + let b = &body[9..14]; + let pts = u64::from((b[0] >> 1) & 7) << 30 + | u64::from(b[1]) << 22 + | u64::from(b[2] >> 1) << 15 + | u64::from(b[3]) << 7 + | u64::from(b[4] >> 1); + pes.push((pts, body.len() - 9 - usize::from(body[8]))); + } else if let Some(last) = pes.last_mut() { + last.1 += body.len(); + } + } + assert_eq!(pes.len(), 3, "one PES a sync frame: {pes:?}"); + for (k, &(pts, bytes)) in pes.iter().enumerate() { + assert_eq!(bytes, 128, "frame {k} carries one sync frame"); + assert_eq!(pts, pes[0].0 + k as u64 * 2_880, "frame {k} is presented 32 ms after the last"); + } +} + /// Full SCTE-35 round-trip: import `bbb.ts` (real H.264 + AAC) into a broadcast /// that also carries a `.scte35` cue track, export to TS, re-import, and assert /// the splice_info_section came back byte-for-byte. The PMT must advertise the From 11d68e77e0afca1e33aabdc39015f35ab122996c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 11:34:34 -0700 Subject: [PATCH 19/40] fix(moq-mux): follow the source clock in the TS jitter buffer A source whose clock ran slower than ours made every frame late after delay / drift (under five hours at 30 ppm and 500 ms), and a faster one held more and more. The jitter buffer now steers its clock's rate, within 500 ppm, so the least slack a frame arrives with stays at the delay; that also wears away a lead the first frame anchored it with. The export lays each slice out when that clock reaches it, the tail of a finished broadcast included, so moq export ts writes slices as they come and its own pacer, which would have fought the drift, is gone. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 8 +- rs/moq-cli/src/publish.rs | 5 +- rs/moq-cli/src/subscribe.rs | 169 +-------------------- rs/moq-mux/src/container/ts/export.rs | 21 ++- rs/moq-mux/src/container/ts/export_test.rs | 46 +++++- rs/moq-mux/src/jitter.rs | 133 ++++++++++++++-- rs/moq-mux/src/pace.rs | 9 +- 7 files changed, 195 insertions(+), 196 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 0a944ec20cf..f3fb4fcb397 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -367,8 +367,12 @@ with the clock started when the first frame arrived, so the output keeps the source's pace and muxes all tracks in decode order whatever their arrival skew: two exporters of one broadcast emit them in one order. A frame that arrives later than its deadline is dropped, and video then resumes at its next -keyframe. The delay is also how stale a group may get before it is skipped. -`--delay 0` holds nothing and drops nothing, writing frames in arrival order. +keyframe. The clock follows the source's, so a source whose clock runs a little +fast or slow (the 30 ppm a broadcast system clock may) neither goes late nor +piles up over a long run, and a lead it started with wears away. Each 25 ms +slice of the output is written when the clock reaches it. The delay is also how +stale a group may get before it is skipped. `--delay 0` holds nothing and drops +nothing, writing frames in arrival order as they come. A stdout export ends with the broadcast. `export ts --linger 10s` waits that long for the broadcast to come back instead: a publisher that restarts within diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 1d94a98f92d..d6eceeb339d 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -633,8 +633,9 @@ mod tests { /// `next()` blocks, surfaced here as a timeout once the buffered frames are gone. async fn drain(mut exporter: Export) -> Vec { let mut out = Vec::new(); - // Past the export's release delay, so the first frame goes out before it ends. - let wait = RECORDING_MAX_AGE + Duration::from_secs(1); + // Past the export's release delay and the send-ahead window behind it, so the first + // frame goes out before it ends. + let wait = RECORDING_MAX_AGE * 3 + Duration::from_secs(1); while let Ok(res) = tokio::time::timeout(wait, exporter.next()).await { match res.expect("exporter error") { Some(frame) => out.extend_from_slice(&frame.payload), diff --git a/rs/moq-cli/src/subscribe.rs b/rs/moq-cli/src/subscribe.rs index cb87f0744b2..c2574ec7692 100644 --- a/rs/moq-cli/src/subscribe.rs +++ b/rs/moq-cli/src/subscribe.rs @@ -314,13 +314,9 @@ impl Subscribe { } // A TS byte stream carries no per-frame timing, so delivery time is the only - // carrier of each frame's spacing: the exporter slices its output on the PCR - // grid and stamps each slice at its slot boundary, on the contract that the - // caller writes the bytes at the time the stamp asserts. Draining on arrival - // instead collapses the clock into position clusters no downstream stage can - // repair (#2984). The export already releases at the source's pace, so the - // pacer only spreads each burst of settled slices over the time they cover. - let mut delivery = Delivery::new(self.args.max_age); + // carrier of each frame's spacing (#2984). The export lays each slice of the PCR + // grid out at its time on its own clock, which follows the source's, so each is + // written as it comes. let linger = self.args.linger; loop { let end = loop { @@ -329,7 +325,8 @@ impl Subscribe { Ok(None) => break Ok(()), Err(err) => break Err(err), }; - delivery.deliver(&frame, ts.discontinuity(), &mut stdout).await?; + stdout.write_all(&frame.payload).await?; + stdout.flush().await?; }; // Any end waits out the linger, and on expiry the last one is the result: a @@ -392,165 +389,9 @@ async fn resume_within( } } -/// Paced stdout delivery for the TS export: sleeps until each frame's send instant. -/// -/// The export holds every frame its delay and lays the multiplex out a slot at a time, -/// so it hands slices over at the source's pace, a few at once. The pacer spreads each -/// handful over the slots it covers, holding up to the delay ahead of the wall clock. -/// A sink that falls behind writes what is overdue at once and catches up. -struct Delivery { - discontinuity: u64, - pacer: moq_mux::Pacer, -} - -impl Delivery { - fn new(lead: Duration) -> Self { - Self { - pacer: moq_mux::Pacer::default().with_lead(lead), - discontinuity: 0, - } - } - - /// Write one export frame to `out` at its paced instant. A new `discontinuity` - /// (the program clock restarted) re-anchors the pacing on this frame. - async fn deliver( - &mut self, - frame: &moq_mux::container::Frame, - discontinuity: u64, - out: &mut (impl tokio::io::AsyncWrite + Unpin), - ) -> anyhow::Result<()> { - // tokio's clock rather than the bare std one so tests can pause it; in - // production they are identical. - let now = tokio::time::Instant::now().into_std(); - let send_at = match std::mem::replace(&mut self.discontinuity, discontinuity) == discontinuity { - true => self.pacer.pace(frame.timestamp, now), - false => self.pacer.hurry(frame.timestamp, now), - }; - tokio::time::sleep_until(tokio::time::Instant::from_std(send_at)).await; - out.write_all(&frame.payload).await?; - out.flush().await?; - Ok(()) - } -} - #[cfg(test)] mod tests { use super::*; - use hang::moq_net::{Timescale, Timestamp}; - - fn frame(value: u64, scale: Timescale) -> moq_mux::container::Frame { - moq_mux::container::Frame { - timestamp: Timestamp::new(value, scale).unwrap(), - duration: None, - payload: bytes::Bytes::from_static(&[0x47; 188]), - keyframe: false, - } - } - - /// Regression for #2984: the TS stdout writer must deliver each frame at the - /// instant its timestamp asserts, not as fast as frames arrive. The exporter - /// stamps PCR grid frames in microseconds and media frames at the source's own - /// timescale, so the spacing must also survive a scale change mid-stream. - #[tokio::test(start_paused = true)] - async fn ts_frames_are_paced_on_the_media_clock() { - let mut delivery = Delivery::new(Duration::from_millis(500)); - let mut out = Vec::new(); - - let start = tokio::time::Instant::now(); - - // The first frame anchors the pacer and is written immediately. - delivery - .deliver(&frame(0, Timescale::MICRO), 0, &mut out) - .await - .unwrap(); - assert_eq!(start.elapsed(), Duration::ZERO); - - // A PCR slot 25ms later waits for its grid boundary. - delivery - .deliver(&frame(25_000, Timescale::MICRO), 0, &mut out) - .await - .unwrap(); - assert_eq!(start.elapsed(), Duration::from_millis(25)); - - // A media frame at the source's 90 kHz timescale paces on the same clock. - delivery - .deliver(&frame(3_600, Timescale::new(90_000).unwrap()), 0, &mut out) - .await - .unwrap(); - assert_eq!(start.elapsed(), Duration::from_millis(40)); - - assert_eq!(out.len(), 3 * 188, "every payload was written"); - } - - /// A sink that falls behind writes what is overdue at once, then paces again. - #[tokio::test(start_paused = true)] - async fn a_slow_sink_catches_up() { - let mut delivery = Delivery::new(Duration::from_millis(500)); - let mut out = Vec::new(); - - let start = tokio::time::Instant::now(); - delivery - .deliver(&frame(0, Timescale::MICRO), 0, &mut out) - .await - .unwrap(); - - // The writer stalls for 2s (a blocked pipe): every slot after this is overdue. - tokio::time::advance(Duration::from_secs(2)).await; - delivery - .deliver(&frame(400_000, Timescale::MICRO), 0, &mut out) - .await - .unwrap(); - assert_eq!( - start.elapsed(), - Duration::from_secs(2), - "an overdue slot writes at once" - ); - } - - /// Slots the export has ready at once still go out on their grid. - #[tokio::test(start_paused = true)] - async fn a_buffered_producer_keeps_pacing() { - let mut delivery = Delivery::new(Duration::from_millis(500)); - let mut out = Vec::new(); - - let start = tokio::time::Instant::now(); - delivery - .deliver(&frame(0, Timescale::MICRO), 0, &mut out) - .await - .unwrap(); - - for slot in 1..=40u64 { - delivery - .deliver(&frame(slot * 25_000, Timescale::MICRO), 0, &mut out) - .await - .unwrap(); - assert_eq!( - start.elapsed(), - Duration::from_millis(slot * 25), - "slot {slot} must be paced, not shed" - ); - } - } - #[tokio::test(start_paused = true)] - async fn a_rewind_re_anchors_the_pacer() { - let mut delivery = Delivery::new(Duration::from_millis(500)); - let mut out = tokio::io::sink(); - delivery - .deliver(&frame(10_000_000, Timescale::MICRO), 0, &mut out) - .await - .unwrap(); - let now = tokio::time::Instant::now(); - delivery - .deliver(&frame(0, Timescale::MICRO), 1, &mut out) - .await - .unwrap(); - assert_eq!(now.elapsed(), Duration::ZERO); - delivery - .deliver(&frame(40_000, Timescale::MICRO), 1, &mut out) - .await - .unwrap(); - assert_eq!(now.elapsed(), Duration::from_millis(40)); - } /// A broadcast that returns but never serves its catalog gives up at the linger, /// rather than waiting on the catalog past it. diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 749f6034f58..d2943faf6b5 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -741,9 +741,11 @@ impl Export { /// /// Each [`Frame`] carries one slice of the PCR grid in `payload`: the clock /// packets that slice opens with, followed by the muxed bytes belonging to it. - /// It is stamped with the media time that slice starts at, so a transport can - /// pace delivery on the media clock, and `keyframe` marks the slice a video - /// keyframe begins in. The leading PAT/PMT rides on the first slice, and is + /// With a [delay](Self::with_delay), each slice is handed over at its time on the + /// jitter buffer's clock, which follows the source's, so a sink writes it as it + /// comes. It is stamped with the media time that slice starts at, for a transport + /// that stamps its own delivery, and `keyframe` marks the slice a video keyframe + /// begins in. The leading PAT/PMT rides on the first slice, and is /// re-emitted at video keyframes and periodically for mid-stream tune-in. /// Returns `None` when the broadcast ends. `duration` is always `None`: the /// muxer has no use for it. @@ -883,19 +885,24 @@ impl Export { } // 5. Once every track has drained, nothing more can ride the slots still open, so - // they go out. That's independent of the catalog: a retained track finishes while - // the broadcast stays live, and holding its tail until the catalog closed would - // strand it indefinitely. + // they go out: at their times on the clock, or at once without one. That's + // independent of the catalog: a retained track finishes while the broadcast stays + // live, and holding its tail until the catalog closed would strand it indefinitely. let drained = !self.tracks.is_empty() && self.tracks.values().all(|t| t.finished) && self.jitter.is_empty() && self.held.is_none(); if drained { - self.lay(None)?; + if self.slot_due().is_none() { + self.lay(None)?; + } if let Some(out) = self.queue.pop_front() { self.emitted_epoch = self.epoch; return Poll::Ready(Ok(Some(out))); } + if !self.schedule.is_empty() { + return Poll::Pending; + } // SI emission rides media frames, so a snapshot that arrived behind the // last one gets one trailing flush before the stream ends. if self.catalog.is_none() && !self.si_flushed { diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 03c86ba9ad3..8ff86abdc97 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -843,7 +843,11 @@ async fn export_pcr_wraps_below_zero_at_start() { .await .unwrap() .with_delay(RECORDING_MAX_AGE); - let frames = drain_frames(&mut exporter).await; + // Every slot up to the last decode time, the clock laying them out at theirs. + let mut frames = Vec::new(); + while let Ok(Ok(Some(frame))) = tokio::time::timeout(DRAIN, exporter.next()).await { + frames.push(frame); + } // A slot's clock packets lead the frame carrying that slot's bytes, and the // frame is stamped at the slot boundary so the caller's pacer delivers the @@ -5093,6 +5097,46 @@ async fn pcr_stamps_step_by_the_grid() { ); } +/// #2984: a TS byte stream carries no per-frame timing, so the slices have to go out at the +/// times they assert, not as fast as frames arrive. With a delay the export hands each slice +/// over at its time on its own clock, however the frames arrive: here a second at a time. +#[tokio::test(start_paused = true)] +async fn slices_go_out_on_the_clock() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut audio = aac_rendition(&mut broadcast, &mut catalog, "a.aac"); + let mut export = Export::new(crate::source::announced(&consumer)) + .await + .unwrap() + .with_delay(Duration::from_millis(1_500)); + let writer = tokio::spawn(async move { + for second in 0..4 { + for ms in (0..1_000).step_by(20) { + write_aac(&mut audio, second * 1_000 + ms); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + audio.finish().unwrap(); + (broadcast, catalog) + }); + let mut handed = Vec::new(); + while let Ok(Ok(Some(_))) = tokio::time::timeout(Duration::from_secs(5), export.next()).await { + handed.push(tokio::time::Instant::now()); + } + writer.await.unwrap(); + assert!(handed.len() > 100, "too few slices to judge: {}", handed.len()); + // Each slice follows the last by a slot, give or take the clock steering toward the + // source's, through to the end of the broadcast. + for w in handed.windows(2) { + let step = w[1] - w[0]; + assert!( + step.abs_diff(PCR_INTERVAL) < Duration::from_micros(100), + "a slice went out {step:?} after the last" + ); + } +} + /// The same positional property on a real reordered (B-frame) capture with a second /// rendition, where the exporter has no uniform cadence to lean on: frames arrive in /// decode order, and the two tracks advance the media clock at different rates. The diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index 784f2578493..c71d3dce881 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -58,22 +58,51 @@ pub(crate) enum Push { /// keep in step with, so a discontinuity there (a consumer skipping a stale group at join) /// keeps the clock. No two tracks are ever on different clocks. /// +/// The clock follows the source's: a source clock running slower than ours would make +/// every frame late in the end, and a faster one would hold more and more. So the clock +/// keeps the least slack a frame arrived with near the delay, by running its decode +/// timeline up to [`MAX_DRIFT`] faster or slower than ours, which also wears away a lead +/// the first frame anchored it with. +/// /// A zero delay holds nothing and drops nothing: each frame goes out as soon as it is /// read, ordered only among the frames read together. pub(crate) struct Buffer { delay: Duration, - /// The clock every frame is pushed under: its generation, and the arrival and decode - /// time of the frame that anchored it. - clock: Option<(u64, Instant, Timestamp)>, + /// The clock every frame is pushed under. + clock: Option, tracks: BTreeMap>, /// The latest deadline given out. horizon: Option, /// Whether a frame has gone out since the clock started. released: bool, + /// The least slack (deadline less arrival, in nanoseconds) a frame arrived with since the + /// clock was last steered, and the decode time it was steered at. + slack: Option<(i128, Timestamp)>, timer: Option>>, dropped: u64, } +/// How often the clock is steered, in decode time: long enough that the least slack covers +/// a group's worth of frames sent ahead by different amounts. +const STEER: Duration = Duration::from_secs(2); +/// How quickly the clock steers its slack back to the delay: it closes the gap over this +/// long, and a source drifting at `r` keeps it `r` times this off. +const RESPONSE: Duration = Duration::from_secs(60); +/// The fastest the clock runs off ours, in parts per billion: far past the 30 ppm a source's +/// system clock may be off (ISO 13818-1 2.4.2.1), with ours as far off the other way. +const MAX_DRIFT: i128 = 500_000; + +/// The decode timeline's mapping onto ours: where a frame decoding at `base` is due, less the +/// delay, and how much longer than the decode timeline ours runs. +#[derive(Clone, Copy)] +struct Clock { + generation: u64, + anchor: Instant, + base: Timestamp, + /// In parts per billion. + drift: i128, +} + struct Track { /// Frames in arrival order, each with its generation and deadline. queue: VecDeque<(u64, Instant, T)>, @@ -93,6 +122,7 @@ impl Buffer { tracks: BTreeMap::new(), horizon: None, released: false, + slack: None, timer: None, dropped: 0, } @@ -100,8 +130,14 @@ impl Buffer { /// Queue a frame, or drop it if it cannot make its deadline. pub fn push(&mut self, key: K, arrival: Arrival) -> Push { - let (mut generation, anchor, base) = *self.clock.get_or_insert((0, arrival.arrived, arrival.decode)); - let mut deadline = self.deadline(anchor, base, arrival.decode); + let clock = *self.clock.get_or_insert(Clock { + generation: 0, + anchor: arrival.arrived, + base: arrival.decode, + drift: 0, + }); + let mut generation = clock.generation; + let mut deadline = self.deadline(&clock, arrival.decode); // A track still on an older generation may be crossing the discontinuity another // track opened the newest one for. let jumped = self.tracks.get(&key).is_some_and(|track| { @@ -111,9 +147,17 @@ impl Buffer { if jumped && self.released { generation += 1; let after = self.horizon.and_then(|horizon| horizon.checked_sub(self.delay)); - let anchor = after.map_or(arrival.arrived, |after| after.max(arrival.arrived)); - self.clock = Some((generation, anchor, arrival.decode)); - deadline = self.deadline(anchor, arrival.decode, arrival.decode); + let clock = Clock { + generation, + anchor: after.map_or(arrival.arrived, |after| after.max(arrival.arrived)), + base: arrival.decode, + drift: clock.drift, + }; + self.clock = Some(clock); + self.slack = None; + deadline = self.deadline(&clock, arrival.decode); + } else if self.released && !self.delay.is_zero() { + self.steer(deadline, &arrival); } let track = self.tracks.entry(key).or_insert_with(|| Track { @@ -153,21 +197,49 @@ impl Buffer { }) } + /// Account for the slack a frame arrived with, and every [`STEER`] of decode time set the + /// clock's drift to close the gap between the least slack and the delay over [`RESPONSE`]. + fn steer(&mut self, deadline: Option, arrival: &Arrival) { + let (Some(clock), Some(deadline)) = (self.clock, deadline) else { + return; + }; + let slack = match deadline >= arrival.arrived { + true => (deadline - arrival.arrived).as_nanos() as i128, + false => -((arrival.arrived - deadline).as_nanos() as i128), + }; + let (least, since) = self.slack.get_or_insert((slack, arrival.decode)); + *least = (*least).min(slack); + if arrival.decode.as_nanos().saturating_sub(since.as_nanos()) < STEER.as_nanos() { + return; + } + let gap = *least - self.delay.as_nanos() as i128; + let drift = (-gap * 1_000_000_000 / RESPONSE.as_nanos() as i128).clamp(-MAX_DRIFT, MAX_DRIFT); + // Re-anchor where this frame decodes, so the new drift starts from there. + if let Some(anchor) = deadline.checked_sub(self.delay) { + self.clock = Some(Clock { + anchor, + base: arrival.decode, + drift, + ..clock + }); + } + self.slack = None; + } + /// When a frame decoding at `decode` goes out on the current clock, once there is one. pub fn at(&self, decode: Timestamp) -> Option { - let (_, anchor, base) = self.clock?; - self.deadline(anchor, base, decode) + self.deadline(self.clock.as_ref()?, decode) } - /// When a frame decoding at `decode` goes out on the clock anchored at `anchor` and - /// `base`, if any instant holds it. - fn deadline(&self, anchor: Instant, base: Timestamp, decode: Timestamp) -> Option { + /// When a frame decoding at `decode` goes out on `clock`, if any instant holds it. + fn deadline(&self, clock: &Clock, decode: Timestamp) -> Option { // Signed, since a frame can decode before the one that anchored the clock. - let offset = decode.as_nanos() as i128 - base.as_nanos() as i128 + self.delay.as_nanos() as i128; + let since = decode.as_nanos() as i128 - clock.base.as_nanos() as i128; + let offset = since + since * clock.drift / 1_000_000_000 + self.delay.as_nanos() as i128; let nanos = u64::try_from(offset.unsigned_abs()).ok()?; match offset >= 0 { - true => anchor.checked_add(Duration::from_nanos(nanos)), - false => anchor.checked_sub(Duration::from_nanos(nanos)), + true => clock.anchor.checked_add(Duration::from_nanos(nanos)), + false => clock.anchor.checked_sub(Duration::from_nanos(nanos)), } } @@ -232,6 +304,7 @@ impl Buffer { self.tracks.clear(); self.horizon = None; self.released = false; + self.slack = None; self.timer = None; } } @@ -432,6 +505,34 @@ mod tests { ); } + /// A source whose clock runs off ours, slower or faster, neither goes late nor piles up: + /// two hours at 200 ppm is 1.44 s, over ten times the delay. + #[tokio::test(start_paused = true)] + async fn the_clock_follows_a_drifting_source() { + for ppm in [200i128, -200] { + let start = Instant::now(); + let mut buffer = Buffer::new(DELAY); + let mut most = 0; + for k in 0..2 * 3_600 * 25u64 { + let decode = Duration::from_millis(k * 40); + let source = decode.as_nanos() as i128 * (1_000_000 + ppm) / 1_000_000; + let arrived = start + Duration::from_nanos(source as u64); + tokio::time::advance(arrived.saturating_duration_since(Instant::now())).await; + let frame = Arrival { + arrived, + decode: Timestamp::from_micros(decode.as_micros() as u64).unwrap(), + discontinuity: 0, + sync: true, + item: "v", + }; + assert_eq!(buffer.push(1, frame), Push::Queued, "{ppm} ppm: frame {k} was late"); + most = most.max(buffer.tracks[&1].queue.len()); + due(&mut buffer); + } + assert!(most <= 10, "{ppm} ppm: the buffer grew to {most} frames"); + } + } + /// Zero holds nothing: frames read together go out at once in decode order, and a /// frame behind the clock still goes out. #[tokio::test(start_paused = true)] diff --git a/rs/moq-mux/src/pace.rs b/rs/moq-mux/src/pace.rs index 83bae1786dd..c482bfa7462 100644 --- a/rs/moq-mux/src/pace.rs +++ b/rs/moq-mux/src/pace.rs @@ -13,10 +13,11 @@ use moq_net::Timestamp; /// carries no per-frame timing: [`ts::Export`](crate::container::ts::Export) /// slices its output on the PCR grid and stamps each slice at its slot boundary, /// and a caller that drains them on arrival collapses the clock into position -/// clusters no downstream stage can repair. "Deliver" is up to the transport: -/// a paced sink sleeps until the returned instant before writing, while a -/// transport with receiver-side buffering (e.g. SRT's TSBPD) stamps the payload -/// with it and sends immediately. +/// clusters no downstream stage can repair. (With a delay, the TS export already +/// hands each slice over at its time, on a clock that follows the source's.) +/// "Deliver" is up to the transport: a paced sink sleeps until the returned +/// instant before writing, while a transport with receiver-side buffering (e.g. +/// SRT's TSBPD) stamps the payload with it and sends immediately. /// /// `send_at = anchor + (ts - base)` maps the frame's media time onto the wall /// clock, where `base`'s media time and `anchor`'s wall instant were pinned to From 5e7650a85b130226ac0cf75fdfed8c8f633f6c23 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 11:37:49 -0700 Subject: [PATCH 20/40] fix(moq-mux): lay a padded TS slot alike in every exporter How many packets a slot carried and the PCR it opened with came from a running credit and a clock anchored at the exporter's first slot, so two exporters that joined at different times stamped and padded the same slot differently. Both are now functions of the slot alone: the packets the rate carries before it on the media timeline, and the time of its first byte at the rate. Two exporters padded to a rate now lay the same packets once the later one's first group is out, the continuity counter aside. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export_test.rs | 25 +++++++-- rs/moq-mux/src/container/ts/schedule.rs | 63 +++++++++++----------- 2 files changed, 52 insertions(+), 36 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 8ff86abdc97..0b5698fff17 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -5685,10 +5685,15 @@ async fn export_reordered( pps: &'static [u8], pyramid: &'static [u64], jitter: Option<(u64, Duration)>, + rate: Option, ) -> (Vec, Vec) { let delay = Duration::from_millis(500); let mut rig = Reordered::new(sps, pps, pyramid); - let mut a = rig.export(delay).await; + let padded = |export: Export| match rate { + Some(rate) => export.with_mux_rate(rate), + None => export, + }; + let mut a = padded(rig.export(delay).await); let start = tokio::time::Instant::now(); let mut b = None; let (mut out_a, mut out_b) = (Vec::new(), Vec::new()); @@ -5707,7 +5712,7 @@ async fn export_reordered( out_b.extend(poll_frames(b)); } if tick + 1 == JOIN { - b = Some(Export::new(rig.source.clone()).await.unwrap().with_delay(delay)); + b = Some(padded(Export::new(rig.source.clone()).await.unwrap().with_delay(delay))); } } rig.finish(); @@ -5865,6 +5870,18 @@ async fn reordered_video_loses_nothing_on_a_clean_path() { assert_on_time(&out); } +/// Two exporters of a broadcast padded to a multiplex rate, one joining a second late, lay the +/// same packets once the joiner's first group is out: how many packets a slot carries, the PCR +/// it opens with and which units ride it are functions of the media, not of when either +/// started. The continuity counter is the one exception ([`assert_only_continuity_differs`]). +#[tokio::test(start_paused = true)] +async fn late_join_at_a_rate_matches_a_running_exporter() { + let (runner, joiner) = export_reordered(SPS, PPS, &[], None, Some(2_000_000)).await; + let keyframes: Vec = joiner.iter().filter(|f| f.keyframe).map(|f| f.timestamp).collect(); + assert!(keyframes.len() > 2, "too few keyframes to judge: {keyframes:?}"); + assert_only_continuity_differs(&runner, &joiner, keyframes[1]); +} + /// With nothing declared, the reorder delay grows to the deepest reordering muxed so far. This /// is the one path where it depends on when an exporter joined: a joiner that has muxed only /// shallow groups runs a shallower clock than a runner that saw a deep one before the join, @@ -5873,7 +5890,7 @@ async fn reordered_video_loses_nothing_on_a_clean_path() { async fn undeclared_reorder_converges_after_its_deepest_reorder() { // The runner sees a pyramid in group 2; the joiner, arriving at group 5, first in group 7. // Group 9 repeats it on the settled delay. - let (runner, joiner) = export_reordered(SPS, PPS, &[2, 7, 9], None).await; + let (runner, joiner) = export_reordered(SPS, PPS, &[2, 7, 9], None, None).await; let keyframes: Vec = joiner.iter().filter(|f| f.keyframe).map(|f| f.timestamp).collect(); let deepest = reordered_at(8 * GOP); assert!( @@ -5901,7 +5918,7 @@ async fn undeclared_reorder_converges_after_its_deepest_reorder() { #[tokio::test(start_paused = true)] async fn jitter_published_after_the_tables_keeps_the_encoders_clock() { let jitter = Duration::from_millis(80); - let (runner, joiner) = export_reordered(SPS, PPS, &[], Some((2 * GOP, jitter))).await; + let (runner, joiner) = export_reordered(SPS, PPS, &[], Some((2 * GOP, jitter)), None).await; let keyframes: Vec = joiner.iter().filter(|f| f.keyframe).map(|f| f.timestamp).collect(); assert_only_continuity_differs(&runner, &joiner, keyframes[1]); diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index e8a4ed0c992..a54952b68a8 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -15,8 +15,9 @@ const _: () = assert!( .as_nanos() .is_multiple_of(PCR_INTERVAL.as_nanos()) ); -/// One packet in the fixed-point unit of the rate credit: one slot at `rate` bits per second -/// is exactly `rate` units, so no slot rounds on its own and the remainder carries over. +/// One packet in bits per second of slots: `rate` bits per second fills slot `index` up to +/// `index * rate / PACKET` packets since the start of the timeline, so no slot rounds on +/// its own. const PACKET: u64 = TsPacket::SIZE as u64 * 8 * SLOTS_PER_SECOND; /// The most of a packet a decoder buffer can receive: everything after its 4-byte header. const PAYLOAD: usize = TsPacket::SIZE - 4; @@ -35,12 +36,27 @@ pub(super) fn slot(nanos: u128) -> u128 { nanos / PCR_INTERVAL.as_nanos() } +/// One slot in system clock ticks. +const SLOT_TICKS: u128 = PCR_INTERVAL.as_nanos() * SYSTEM_CLOCK / 1_000_000_000; + /// The PCR, in system clock ticks, that opens slot `index`: one slot behind its boundary, /// so the slot's bytes are timed up to that boundary. It backs off through the 33-bit wrap /// rather than saturating, since the wire field is a circular clock. fn grid_pcr(index: u128) -> u128 { - let slot = PCR_INTERVAL.as_nanos() * SYSTEM_CLOCK / 1_000_000_000; - (index % PCR_WRAP * slot + PCR_WRAP - slot) % PCR_WRAP + (index % PCR_WRAP * SLOT_TICKS + PCR_WRAP - SLOT_TICKS) % PCR_WRAP +} + +/// How many packets `rate` bits per second carries before slot `index` on the media timeline. +fn packets_before(index: u128, rate: u64) -> u128 { + index * u128::from(rate) / u128::from(PACKET) +} + +/// The PCR that opens slot `index` at `rate`: the time of its first byte, the timeline's +/// packets before it at the rate, one slot behind like [`grid_pcr`]. It is a function of the +/// slot alone, so every exporter of a broadcast stamps a slot alike. +fn rate_pcr(index: u128, rate: u64) -> u128 { + let at = packets_before(index, rate) * TsPacket::SIZE as u128 * 8 * SYSTEM_CLOCK / u128::from(rate); + (at % PCR_WRAP + PCR_WRAP - SLOT_TICKS) % PCR_WRAP } /// A PID's buffers in a receiver, as the schedule respects them (ISO 13818-1 2.4.2). @@ -203,7 +219,8 @@ impl Slot { /// With a multiplex rate each slot carries exactly what the rate allows, padded with null /// packets, and a unit that is not complete by its due slot fails the export rather than /// arrive late. Each slot's PCR is the time of its own first byte at the rate, so it matches -/// its byte position exactly. Without a rate the stream is unpadded, its PCRs on the grid, +/// its byte position exactly, and how many packets a slot carries and the PCR it opens with +/// are functions of the slot alone, so two exporters of a broadcast lay a slot alike. Without a rate the stream is unpadded, its PCRs on the grid, /// and a unit still incomplete in its due slot goes out whole there. A rate that turns up /// mid-stream takes over from the next slot. pub(super) struct Schedule { @@ -213,8 +230,6 @@ pub(super) struct Schedule { window: u128, /// The multiplex rate, in bits per second. rate: Option, - /// The rate's allowance not yet spent, in [`PACKET`] units. - credit: u64, /// The next slot to lay out. next: Option, /// The latest due slot pushed: the clock runs at least that far. @@ -225,8 +240,6 @@ pub(super) struct Schedule { buffers: HashMap, /// Units sent into a decoder buffer: their PID, due slot and bytes held. decoding: VecDeque<(u16, u128, usize)>, - /// The PCR of the first slot at the rate, and the packets laid out since. - clock: Option<(u128, u128)>, } impl Schedule { @@ -235,23 +248,17 @@ impl Schedule { units: VecDeque::new(), window: slot(window.as_nanos()), rate: None, - credit: 0, next: None, horizon: None, last: None, buffers: HashMap::new(), decoding: VecDeque::new(), - clock: None, } } /// Pad to `rate` bits per second from the next slot on, or stop padding. pub fn set_rate(&mut self, rate: Option) { - if self.rate != rate { - self.rate = rate; - self.credit = 0; - self.clock = None; - } + self.rate = rate; } /// Hold the packets on `pid` to `buffer`. @@ -297,8 +304,6 @@ impl Schedule { self.next = None; self.horizon = None; self.last = None; - self.credit = 0; - self.clock = None; } /// The first slot `unit` may go out in: a window ahead of its due slot, or only the slot @@ -349,17 +354,11 @@ impl Schedule { if known.is_some_and(|known| index + self.window >= known) { return Ok(None); } - if self.next != Some(index) { - // A skipped gap carried no bytes, so the clock starts over past it. - self.clock = None; - } self.next = Some(index + 1); let (take, nulls, pcr) = match self.rate { Some(rate) => { - self.credit += rate; - let allowed = (self.credit / PACKET).max(1); - self.credit -= (self.credit / PACKET).min(allowed) * PACKET; + let allowed = (packets_before(index + 1, rate) - packets_before(index, rate)).max(1); let take = self.admit(index, allowed as usize - 1); if let Some(unit) = self.missed(index, &take) { anyhow::bail!( @@ -368,10 +367,7 @@ impl Schedule { ); } let media: usize = take.iter().sum(); - let (base, laid) = self.clock.get_or_insert((grid_pcr(index), 0)); - let pcr = (*base + *laid * TsPacket::SIZE as u128 * 8 * SYSTEM_CLOCK / u128::from(rate)) % PCR_WRAP; - *laid += u128::from(allowed); - (take, allowed as usize - 1 - media, pcr) + (take, allowed as usize - 1 - media, rate_pcr(index, rate)) } None => { let mut take = self.admit(index, usize::MAX); @@ -641,8 +637,10 @@ mod tests { assert_eq!(sent, [(39, 2)]); } - /// Each PCR is the time of its slot's first byte at the rate, so a rate that does not - /// divide into whole packets per slot moves it off the grid by less than a packet. + /// Each PCR is the time of its slot's first byte at the rate, to within a tick of the + /// system clock, so a rate that does not divide into whole packets per slot moves it off + /// the grid by less than a packet. Both are functions of the slot alone, so a schedule + /// started later lays the same slot alike. #[test] fn the_pcr_is_its_byte_position_at_the_rate() { let mut schedule = Schedule::new(Duration::ZERO); @@ -658,7 +656,8 @@ mod tests { while let Some(slot) = schedule.next(None).unwrap() { let base = *first.get_or_insert(u128::from(slot.pcr)); let expected = base + laid * TsPacket::SIZE as u128 * 8 * SYSTEM_CLOCK / u128::from(rate); - assert_eq!(u128::from(slot.pcr), expected, "slot {}", slot.index); + assert!(u128::from(slot.pcr).abs_diff(expected) <= 1, "slot {}", slot.index); + assert_eq!(u128::from(slot.pcr), rate_pcr(slot.index, rate), "slot {}", slot.index); let grid = grid_pcr(slot.index); let packet = TsPacket::SIZE as u128 * 8 * SYSTEM_CLOCK / u128::from(rate); assert!(grid - u128::from(slot.pcr) < packet, "slot {} strays a packet off the grid", slot.index); From 2ef0d8ada642ce72dd08207280922e30bf34a48b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 11:45:27 -0700 Subject: [PATCH 21/40] test(ts): grade a broadcast-sized CPB and PCR accuracy just test ts --hrd encodes the 1080p clip moq-dev/moq#4645 graded with: a 9 Mbit NAL HRD kept near full by noise, so the source sends pictures most of a second ahead and loads the decoder buffer the way a contribution encoder does. CI runs it under the strict T-STD gate at the default delay. --delay passes the exporter's. With the rate known, the harness also runs TSDuck's pcrverify and fails on any PCR more than 500 ns off its byte position, which pcr-schedule's packet of slack let through. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/interop.yml | 9 ++++++ test/justfile | 3 +- test/ts/README.md | 20 +++++++++---- test/ts/run.sh | 53 +++++++++++++++++++++++++++++++++-- 4 files changed, 76 insertions(+), 9 deletions(-) diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index 38629e72313..2c4f8affffe 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -137,6 +137,15 @@ jobs: env: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + # 1080p video filling a broadcast-sized 9 Mbit CPB, at the default delay: the + # source sends pictures most of a second ahead of their decode time, and the + # export has to as well without overflowing the receiver's buffers. + - name: TS broadcast-sized CPB + run: nix develop --command just test ts --hrd + shell: bash -leo pipefail {0} + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + # The same round-trip over an open-GOP clip: every leading picture after a # recovery point must come back, in decode order. - name: TS open GOP diff --git a/test/justfile b/test/justfile index b7267ec9270..279564f6283 100644 --- a/test/justfile +++ b/test/justfile @@ -82,7 +82,8 @@ drill-sensitivity *args: # (skip the round-trip), `--strict` (fail on shape warnings), `--source file.ts` # (use a real capture), `--live` (grade PCR release timing and byte position off # the pipe instead, which a capture cannot carry; nightly runs this arm), -# `--open-gop` (an open-GOP clip whose leading pictures must survive). See +# `--open-gop` (an open-GOP clip whose leading pictures must survive), `--hrd` +# (1080p video filling a broadcast-sized CPB), `--delay 1s` (the exporter's). See # ts/README.md. # Check the subscriber's MPEG-TS output for IRD compliance. diff --git a/test/ts/README.md b/test/ts/README.md index cc8ea07ade5..6a5955e3489 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -31,6 +31,8 @@ just test ts --with-eit # add a synthetic EPG first, report which SI sur just test ts --live # grade PCR release timing off the live pipe just test ts --pair # two exporters of one broadcast, grade table anchoring just test ts --open-gop # open-GOP clip; its leading pictures must survive +just test ts --hrd # 1080p video filling a broadcast-sized 9 Mbit CPB +just test ts --delay 1s # pass the exporter's --delay ``` `--live` swaps the analyzer, not the rig: the same round-trip runs, but the @@ -42,7 +44,10 @@ The default arm runs `pcr-timing.py` over its capture too, after `compliance.py` for the PCR checks `compliance.py` leaves to it: the value interval (hard), and whether the bytes between consecutive PCRs are the ones the mux rate implies ([`pcr-schedule`](#byte-schedule), which gates whenever the rate is known, as the -generated clip's is). +generated clip's is). With the rate known it also runs TSDuck's `pcrverify` past +the first three seconds, failing on any PCR more than 500 ns off its byte position +at the rate: the PCR_accuracy_error an IRD or a TR 101 290 probe reports, which +`pcr-schedule`'s packet of slack lets through. The live arm passes only when the grader's verdict *and* the publisher's exit status are clean. The grader can only speak for what reached it, and the sample floor @@ -331,7 +336,11 @@ unpadded; the report counts those intervals as `unpadded_lead`. At the default 10 Mb/s the generated clip compresses to almost nothing, so padding dominates and no keyframe outgrows its 31 kB slot. At 2 Mb/s its 13-19 kB keyframes outgrow a 6 kB slot, so the exporter has to spread each over the slots before its -DTS; CI runs that too (`just test ts --bitrate 2000000`). A real constant-rate +DTS; CI runs that too (`just test ts --bitrate 2000000`). `--hrd` goes further: a +1080p encode with a 9 Mbit NAL HRD kept near full by noise, the shape of a +contribution encoder's output, which sends pictures most of a second ahead of their +decode time and loads the decoder buffer past 60 % at the default delay (the recipe +moq-dev/moq#4645 graded with; CI runs it too). A real constant-rate capture discriminates further. A 60 s cut of a 9.95 Mb/s broadcast clip round-tripped through the harness before the export kept a schedule came back with a median of 1,316 B between PCRs against 31,081 B nominal and 3.1 % of intervals @@ -547,9 +556,10 @@ exporter re-emits SI on its own repetition cadence rather than the source's. ## CI -`.github/workflows/interop.yml` runs `just test ts`, `just test ts --open-gop`, -`just test ts-eit`, and `just test ts-tstd` after the interop matrix (nightly, on -demand, and on PRs touching `test/ts/`). +`.github/workflows/interop.yml` runs `just test ts`, `just test ts --bitrate +2000000`, `just test ts --hrd`, `just test ts --open-gop`, `just test ts-eit`, and +`just test ts-tstd` after the interop matrix (nightly, on demand, and on PRs +touching `test/ts/`). `ts-eit` is `eit-roundtrip.sh`: it builds the sparse-schedule and pending-version fixtures from a generated clip, round-trips them through a relay, and censuses the capture, so a break in the generators or in the SI diff --git a/test/ts/run.sh b/test/ts/run.sh index 9e03aaddcbe..6fe059d1655 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -18,6 +18,8 @@ # ./run.sh --live # grade PCR release timing off the live pipe # ./run.sh --pair # two exporters of one broadcast, grade table anchoring # ./run.sh --open-gop # open-GOP clip; its leading pictures must survive +# ./run.sh --hrd # 1080p video filling a broadcast-sized 9 Mbit CPB +# ./run.sh --delay 1s # pass the exporter's --delay # `--live` swaps the analyzer, not the rig. compliance.py grades a captured file # on the stream's own PCR clock, which is the right basis for the IRD model it @@ -61,6 +63,8 @@ WITH_EIT="" # add a synthetic EPG to the source and report which SI survived LIVE="" # grade the exporter's stdout as it arrives, rather than a capture PAIR="" # subscribe twice and grade the two captures against each other OPEN_GOP="" # publish open GOP with leading pictures, and grade them through the round-trip +HRD="" # publish video that fills a broadcast-sized CPB, as a contribution encoder does +EXPORT=() # extra `export ts` flags (--delay) # How far into the run the second subscriber joins. A late join is the point: two # exporters started together can share a cadence by starting together, which is # exactly the thing under test. @@ -122,6 +126,14 @@ while [[ $# -gt 0 ]]; do OPEN_GOP=1 shift ;; + --hrd) + HRD=1 + shift + ;; + --delay) + EXPORT=(--delay "$2") + shift 2 + ;; *) PASSTHRU+=("$1") shift @@ -149,6 +161,10 @@ if [[ -n "$PAIR" ]]; then fi # open-gop.py grades a capture against its source, which only the plain round-trip keeps. +if [[ -n "$HRD" && -n "$OPEN_GOP$SOURCE$ANALYZE_ONLY" ]]; then + echo "error: --hrd generates its own clip and cannot be combined with --open-gop, --source, or --analyze-only" >&2 + exit 1 +fi if [[ -n "$OPEN_GOP" && -n "$ANALYZE_ONLY$LIVE$PAIR" ]]; then echo "error: --open-gop cannot be combined with --analyze-only, --live, or --pair" >&2 exit 1 @@ -237,6 +253,17 @@ if [[ -n "$SOURCE" ]]; then sed 's/^/ tsp: /' "$HARNESS_RUN/tsp-cut.log" >&2 || true exit 1 } +elif [[ -n "$HRD" ]]; then + echo "### generating ~${DURATION}s 1080p clip filling a 9 Mbit CPB with ffmpeg" + # A contribution encoder's shape: CBR video whose HRD declares the whole buffer, kept + # near full by noise, so the source sends pictures most of a second ahead of their + # decode time and the export has to as well (moq-dev/moq#4645). + ffmpeg -y -hide_banner -loglevel error \ + -f lavfi -i "testsrc2=size=1920x1080:rate=25,noise=alls=12:allf=t" \ + -t "$DURATION" -an \ + -c:v libx264 -profile:v high -level 4.0 -preset veryfast -pix_fmt yuv420p \ + -x264-params "keyint=25:min-keyint=25:scenecut=0:nal-hrd=cbr" -b:v 9M -maxrate 9M -bufsize 9M \ + -f mpegts -muxrate "$BITRATE" -pcr_period 20 -pes_payload_size 0 "$SRC_TS" else echo "### generating ~${DURATION}s broadcast-like ${OPEN_GOP:+open-GOP }clip with ffmpeg" # CBR with a 20 ms PCR, like a contribution feed. Not cosmetic: `regulate` @@ -318,7 +345,7 @@ grade_live() { # status we are here to record. set +e timeout -k 3 $((DURATION + 20)) \ - "$MOQ" --connect "$URL" --broadcast "$BROADCAST" export ts 2>"$HARNESS_RUN/sub.log" | + "$MOQ" --connect "$URL" --broadcast "$BROADCAST" export ts ${EXPORT[@]+"${EXPORT[@]}"} 2>"$HARNESS_RUN/sub.log" | python3 "$DIR/pcr-timing.py" --live --seconds "$DURATION" --release-pct-max 1 $STRICT \ ${SCHEDULE[@]+"${SCHEDULE[@]}"} ${PASSTHRU[@]+"${PASSTHRU[@]}"} >"$HARNESS_RUN/timing.out" 2>&1 printf '%s\n' "${PIPESTATUS[0]} ${PIPESTATUS[1]}" >"$HARNESS_RUN/timing.rc" @@ -327,7 +354,7 @@ grade_live() { # shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' capture() { timeout -k 3 $((DURATION + 20)) \ - "$MOQ" --connect "$URL" --broadcast "$BROADCAST" export ts >"$SUB_TS" 2>"$HARNESS_RUN/sub.log" + "$MOQ" --connect "$URL" --broadcast "$BROADCAST" export ts ${EXPORT[@]+"${EXPORT[@]}"} >"$SUB_TS" 2>"$HARNESS_RUN/sub.log" } # shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' @@ -338,7 +365,7 @@ capture_b() { # the media, because it has no shared history to derive them from. sleep "$PAIR_JOIN" timeout -k 3 $((DURATION + 20)) \ - "$MOQ" --connect "$URL" --broadcast "$BROADCAST" export ts >"$SUB_B_TS" 2>"$HARNESS_RUN/sub-b.log" + "$MOQ" --connect "$URL" --broadcast "$BROADCAST" export ts ${EXPORT[@]+"${EXPORT[@]}"} >"$SUB_B_TS" 2>"$HARNESS_RUN/sub-b.log" } if [[ -n "$LIVE" ]]; then @@ -529,3 +556,23 @@ if ! python3 "$DIR/pcr-timing.py" "$SUB_TS" ${SCHEDULE[@]+"${SCHEDULE[@]}"} $STR dump_logs exit 1 fi + +# pcr-schedule allows each PCR a packet of slack, but an IRD or a TR 101 290 probe holds +# every PCR to within 500 ns of its byte position at the mux rate (PCR_accuracy_error). +# TSDuck's pcrverify measures exactly that. The first seconds are skipped: the export +# starts unpadded until the importer has measured the source's rate and recorded it. +if [[ ${#SCHEDULE[@]} -gt 0 ]]; then + echo + echo "### PCR accuracy (tsp -P pcrverify, +/-500 ns at ${BITRATE} b/s)" + SKIP=$((3 * BITRATE / 8 / 188)) + tsp -I file "$SUB_TS" -P skip "$SKIP" -P pcrverify --bitrate "$BITRATE" --absolute --jitter-max 13 \ + -O drop >"$HARNESS_RUN/pcrverify.log" 2>&1 || true + VERDICT=$(grep -E "PCR OK, " "$HARNESS_RUN/pcrverify.log" | tail -1) + echo " ${VERDICT:-no verdict}" + if [[ ! "$VERDICT" =~ ([0-9,]+)\ PCR\ OK,\ 0\ with ]]; then + echo >&2 + echo "error: PCRs stray from their byte position (see $HARNESS_RUN/pcrverify.log)" >&2 + dump_logs + exit 1 + fi +fi From a8b1517a5f7fd0e40874f51ad579dfb2f3510c2f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 11:47:24 -0700 Subject: [PATCH 22/40] quest(m1): record the T-STD measurement after t0ms's grading Co-Authored-By: Claude Opus 5.5 --- quest/m1/tstd/README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/quest/m1/tstd/README.md b/quest/m1/tstd/README.md index 7bd347c82f6..5ad08a3123d 100644 --- a/quest/m1/tstd/README.md +++ b/quest/m1/tstd/README.md @@ -16,11 +16,14 @@ Padding, pacing, and muxing all assume a fixed delay, so the export gets one first. t0ms is testing whether T-STD compliance is feasible at all; record the result here. If it isn't, re-plan this line. -Measured (2026-09-30, #4645): with the fixed-delay jitter buffer and the -constant-rate schedule, a clean-path round trip of the generated clip passes -the strict T-STD check at 10 Mb/s and at 2 Mb/s, where every keyframe outgrows -a PCR interval. A burst that does not fit the delay fails the export rather -than overrun the rate. +Measured (2026-10-01, #4645): with the fixed-delay jitter buffer, per-PID +admission against each PID's T-STD buffers, and PCRs at their byte position, a +clean-path round trip passes the strict T-STD check and TSDuck's ±500 ns +pcrverify at the default 500 ms delay, for the generated clip at 10 Mb/s and +2 Mb/s and for a 1080p encode filling a 9 Mbit CPB (`just test ts --hrd`, after +t0ms's recipe). A unit that cannot arrive by its DTS fails the export rather +than arrive late. t0ms's broadcast capture (PAFF H.264, MP2, AC-3, teletext) +is still to be re-graded on this head. This README owns the end-to-end proof: the #4613 netem rig (10% loss, a real ~10 Mb/s broadcast TS) passes the strict T-STD check, and the recipe runs From 49efbc9a1ea525b35a9b3c5cb609fa71c3a315dc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 11:51:00 -0700 Subject: [PATCH 23/40] style(moq-mux): format Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/codec/h265/mod.rs | 9 +++- rs/moq-mux/src/container/ts/export.rs | 27 +++++++++-- rs/moq-mux/src/container/ts/export_test.rs | 56 ++++++++++++++++++---- rs/moq-mux/src/container/ts/schedule.rs | 11 ++++- test/ts/README.md | 2 +- 5 files changed, 86 insertions(+), 19 deletions(-) diff --git a/rs/moq-mux/src/codec/h265/mod.rs b/rs/moq-mux/src/codec/h265/mod.rs index fe34194cf3b..be53ae8bac0 100644 --- a/rs/moq-mux/src/codec/h265/mod.rs +++ b/rs/moq-mux/src/codec/h265/mod.rs @@ -236,7 +236,14 @@ pub(crate) fn sps_reorder(nal: &[u8]) -> Option { pub(crate) fn sps_hrd(nal: &[u8]) -> Option { let sps = SpsNALUnit::parse(&mut &nal[..]).ok()?; let highest = sps.rbsp.sps_max_sub_layers_minus1 as usize; - let hrd = sps.rbsp.vui_parameters.as_ref()?.vui_timing_info.as_ref()?.hrd_parameters.as_ref()?; + let hrd = sps + .rbsp + .vui_parameters + .as_ref()? + .vui_timing_info + .as_ref()? + .hrd_parameters + .as_ref()?; let layer = hrd.sub_layers.get(highest).or(hrd.sub_layers.last())?; // The NAL HRD's schedules come first when both are present. let schedules = usize::try_from(layer.cpb_cnt_minus1).ok()? + 1; diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index d2943faf6b5..cc14c03291a 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -258,7 +258,10 @@ impl Track { framing: catalog::Framing::Pes, .. } - ) && self.descriptors.iter().any(|descriptor| descriptor.tag == AC3_DESCRIPTOR) + ) && self + .descriptors + .iter() + .any(|descriptor| descriptor.tag == AC3_DESCRIPTOR) } /// The receiver's buffers for the track's PID, if the T-STD gives it any. @@ -391,7 +394,10 @@ impl Timing { None => self.framerate.map_or(0, |fps| (90_000.0 / fps).ceil() as u64), }; let delay = u64::from(declared.depth).saturating_mul(period); - let lookahead = 1u64.checked_shl(declared.depth).unwrap_or(u64::MAX).saturating_mul(period); + let lookahead = 1u64 + .checked_shl(declared.depth) + .unwrap_or(u64::MAX) + .saturating_mul(period); (delay.min(MAX_REORDER), lookahead.min(MAX_REORDER)) } } @@ -1642,7 +1648,14 @@ impl Export { } /// Queue one unit of `name`'s packets on `pid`, decoding at `decode`, on the [`Schedule`]. - fn push_unit(&mut self, name: &str, pid: u16, decode: Timestamp, out: Vec, keyframe: bool) -> anyhow::Result<()> { + fn push_unit( + &mut self, + name: &str, + pid: u16, + decode: Timestamp, + out: Vec, + keyframe: bool, + ) -> anyhow::Result<()> { let track = self.tracks.get(name).context("missing track")?; if let Some(buffer) = track.buffer() { self.schedule.set_buffer(pid, buffer); @@ -2424,8 +2437,12 @@ impl DecodeClock { /// The HRD declared by the first SPS in a video rendition's avcC/hvcC. fn declared_hrd(stream_type: StreamType, description: &[u8]) -> Option { match stream_type { - StreamType::H264 => crate::codec::h264::sps_hrd(crate::codec::h264::Avcc::parse(description).ok()?.sps.first()?), - StreamType::H265 => crate::codec::h265::sps_hrd(crate::codec::h265::Hvcc::parse(description).ok()?.sps.first()?), + StreamType::H264 => { + crate::codec::h264::sps_hrd(crate::codec::h264::Avcc::parse(description).ok()?.sps.first()?) + } + StreamType::H265 => { + crate::codec::h265::sps_hrd(crate::codec::h265::Hvcc::parse(description).ok()?.sps.first()?) + } _ => None, } } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 0b5698fff17..b10fa63a696 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -695,7 +695,11 @@ fn video_pes_timing(ts: &[u8]) -> Vec<(u64, u64)> { if packet[1] & 0x40 == 0 || packet[3] & 0x10 == 0 { continue; } - let start = 4 + if packet[3] & 0x20 != 0 { usize::from(packet[4]) + 1 } else { 0 }; + let start = 4 + if packet[3] & 0x20 != 0 { + usize::from(packet[4]) + 1 + } else { + 0 + }; let Some(pes) = packet.get(start..).filter(|pes| pes.len() >= 19) else { continue; }; @@ -727,7 +731,11 @@ async fn export_decodes_where_the_source_did() { let ts = drain(consumer).await; let exported = video_pes_timing(&ts); - assert!(exported.len() > 100, "expected the full feed, got {} frames", exported.len()); + assert!( + exported.len() > 100, + "expected the full feed, got {} frames", + exported.len() + ); // The capture ends mid-reorder, so the last frame's slot is one the encoder filled with a // picture past the end. Timestamps cross MoQ in microseconds, so each is within a tick. for (i, &(pts, dts)) in exported[..exported.len() - 1].iter().enumerate() { @@ -735,7 +743,10 @@ async fn export_decodes_where_the_source_did() { .range(pts - 1..=pts + 1) .next() .unwrap_or_else(|| panic!("frame {i} presented at {pts} is not in the source")); - assert!(dts.abs_diff(want) <= 2, "frame {i} decodes at {dts}, the source at {want}"); + assert!( + dts.abs_diff(want) <= 2, + "frame {i} decodes at {dts}, the source at {want}" + ); } } @@ -1104,7 +1115,11 @@ async fn export_splits_a_multi_frame_ac3_pes() { if pid != AC3_PID || packet[3] & 0x10 == 0 { continue; } - let start = 4 + if packet[3] & 0x20 != 0 { usize::from(packet[4]) + 1 } else { 0 }; + let start = 4 + if packet[3] & 0x20 != 0 { + usize::from(packet[4]) + 1 + } else { + 0 + }; let body = &packet[start..]; if packet[1] & 0x40 != 0 { let b = &body[9..14]; @@ -1121,7 +1136,11 @@ async fn export_splits_a_multi_frame_ac3_pes() { assert_eq!(pes.len(), 3, "one PES a sync frame: {pes:?}"); for (k, &(pts, bytes)) in pes.iter().enumerate() { assert_eq!(bytes, 128, "frame {k} carries one sync frame"); - assert_eq!(pts, pes[0].0 + k as u64 * 2_880, "frame {k} is presented 32 ms after the last"); + assert_eq!( + pts, + pes[0].0 + k as u64 * 2_880, + "frame {k} is presented 32 ms after the last" + ); } } @@ -4012,7 +4031,11 @@ fn assert_on_time(frames: &[Frame]) -> usize { if pid == 0x1fff || packet[3] & 0x10 == 0 { continue; } - let start = 4 + if packet[3] & 0x20 != 0 { usize::from(packet[4]) + 1 } else { 0 }; + let start = 4 + if packet[3] & 0x20 != 0 { + usize::from(packet[4]) + 1 + } else { + 0 + }; let pes = &packet[start.min(188)..]; if packet[1] & 0x40 != 0 && pes.len() >= 14 && pes[..3] == [0, 0, 1] && pes[7] & 0x80 != 0 { let stamp = |b: &[u8]| { @@ -4022,7 +4045,11 @@ fn assert_on_time(frames: &[Frame]) -> usize { | i128::from(b[3]) << 7 | i128::from(b[4] >> 1) }; - let decode = if pes[7] & 0x40 != 0 { stamp(&pes[14..19]) } else { stamp(&pes[9..14]) }; + let decode = if pes[7] & 0x40 != 0 { + stamp(&pes[14..19]) + } else { + stamp(&pes[9..14]) + }; let decode = first + ((decode - first as i128).rem_euclid(WRAP)) as f64; units.push((pid, decode, at)); } else if let Some(unit) = units.iter_mut().rev().find(|unit| unit.0 == pid) { @@ -4033,7 +4060,10 @@ fn assert_on_time(frames: &[Frame]) -> usize { let mut judged = 0; for &(pid, decode, end) in &units { if let Some(previous) = last.insert(pid, decode) { - assert!(decode >= previous, "PID {pid} decodes at {decode} after a unit at {previous}"); + assert!( + decode >= previous, + "PID {pid} decodes at {decode} after a unit at {previous}" + ); } let Some(arrived) = time(end + 1) else { continue }; assert!( @@ -5529,7 +5559,10 @@ async fn export_pcr_is_its_byte_position_at_the_rate() { for &(index, pcr) in &pcrs { let at = (index - first) as u128 * 188 * 8 * 27_000_000 / 1_000_000; let off = (u128::from(pcr - base)).abs_diff(at); - assert!(off <= 1, "the PCR at packet {index} is {off} ticks off its byte position"); + assert!( + off <= 1, + "the PCR at packet {index} is {off} ticks off its byte position" + ); } } @@ -5929,7 +5962,10 @@ async fn jitter_published_after_the_tables_keeps_the_encoders_clock() { for (i, step) in decode.windows(2).map(|w| w[1] - w[0]).enumerate() { assert_eq!(step, FRAME_TICKS, "frame {i} does not decode one period after the last"); } - for (pts, dts) in timing.into_iter().filter(|(pts, _)| (pts / FRAME_TICKS).is_multiple_of(GOP)) { + for (pts, dts) in timing + .into_iter() + .filter(|(pts, _)| (pts / FRAME_TICKS).is_multiple_of(GOP)) + { assert_eq!(dts, Some(pts - FRAME_TICKS), "a keyframe decodes a period early"); } assert_decodes_after_the_clock(&runner); diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index a54952b68a8..9f300afaa8c 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -294,7 +294,10 @@ impl Schedule { /// Whether nothing is left to lay out: every unit sent, and the clock past its decode time. pub fn is_empty(&self) -> bool { - self.units.is_empty() && self.horizon.is_none_or(|horizon| self.next.is_some_and(|next| next > horizon)) + self.units.is_empty() + && self + .horizon + .is_none_or(|horizon| self.next.is_some_and(|next| next > horizon)) } /// Drop everything queued and start the grid afresh. @@ -660,7 +663,11 @@ mod tests { assert_eq!(u128::from(slot.pcr), rate_pcr(slot.index, rate), "slot {}", slot.index); let grid = grid_pcr(slot.index); let packet = TsPacket::SIZE as u128 * 8 * SYSTEM_CLOCK / u128::from(rate); - assert!(grid - u128::from(slot.pcr) < packet, "slot {} strays a packet off the grid", slot.index); + assert!( + grid - u128::from(slot.pcr) < packet, + "slot {} strays a packet off the grid", + slot.index + ); laid += 1 + (slot.packets.len() / TsPacket::SIZE + slot.nulls) as u128; } } diff --git a/test/ts/README.md b/test/ts/README.md index 6a5955e3489..ad5d2367c25 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -46,7 +46,7 @@ whether the bytes between consecutive PCRs are the ones the mux rate implies ([`pcr-schedule`](#byte-schedule), which gates whenever the rate is known, as the generated clip's is). With the rate known it also runs TSDuck's `pcrverify` past the first three seconds, failing on any PCR more than 500 ns off its byte position -at the rate: the PCR_accuracy_error an IRD or a TR 101 290 probe reports, which +at the rate: the `PCR_accuracy_error` an IRD or a TR 101 290 probe reports, which `pcr-schedule`'s packet of slack lets through. The live arm passes only when the grader's verdict *and* the publisher's exit status From acc5fdd858fe609038065f5ed42ebe08a41dcd80 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 13:32:44 -0700 Subject: [PATCH 24/40] fix(moq-mux): only a declared restart breaks the TS program clock The consumer counted a skipped group and a declared marker alike, so a group skipped after output started opened a jitter generation, rewound the mux and flagged a PCR break although the timeline carried on. The consumer now also counts restarts (declared markers) alone, and the TS export keys its generations and decode clock on those: a skipped group keeps the clock, and anything it made late is dropped like any other late frame. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/consumer.rs | 38 ++++++++++++++ rs/moq-mux/src/container/source.rs | 14 ++--- rs/moq-mux/src/container/ts/export.rs | 27 +++++----- rs/moq-mux/src/container/ts/export_test.rs | 60 ++++++++++++++++++++++ rs/moq-mux/src/jitter.rs | 56 ++++++++++---------- 5 files changed, 145 insertions(+), 50 deletions(-) diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 35ebe48ca31..66e61a015c6 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -96,6 +96,9 @@ pub struct Consumer { // Increments on a declared marker, an unproven delivered hole, and a latency skip. discontinuity: u64, + // Increments on a declared marker only: the publisher restarted its timeline. + restarts: u64, + // Exclusive audio endpoint delivered before terminal codec packets. end: Option, } @@ -141,6 +144,7 @@ impl Consumer { floor: None, presented_end: None, discontinuity: 0, + restarts: 0, end: None, } } @@ -154,6 +158,13 @@ impl Consumer { self.discontinuity } + /// A counter that increments only when the publisher declared a marker group: its + /// timeline restarted. A hole or a latency skip moves the playhead + /// ([`Self::discontinuity`]) but the timeline carries on, so this stays put. + pub(crate) fn restarts(&self) -> u64 { + self.restarts + } + /// The exclusive audio endpoint delivered before terminal codec packets. pub fn end(&self) -> Option { self.end @@ -301,6 +312,7 @@ impl Consumer { self.note_group_edge(); if marker { self.bump_playhead(); + self.restarts += 1; } return Poll::Ready(Ok(Some(Event::GroupEnd))); } @@ -397,6 +409,7 @@ impl Consumer { if hole || had_marker { self.bump_playhead(); } + self.restarts += u64::from(had_marker); let new_current = self.pending.front().map(|g| g.sequence).unwrap(); tracing::debug!(old = self.current, new = new_current, "skipping slow groups"); @@ -910,6 +923,31 @@ mod tests { assert_eq!(consumer.discontinuity(), 1); } + /// A missing group moves the playhead, but the timeline carries on: it is not a restart. + /// A declared marker is both. + #[tokio::test] + async fn only_a_marker_restarts_the_timeline() { + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let consumer_track = + track.subscribe(moq_net::track::Subscription::default().with_max_age(Duration::from_secs(2))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Audio)); + + write_group(&mut track, 0, &[ts(0)]); + // Group 1 never arrives. + write_group(&mut track, 2, &[ts(1_000_000)]); + write_marker_group(&mut track, 3, ts(1_000_000)); + write_group(&mut track, 4, &[ts(2_000_000)]); + track.finish().unwrap(); + + assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); + assert_eq!(consumer.discontinuity(), 1, "the hole moves the playhead"); + assert_eq!(consumer.restarts(), 0, "but the timeline carried on"); + assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(2_000_000)); + assert_eq!(consumer.discontinuity(), 2); + assert_eq!(consumer.restarts(), 1, "the marker declares a restart"); + } + #[tokio::test] async fn empty_groups_mean_nothing() { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index 07a7ba13e4f..41fb2ce40e9 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -184,16 +184,16 @@ impl ExportSource { self.description.as_ref() } - /// The underlying consumer's playhead generation, or 0 until the + /// How many times the publisher declared its timeline restarted, or 0 until the /// subscription resolves. /// - /// See [`Consumer::discontinuity`]. Sample it alongside each frame returned by - /// [`poll_read`](Self::poll_read): the frame read while the counter changes is - /// the first after a playhead event, so anything anchored on the media clock (a - /// repetition cadence, a clock grid, a pacer) has to re-anchor to it. - pub fn discontinuity(&self) -> u64 { + /// See [`Consumer::restarts`]. Sample it alongside each frame returned by + /// [`poll_read`](Self::poll_read): the frame read while the counter changes is the + /// first on the new timeline, so anything anchored on the media clock has to + /// re-anchor to it. A skipped group is not a restart: the timeline carries on. + pub fn restarts(&self) -> u64 { match &self.state { - SourceState::Active(consumer) => consumer.discontinuity(), + SourceState::Active(consumer) => consumer.restarts(), _ => 0, } } diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index cc14c03291a..b1bf7642a77 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -167,7 +167,8 @@ pub struct Export { /// A frame read from its source. struct Pending { frame: Frame, - discontinuity: u64, + /// How many times the source had restarted its timeline when the frame was read. + restart: u64, /// The earliest the frame could have arrived: when its source was last found empty. arrived: web_async::time::Instant, } @@ -191,8 +192,8 @@ struct Track { /// no later than it is read; the jitter buffer judges it on the earlier bound, so a caller /// that polls late (a sink sleeping to pace its writes) does not make it late. empty: web_async::time::Instant, - /// The source's discontinuity counter the decode clock (`clock`, `timeline`) runs under. - discontinuity: u64, + /// The source's restart counter the decode clock runs under. + restart: u64, finished: bool, pid: u16, kind: Kind, @@ -214,7 +215,7 @@ impl Track { source, pending: None, empty: web_async::time::Instant::now(), - discontinuity: 0, + restart: 0, finished: false, pid: 0, kind, @@ -231,10 +232,10 @@ impl Track { /// reordered video arrives in decode order with PTS 0, 120, 40, 80, so a PTS deadline would /// strand a B-frame behind its reference or send it first. fn queue(&mut self, name: &str, pending: Pending, jitter: &mut jitter::Buffer) { - if pending.discontinuity != self.discontinuity { + if pending.restart != self.restart { // The source may have restarted its timeline, so the decode clock restarts too. self.release(name, jitter, true); - self.discontinuity = pending.discontinuity; + self.restart = pending.restart; self.clock = DecodeClock::default(); } let Kind::Video(stream_type) = self.kind else { @@ -300,11 +301,7 @@ impl Track { description: Option, jitter: &mut jitter::Buffer, ) { - let Pending { - frame, - discontinuity, - arrived, - } = pending; + let Pending { frame, restart, arrived } = pending; let dts = dts.filter(|&dts| dts != to_ticks(frame.timestamp)); let decode = dts .and_then(|ticks| Timestamp::from_scale(ticks, 90_000).ok()) @@ -312,7 +309,7 @@ impl Track { let arrival = Arrival { arrived, decode, - discontinuity, + restart, sync: frame.keyframe || !matches!(self.kind, Kind::Video(_)), item: Queued { frame, @@ -982,12 +979,12 @@ impl Export { } let pending = Pending { frame, - discontinuity: track.source.discontinuity(), + restart: track.source.restarts(), arrived: track.empty, }; // A new timeline must reach the reset before tune-in alignment can drop it. if let Some(start) = video_start - && !is_video && pending.discontinuity == track.discontinuity + && !is_video && pending.restart == track.restart && pending.frame.timestamp < start { continue; @@ -1286,7 +1283,7 @@ impl Export { for (name, track) in self.tracks.iter_mut() { track.finished = true; track.pending = None; - track.discontinuity = 0; + track.restart = 0; track.clock = DecodeClock::default(); self.stale.insert(name.clone()); } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index b10fa63a696..385fcf0a4fd 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -5971,6 +5971,66 @@ async fn jitter_published_after_the_tables_keeps_the_encoders_clock() { assert_decodes_after_the_clock(&runner); } +/// A group the consumer skips (here one that never arrives) leaves the timeline where it was, +/// so the export keeps its clock: the program clock runs on with no break flagged, and a +/// frame the wait made late would be dropped like any other. +#[tokio::test(start_paused = true)] +async fn a_skipped_group_keeps_the_clock() { + use crate::container::Container as _; + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut track = broadcast + .create_track("a.aac", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + { + let mut cfg = AudioConfig::new(AAC { profile: 2 }, 48_000, 2); + cfg.container = Container::Legacy; + catalog.modify().unwrap().audio.renditions.insert("a.aac".to_string(), cfg); + } + + let delay = Duration::from_millis(500); + let mut export = Export::new(crate::source::announced(&consumer)) + .await + .unwrap() + .with_delay(delay); + let start = tokio::time::Instant::now(); + let mut out = Vec::new(); + // 100 ms groups of five 20 ms frames, a minute in; group 20 never arrives. + for sequence in 0..60u64 { + tokio::time::sleep_until(start + Duration::from_millis(sequence * 100)).await; + if sequence != 20 { + let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + for k in 0..5u64 { + let frame = Frame { + timestamp: Timestamp::from_millis(60_000 + sequence * 100 + k * 20).unwrap(), + duration: None, + payload: Bytes::from_static(&[0x21, 0x10, 0x04, 0x60]), + keyframe: k == 0, + }; + HangContainer::Legacy(crate::container::Kind::Audio) + .write(&mut group, &[frame]) + .unwrap(); + } + group.finish().unwrap(); + } + out.extend(poll_frames(&mut export)); + } + track.finish().unwrap(); + out.extend(drain_frames(&mut export).await); + + assert_eq!(export.discontinuity(), 0, "a skipped group is not a new program clock"); + assert_eq!(count_discontinuity(&out), 0, "no PCR break is flagged"); + let ts: Vec = out.iter().flat_map(|f| f.payload.iter().copied()).collect(); + let (_, audio) = collect_pes_pts(&ts); + assert!( + audio.iter().any(|&pts| pts >= (60_000 + 5_000) * 90), + "the audio after the skip went out" + ); + assert_on_time(&out); +} + /// Publish a Legacy AAC rendition named `name`. fn aac_rendition( broadcast: &mut moq_net::broadcast::Producer, diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index c71d3dce881..4abeb5c90e4 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -15,8 +15,8 @@ pub(crate) struct Arrival { pub arrived: Instant, /// When the frame decodes, nondecreasing within a track. pub decode: Timestamp, - /// The source's discontinuity counter when the frame was read. - pub discontinuity: u64, + /// How many times the source had restarted its timeline when the frame was read. + pub restart: u64, /// Whether the frame decodes without the track's earlier frames. pub sync: bool, pub item: T, @@ -25,7 +25,7 @@ pub(crate) struct Arrival { /// A frame [`Buffer::poll_next`] let go. pub(crate) struct Ready { pub track: K, - /// Counts up each time a discontinuity moved the timeline, and the clock with it. Every + /// Counts up each time a restart moved the timeline, and the clock with it. Every /// frame of one generation goes out before any frame of the next. pub generation: u64, pub item: T, @@ -49,14 +49,15 @@ pub(crate) enum Push { /// different skew, each within its deadline, emit them in the same order. A frame that /// arrives past its deadline would break that order, so it is dropped and counted. /// -/// A source's discontinuity (a skipped group, a declared marker) may have moved its -/// timeline, so once frames have gone out it opens a new generation with its own clock, -/// anchored at that frame though never ahead of a deadline already given out. Every -/// track's next frame runs on it: one crossing the same discontinuity joins it when the -/// frame lands on its clock, neither late nor held more than a delay past everything -/// queued, and opens another otherwise. Before anything has gone out there is nothing to -/// keep in step with, so a discontinuity there (a consumer skipping a stale group at join) -/// keeps the clock. No two tracks are ever on different clocks. +/// A skipped group leaves the timeline where it was, so it keeps the clock: frames it made +/// late are dropped like any other. A source that restarts its timeline (a declared +/// marker) has moved it, so once frames have gone out the restart opens a new generation +/// with its own clock, anchored at that frame though never ahead of a deadline already +/// given out. Every track's next frame runs on it: one crossing the same restart joins it +/// when the frame lands on its clock, neither late nor held more than a delay past +/// everything queued, and opens another otherwise. Before anything has gone out there is +/// nothing to keep in step with, so a restart there keeps the clock. No two tracks are +/// ever on different clocks. /// /// The clock follows the source's: a source clock running slower than ours would make /// every frame late in the end, and a faster one would hold more and more. So the clock @@ -106,8 +107,8 @@ struct Clock { struct Track { /// Frames in arrival order, each with its generation and deadline. queue: VecDeque<(u64, Instant, T)>, - /// The source's discontinuity counter at the last frame. - discontinuity: u64, + /// The source's restart counter at the last frame. + restart: u64, /// The generation of the last frame. generation: u64, /// A frame was dropped, so later ones are too until the next sync frame. @@ -138,10 +139,10 @@ impl Buffer { }); let mut generation = clock.generation; let mut deadline = self.deadline(&clock, arrival.decode); - // A track still on an older generation may be crossing the discontinuity another - // track opened the newest one for. + // A track still on an older generation may be crossing the restart another track + // opened the newest one for. let jumped = self.tracks.get(&key).is_some_and(|track| { - track.discontinuity != arrival.discontinuity + track.restart != arrival.restart && (track.generation == generation || !self.lands(deadline, arrival.arrived)) }); if jumped && self.released { @@ -162,11 +163,11 @@ impl Buffer { let track = self.tracks.entry(key).or_insert_with(|| Track { queue: VecDeque::new(), - discontinuity: arrival.discontinuity, + restart: arrival.restart, generation, waiting: false, }); - track.discontinuity = arrival.discontinuity; + track.restart = arrival.restart; track.generation = generation; let push = match deadline { _ if track.waiting && !arrival.sync => Push::Waiting, @@ -323,16 +324,16 @@ mod tests { Arrival { arrived, decode: ms(decode), - discontinuity: 0, + restart: 0, sync: true, item, } } - /// The same frame, read after its source's discontinuity counter moved to `discontinuity`. - fn after(discontinuity: u64, arrival: Arrival<&'static str>) -> Arrival<&'static str> { + /// The same frame, read after its source restarted its timeline `restart` times. + fn after(restart: u64, arrival: Arrival<&'static str>) -> Arrival<&'static str> { Arrival { - discontinuity, + restart, ..arrival } } @@ -431,8 +432,7 @@ mod tests { assert_eq!(due(&mut buffer), ["v0", "a0", "a180", "v240"]); } - /// A consumer that skips a stale group at join counts a discontinuity before anything - /// went out. The skip keeps the clock, so the tracks stay on one. + /// A restart before anything went out keeps the clock, so the tracks stay on one. #[tokio::test(start_paused = true)] async fn a_skip_before_the_first_release_keeps_one_clock() { let start = Instant::now(); @@ -453,8 +453,8 @@ mod tests { ); } - /// A discontinuity after frames went out opens a new generation, and every track's next - /// frame runs on it, whether or not that track saw the discontinuity. + /// A restart after frames went out opens a new generation, and every track's next frame + /// runs on it, whether or not that track saw the restart. #[tokio::test(start_paused = true)] async fn every_track_joins_a_new_generation_at_its_next_frame() { let start = Instant::now(); @@ -469,7 +469,7 @@ mod tests { let resume = start + Duration::from_secs(1); tokio::time::advance(resume - Instant::now()).await; assert_eq!(buffer.push(1, after(1, arrival(resume, 5_000, "v5000"))), Push::Queued); - // The audio crosses no discontinuity of its own; the data crosses the same one. + // The audio crosses no restart of its own; the data crosses the same one. let late = resume + Duration::from_millis(10); assert_eq!(buffer.push(2, arrival(late, 5_000, "a5000")), Push::Queued); assert_eq!(buffer.push(3, after(1, arrival(late, 5_000, "d5000"))), Push::Queued); @@ -521,7 +521,7 @@ mod tests { let frame = Arrival { arrived, decode: Timestamp::from_micros(decode.as_micros() as u64).unwrap(), - discontinuity: 0, + restart: 0, sync: true, item: "v", }; From 0160e26841aab7e5876dea2ff4b258afb384507a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 13:53:28 -0700 Subject: [PATCH 25/40] fix(moq-mux): join the TS export at the live edge The export's video and audio subscriptions now start at the newest group (a zero max age), then widen to the delay for reordering, so a join no longer begins on a group up to a delay old and carries that lag. Verbatim streams keep the delay's reach. Tests that write a broadcast whole before exporting it opt into replay. The jitter buffer also tells skips apart from restarts: a skipped group keeps the clock, unless the frames after it would be held more than a delay past everything queued, which means the timeline leapt ahead (a marker group the publisher shed). Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/publish.rs | 17 ++--- rs/moq-mux/src/container/source.rs | 37 ++++++++++- rs/moq-mux/src/container/ts/export.rs | 51 +++++++++++++-- rs/moq-mux/src/container/ts/export_test.rs | 64 ++++++++++++++++-- rs/moq-mux/src/container/ts/import.rs | 3 +- rs/moq-mux/src/container/ts/import_test.rs | 3 +- rs/moq-mux/src/jitter.rs | 76 +++++++++++++++++++--- 7 files changed, 217 insertions(+), 34 deletions(-) diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index d6eceeb339d..c47b5f7bcaa 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -680,12 +680,12 @@ mod tests { .tracks .insert(section.name().to_string(), section_track); let mut section_producer = Producer::new(section, Container::Legacy(moq_mux::container::Kind::Data)); - // bbb's first video keyframe is at 1.4 s; stamp the ancillary streams just after - // it so they clear the export's keyframe alignment (anything before the first - // keyframe is dropped on tune-in). + // The export joins each media track at its newest group, bbb's last at about + // 1.446 s; stamp the ancillary streams just after it so they clear the keyframe + // alignment (anything before the first keyframe is dropped on tune-in). section_producer .write(Frame { - timestamp: Timestamp::from_millis(1410).unwrap(), + timestamp: Timestamp::from_millis(1450).unwrap(), duration: None, payload: bytes::Bytes::from_static(CUE), keyframe: true, @@ -713,7 +713,7 @@ mod tests { let mut pes_producer = Producer::new(pes, Container::Legacy(moq_mux::container::Kind::Data)); pes_producer .write(Frame { - timestamp: Timestamp::from_millis(1410).unwrap(), + timestamp: Timestamp::from_millis(1450).unwrap(), duration: None, payload: bytes::Bytes::from_static(PES_PAYLOAD), keyframe: true, @@ -738,11 +738,8 @@ mod tests { .await } - /// The media track's full retention window, so an exporter started after publishing - /// can still read every retained group. These tests publish a whole feed before - /// exporting it, which the default - /// [`Duration::ZERO`] collapses to the live edge: - /// completeness has to be asked for, exactly as a real recorder does. + /// The export's delay, long enough to reach the ancillary tracks: these tests publish a + /// whole feed before exporting it, and the media tracks start at their newest group. const RECORDING_MAX_AGE: std::time::Duration = Duration::from_secs(30); /// Full CLI round-trip over the hang catalog. #[tokio::test(start_paused = true)] diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index 41fb2ce40e9..5708eacb79e 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -66,6 +66,9 @@ pub(crate) struct ExportSource { /// Wire format, consumed when the subscription resolves into a consumer. media: Option, max_age: std::time::Duration, + /// Start at the newest group, rather than the oldest the max age still reaches, and + /// apply the max age once it is read. + live: bool, transform: Option, /// Resolved codec configuration record (avcC / hvcC / AudioSpecificConfig / /// OpusHead). Some once the codec config is available — from the catalog @@ -125,6 +128,7 @@ impl ExportSource { state: SourceState::Requesting(request, name.to_string()), media: Some(media), max_age, + live: false, transform, description, video_codec: Some(config.codec.clone()), @@ -154,6 +158,7 @@ impl ExportSource { state: SourceState::Requesting(request, name.to_string()), media: Some(media), max_age, + live: false, transform: None, description, video_codec: None, @@ -172,6 +177,7 @@ impl ExportSource { state: SourceState::Requesting(source.request_catalog(), name.to_string()), media: Some(HangContainer::Legacy(crate::container::Kind::Data)), max_age, + live: false, transform: None, description: None, video_codec: None, @@ -179,6 +185,23 @@ impl ExportSource { }) } + /// Start at the newest group the publisher has, the live edge, rather than the oldest + /// one the max age still reaches; the max age applies from the first frame on, to the + /// groups that follow. + pub fn live(mut self) -> Self { + self.live = true; + self + } + + /// The underlying consumer's playhead generation, or 0 until the subscription resolves: + /// it counts skipped groups as well as restarts ([`Consumer::discontinuity`]). + pub fn skips(&self) -> u64 { + match &self.state { + SourceState::Active(consumer) => consumer.discontinuity(), + _ => 0, + } + } + /// The resolved codec-config record, if available. pub fn description(&self) -> Option<&Bytes> { self.description.as_ref() @@ -250,7 +273,12 @@ impl ExportSource { }; (ready!(pending.poll_ok(waiter))?, name.clone()) }; - let subscription = moq_net::track::Subscription::default().with_max_age(self.max_age); + // A zero budget asks only for the newest group. + let max_age = match self.live { + true => std::time::Duration::ZERO, + false => self.max_age, + }; + let subscription = moq_net::track::Subscription::default().with_max_age(max_age); self.state = SourceState::Subscribing(broadcast.track(&name)?.subscribe(subscription)); } @@ -277,7 +305,12 @@ impl ExportSource { let SourceState::Active(consumer) = &mut self.state else { unreachable!("subscription resolved into an Active consumer"); }; - match ready!(consumer.poll_event(waiter))? { + let event = ready!(consumer.poll_event(waiter))?; + if std::mem::take(&mut self.live) { + // Started at the live edge: from here the budget tolerates reordering. + consumer.set_max_age(self.max_age); + } + match event { Some(Event::Frame(frame)) => frame, event => return Poll::Ready(Ok(event)), } diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index b1bf7642a77..e77cb89b086 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -141,6 +141,9 @@ pub struct Export { pcr_discontinuity: bool, /// Lays the muxed packets onto the PCR grid. schedule: Schedule, + /// Read each track from the oldest group the delay still reaches, rather than the live + /// edge: a test exporting a broadcast it wrote whole. + replay: bool, /// Wakes the export when the next grid slot is due on the jitter buffer's clock. slot_timer: Option>>, /// Output frames ready to hand out, one per grid slot. @@ -169,6 +172,8 @@ struct Pending { frame: Frame, /// How many times the source had restarted its timeline when the frame was read. restart: u64, + /// How many times its playhead had jumped, restarts included ([`jitter::Arrival::skip`]). + skip: u64, /// The earliest the frame could have arrived: when its source was last found empty. arrived: web_async::time::Instant, } @@ -301,7 +306,12 @@ impl Track { description: Option, jitter: &mut jitter::Buffer, ) { - let Pending { frame, restart, arrived } = pending; + let Pending { + frame, + restart, + skip, + arrived, + } = pending; let dts = dts.filter(|&dts| dts != to_ticks(frame.timestamp)); let decode = dts .and_then(|ticks| Timestamp::from_scale(ticks, 90_000).ok()) @@ -310,6 +320,7 @@ impl Track { arrived, decode, restart, + skip, sync: frame.keyframe || !matches!(self.kind, Kind::Video(_)), item: Queued { frame, @@ -696,6 +707,7 @@ impl Export { pcr_discontinuity: false, schedule: Schedule::new(Duration::ZERO), slot_timer: None, + replay: false, queue: VecDeque::new(), video_start: None, mux_rate: None, @@ -719,6 +731,14 @@ impl Export { self } + /// Read each track from the oldest group the delay still reaches instead of the live + /// edge, for a test that writes a broadcast whole before exporting it. + #[cfg(test)] + pub(super) fn with_replay(mut self) -> Self { + self.replay = true; + self + } + /// Mux each frame this long after its decode time, like an SRT receiver's TSBPD. /// /// The clock starts at the first frame's arrival, so the output keeps the source's @@ -980,6 +1000,7 @@ impl Export { let pending = Pending { frame, restart: track.source.restarts(), + skip: track.source.skips(), arrived: track.empty, }; // A new timeline must reach the reset before tune-in alignment can drop it. @@ -1161,7 +1182,7 @@ impl Export { let kind = video_kind(config, name)?; let mut track = match old.remove(name) { Some(track) => track, - None => match ExportSource::for_video(&self.source, name, config, self.delay)? { + None => match ExportSource::for_video(&self.source, name, config, self.delay)?.map(|s| at_edge(s, self.replay)) { Some(source) => Track::new(source, kind.clone()), None => continue, }, @@ -1174,7 +1195,7 @@ impl Export { let kind = audio_kind(config, name)?; let mut track = match old.remove(name) { Some(track) => track, - None => match ExportSource::for_audio(&self.source, name, config, self.delay)? { + None => match ExportSource::for_audio(&self.source, name, config, self.delay)?.map(|s| at_edge(s, self.replay)) { Some(source) => Track::new(source, kind.clone()), None => continue, }, @@ -1220,9 +1241,9 @@ impl Export { continue; } let source = if let Some(config) = catalog.video.renditions.get(name) { - ExportSource::for_video(&self.source, name, config, self.delay)? + ExportSource::for_video(&self.source, name, config, self.delay)?.map(|s| at_edge(s, self.replay)) } else if let Some(config) = catalog.audio.renditions.get(name) { - ExportSource::for_audio(&self.source, name, config, self.delay)? + ExportSource::for_audio(&self.source, name, config, self.delay)?.map(|s| at_edge(s, self.replay)) } else if mpegts.tracks.get(name).is_some_and(|t| t.verbatim.is_some()) { Some(ExportSource::for_stream(&self.source, name, self.delay)?) } else { @@ -1245,10 +1266,15 @@ impl Export { self.jitter.dropped() } - /// When the next queued frame, or grid slot while media is queued, is due. + /// When the next queued frame is due, or the next grid slot while media is queued or + /// every source has finished and the tail runs out. #[cfg(test)] pub(super) fn next_due(&self) -> Option { - let slot = self.slot_due().filter(|_| self.schedule.queued()).map(|(_, at)| at); + let finished = !self.tracks.is_empty() && self.tracks.values().all(|track| track.finished); + let slot = self + .slot_due() + .filter(|_| self.schedule.queued() || finished) + .map(|(_, at)| at); self.jitter.next_deadline().into_iter().chain(slot).min() } @@ -2049,6 +2075,17 @@ fn video_kind(config: &VideoConfig, name: &str) -> anyhow::Result { } } +/// Start a media track's source at the live edge, the newest group, so the export does not +/// begin on a group up to a delay old and carry that lag ever after; unless `replay`. +/// Verbatim streams are sparse, and anything of theirs ahead of the first keyframe is +/// dropped anyway, so they keep the delay's reach. +fn at_edge(source: ExportSource, replay: bool) -> ExportSource { + match replay { + true => source, + false => source.live(), + } +} + /// A video level's T-STD limits (H.222.0 2.14.3.1, 2.17.2), in bits per second and bits. #[derive(Clone, Copy, Debug)] struct Level { diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 385fcf0a4fd..1ddaf4951be 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -82,7 +82,7 @@ async fn drain(consumer: moq_net::broadcast::Consumer) -> BytesMut { /// `drain` for an exporter built with an explicit catalog extension. async fn drain_with(exporter: Export) -> BytesMut { - let mut exporter = exporter.with_delay(RECORDING_MAX_AGE); + let mut exporter = exporter.with_delay(RECORDING_MAX_AGE).with_replay(); let mut out = BytesMut::new(); // `while let Ok` stops on the first timeout (`Pending`: no more output). while let Ok(res) = tokio::time::timeout(DRAIN, exporter.next()).await { @@ -2670,6 +2670,7 @@ async fn export_of(consumer: &moq_net::broadcast::Consumer) -> Export Vec { .await .unwrap() .with_mux_rate(20_000_000) - .with_delay(RECORDING_MAX_AGE); + .with_delay(RECORDING_MAX_AGE) + .with_replay(); drain_frames(&mut exporter).await } @@ -5981,7 +5983,7 @@ async fn a_skipped_group_keeps_the_clock() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - let mut track = broadcast + let track = broadcast .create_track("a.aac", hang::container::track_info(hang::catalog::PRIORITY.audio)) .unwrap(); { @@ -6031,6 +6033,53 @@ async fn a_skipped_group_keeps_the_clock() { assert_on_time(&out); } +/// An exporter joining a live broadcast starts at its newest group, the live edge, not at +/// the oldest one its delay still reaches: otherwise its clock would carry that group's age, +/// up to the delay, as latency for as long as it runs. +#[tokio::test(start_paused = true)] +async fn a_late_join_starts_at_the_live_edge() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut audio = aac_rendition(&mut broadcast, &mut catalog, "a.aac"); + let start = tokio::time::Instant::now(); + let mut export = None; + let mut out = Vec::new(); + // One-second groups of 20 ms frames; the exporter joins 5.5 s in with a 2 s delay. + for k in 0..400u64 { + tokio::time::sleep_until(start + Duration::from_millis(k * 20)).await; + if k % 50 == 0 && k > 0 { + audio.cut(None).unwrap(); + } + audio + .write(Frame { + timestamp: Timestamp::from_millis(60_000 + k * 20).unwrap(), + duration: None, + payload: Bytes::from_static(&[0x21, 0x10, 0x04, 0x60]), + keyframe: k % 50 == 0, + }) + .unwrap(); + if k == 275 { + let joined = Export::new(crate::source::announced(&consumer)).await.unwrap(); + export = Some(joined.with_delay(Duration::from_secs(2))); + } + if let Some(export) = export.as_mut() { + out.extend(poll_frames(export)); + } + } + audio.finish().unwrap(); + let mut export = export.unwrap(); + out.extend(drain_frames(&mut export).await); + + let ts: Vec = out.iter().flat_map(|f| f.payload.iter().copied()).collect(); + let (_, pts) = collect_pes_pts(&ts); + assert_eq!( + pts.first().copied(), + Some((60_000 + 5_000) * 90), + "the export starts at the newest group" + ); +} + /// Publish a Legacy AAC rendition named `name`. fn aac_rendition( broadcast: &mut moq_net::broadcast::Producer, @@ -6104,7 +6153,8 @@ async fn a_track_leaving_the_catalog_is_read_to_its_end() { let mut export = Export::new(crate::source::announced(&consumer)) .await .unwrap() - .with_delay(RECORDING_MAX_AGE); + .with_delay(RECORDING_MAX_AGE) + .with_replay(); for ms in (0..200).step_by(20) { write_aac(&mut kept, ms); write_aac(&mut leaving, ms); @@ -6147,7 +6197,11 @@ async fn resume_after(finish: bool) { let (mut broadcast, mut catalog) = publish(); let mut track = aac_rendition(&mut broadcast, &mut catalog, "a.aac"); let ended = source.broadcast().await.unwrap(); - let mut export = Export::new(source.clone()).await.unwrap().with_delay(RECORDING_MAX_AGE); + let mut export = Export::new(source.clone()) + .await + .unwrap() + .with_delay(RECORDING_MAX_AGE) + .with_replay(); for ms in (0..200).step_by(20) { write_aac(&mut track, ms); } diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 201adbdfb60..2a5a1829155 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -6041,7 +6041,8 @@ pub(super) mod test { let mut exporter = crate::container::ts::Export::new(crate::source::announced(&consumer)) .await .unwrap() - .with_delay(delay); + .with_delay(delay) + .with_replay(); let flags = |frame: crate::container::Frame| { frame .payload diff --git a/rs/moq-mux/src/container/ts/import_test.rs b/rs/moq-mux/src/container/ts/import_test.rs index 65d653592a8..66f15788cbe 100644 --- a/rs/moq-mux/src/container/ts/import_test.rs +++ b/rs/moq-mux/src/container/ts/import_test.rs @@ -447,7 +447,8 @@ async fn import_export_import_roundtrip() { let mut exporter = crate::container::ts::Export::new(crate::source::announced(&consumer)) .await .unwrap() - .with_delay(RECORDING_MAX_AGE); + .with_delay(RECORDING_MAX_AGE) + .with_replay(); let mut out = BytesMut::new(); while let Ok(res) = tokio::time::timeout(DRAIN, exporter.next()).await { match res.expect("exporter error") { diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index 4abeb5c90e4..6bef8c287b9 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -17,6 +17,9 @@ pub(crate) struct Arrival { pub decode: Timestamp, /// How many times the source had restarted its timeline when the frame was read. pub restart: u64, + /// How many times the source's playhead had jumped (a skipped group, or a restart) + /// when the frame was read. + pub skip: u64, /// Whether the frame decodes without the track's earlier frames. pub sync: bool, pub item: T, @@ -53,7 +56,9 @@ pub(crate) enum Push { /// late are dropped like any other. A source that restarts its timeline (a declared /// marker) has moved it, so once frames have gone out the restart opens a new generation /// with its own clock, anchored at that frame though never ahead of a deadline already -/// given out. Every track's next frame runs on it: one crossing the same restart joins it +/// given out. So does a skip after which a frame would be held more than a delay past +/// everything queued: its timeline jumped ahead, as when the skipped groups held a marker +/// the publisher shed. Every track's next frame runs on it: one crossing the same restart joins it /// when the frame lands on its clock, neither late nor held more than a delay past /// everything queued, and opens another otherwise. Before anything has gone out there is /// nothing to keep in step with, so a restart there keeps the clock. No two tracks are @@ -107,8 +112,9 @@ struct Clock { struct Track { /// Frames in arrival order, each with its generation and deadline. queue: VecDeque<(u64, Instant, T)>, - /// The source's restart counter at the last frame. + /// The source's restart and skip counters at the last frame. restart: u64, + skip: u64, /// The generation of the last frame. generation: u64, /// A frame was dropped, so later ones are too until the next sync frame. @@ -142,8 +148,10 @@ impl Buffer { // A track still on an older generation may be crossing the restart another track // opened the newest one for. let jumped = self.tracks.get(&key).is_some_and(|track| { - track.restart != arrival.restart - && (track.generation == generation || !self.lands(deadline, arrival.arrived)) + let restarted = track.restart != arrival.restart + && (track.generation == generation || !self.lands(deadline, arrival.arrived)); + let leapt = track.skip != arrival.skip && deadline.is_some_and(|deadline| deadline > self.bound(arrival.arrived)); + restarted || leapt }); if jumped && self.released { generation += 1; @@ -164,10 +172,12 @@ impl Buffer { let track = self.tracks.entry(key).or_insert_with(|| Track { queue: VecDeque::new(), restart: arrival.restart, + skip: arrival.skip, generation, waiting: false, }); track.restart = arrival.restart; + track.skip = arrival.skip; track.generation = generation; let push = match deadline { _ if track.waiting && !arrival.sync => Push::Waiting, @@ -192,10 +202,13 @@ impl Buffer { /// Whether a frame arriving at `arrived` and due at `deadline` is on the clock: not late, /// and held no more than a delay past both its arrival and everything already queued. fn lands(&self, deadline: Option, arrived: Instant) -> bool { - deadline.is_some_and(|deadline| { - let bound = arrived.max(self.horizon.unwrap_or(arrived)) + self.delay; - arrived <= deadline && deadline <= bound - }) + deadline.is_some_and(|deadline| arrived <= deadline && deadline <= self.bound(arrived)) + } + + /// The latest a frame arriving at `arrived` may be due and still be on the clock: a delay + /// past both its arrival and everything already queued. + fn bound(&self, arrived: Instant) -> Instant { + arrived.max(self.horizon.unwrap_or(arrived)) + self.delay } /// Account for the slack a frame arrived with, and every [`STEER`] of decode time set the @@ -325,6 +338,7 @@ mod tests { arrived, decode: ms(decode), restart: 0, + skip: 0, sync: true, item, } @@ -334,6 +348,7 @@ mod tests { fn after(restart: u64, arrival: Arrival<&'static str>) -> Arrival<&'static str> { Arrival { restart, + skip: restart, ..arrival } } @@ -481,6 +496,50 @@ mod tests { assert_eq!(released(&mut buffer), [("a5020", 1)]); } + /// A skipped group whose timeline carried on keeps the clock: a frame it made late is + /// dropped, and the next on-time frame goes out on the same generation. + #[tokio::test(start_paused = true)] + async fn a_skip_keeps_the_clock() { + let start = Instant::now(); + let mut buffer = Buffer::new(DELAY); + buffer.push(1, arrival(start, 0, "a0")); + tokio::time::advance(DELAY).await; + assert_eq!(due(&mut buffer), ["a0"]); + + // Group 1 is skipped: frames from 100 ms on arrive after a 250 ms stall. + let stalled = start + Duration::from_millis(350); + tokio::time::advance(stalled - Instant::now()).await; + let skipped = |decode, item| Arrival { + skip: 1, + ..arrival(stalled, decode, item) + }; + assert_eq!(buffer.push(1, skipped(200, "a200")), Push::Late); + assert_eq!(buffer.push(1, skipped(300, "a300")), Push::Queued); + tokio::time::advance(Duration::from_millis(100)).await; + assert_eq!(released(&mut buffer), [("a300", 0)]); + } + + /// A skip after which a frame would be held more than a delay past everything queued + /// followed a timeline that leapt ahead, as when the skipped groups held a marker the + /// publisher shed, so it opens a generation. + #[tokio::test(start_paused = true)] + async fn a_skip_that_leaps_ahead_opens_a_generation() { + let start = Instant::now(); + let mut buffer = Buffer::new(DELAY); + buffer.push(1, arrival(start, 0, "a0")); + tokio::time::advance(DELAY).await; + assert_eq!(due(&mut buffer), ["a0"]); + + let now = Instant::now(); + let leapt = Arrival { + skip: 1, + ..arrival(now, 5_000, "a5000") + }; + assert_eq!(buffer.push(1, leapt), Push::Queued); + tokio::time::advance(DELAY).await; + assert_eq!(released(&mut buffer), [("a5000", 1)]); + } + /// Every frame of a generation goes out before the next generation's, even one the new /// clock puts earlier. #[tokio::test(start_paused = true)] @@ -522,6 +581,7 @@ mod tests { arrived, decode: Timestamp::from_micros(decode.as_micros() as u64).unwrap(), restart: 0, + skip: 0, sync: true, item: "v", }; From 2dc542b4af5b6b3ba78a8423620e29bc49ca7510 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 14:17:46 -0700 Subject: [PATCH 26/40] fix(moq-mux): hold the TS export's clock to 13818-1 drift limits The jitter clock is the output's PCR, so its steering now keeps to what ISO/IEC 13818-1 2.4.2.1 allows a system clock: within 810 Hz of 27 MHz (30 ppm) of ours, changing by at most 0.075 Hz/s. The old controller ran up to 500 ppm off and stepped freely. It measures the source's rate as the least-squares slope of the least slack over ten minutes, runs at that rate, and pulls the slack back to the delay no faster than the slew limit can brake. A source past 30 ppm fails the export once it has used half the delay, before any frame goes late. pcr-timing.py gains a live pcr-rate check against the same limit. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 8 +- rs/moq-mux/src/container/ts/export.rs | 30 +- rs/moq-mux/src/container/ts/export_test.rs | 7 +- rs/moq-mux/src/jitter.rs | 338 ++++++++++++++------- test/ts/README.md | 15 +- test/ts/pcr-timing.py | 52 +++- 6 files changed, 322 insertions(+), 128 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index f3fb4fcb397..f92c69f907a 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -368,8 +368,12 @@ source's pace and muxes all tracks in decode order whatever their arrival skew: two exporters of one broadcast emit them in one order. A frame that arrives later than its deadline is dropped, and video then resumes at its next keyframe. The clock follows the source's, so a source whose clock runs a little -fast or slow (the 30 ppm a broadcast system clock may) neither goes late nor -piles up over a long run, and a lead it started with wears away. Each 25 ms +fast or slow neither goes late nor piles up over a long run, and a lead it +started with wears away. The output's PCR is that clock, so it keeps to what +ISO/IEC 13818-1 allows a system clock: within 30 ppm of the exporter's, its +rate changing by at most 0.075 Hz/s. Catching up with a source near that limit +takes hours, and one further off fails the export once it has used half the +delay. Each 25 ms slice of the output is written when the clock reaches it. The delay is also how stale a group may get before it is skipped. `--delay 0` holds nothing and drops nothing, writing frames in arrival order as they come. diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index e77cb89b086..96d10739f86 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -236,10 +236,10 @@ impl Track { /// The decode clock runs here, as frames are read, because the deadline is on decode time: /// reordered video arrives in decode order with PTS 0, 120, 40, 80, so a PTS deadline would /// strand a B-frame behind its reference or send it first. - fn queue(&mut self, name: &str, pending: Pending, jitter: &mut jitter::Buffer) { + fn queue(&mut self, name: &str, pending: Pending, jitter: &mut jitter::Buffer) -> anyhow::Result<()> { if pending.restart != self.restart { // The source may have restarted its timeline, so the decode clock restarts too. - self.release(name, jitter, true); + self.release(name, jitter, true)?; self.restart = pending.restart; self.clock = DecodeClock::default(); } @@ -251,7 +251,7 @@ impl Track { } let pts = to_ticks(pending.frame.timestamp); self.clock.push((pending, self.source.description().cloned()), pts); - self.release(name, jitter, false); + self.release(name, jitter, false) } /// Whether the track passes AC-3 through as DVB private data, whose PES may carry several @@ -289,12 +289,13 @@ impl Track { } /// Queue every held video frame whose DTS is settled, or all of them when `flush`. - fn release(&mut self, name: &str, jitter: &mut jitter::Buffer, flush: bool) { + fn release(&mut self, name: &str, jitter: &mut jitter::Buffer, flush: bool) -> anyhow::Result<()> { let (delay, lookahead) = self.timing.reorder(); let lookahead = lookahead.max(self.timing.jitter); while let Some(((pending, description), dts)) = self.clock.pop(lookahead, delay, flush) { - self.push(name, pending, Some(dts), description, jitter); + self.push(name, pending, Some(dts), description, jitter)?; } + Ok(()) } /// Queue a frame decoding at `dts` (90 kHz ticks), else at its PTS. @@ -305,7 +306,7 @@ impl Track { dts: Option, description: Option, jitter: &mut jitter::Buffer, - ) { + ) -> anyhow::Result<()> { let Pending { frame, restart, @@ -329,9 +330,10 @@ impl Track { description, }, }; - if jitter.push(self.pid, arrival) == Push::Late { + if jitter.push(self.pid, arrival)? == Push::Late { tracing::warn!(track = %name, dropped = jitter.dropped(), "frame missed its deadline; dropped"); } + Ok(()) } } @@ -854,7 +856,7 @@ impl Export { for (_, name) in held { let track = self.tracks.get_mut(&name).unwrap(); let pending = track.pending.take().unwrap(); - track.queue(&name, pending, &mut self.jitter); + track.queue(&name, pending, &mut self.jitter)?; } self.fill(waiter)?; } @@ -1014,7 +1016,7 @@ impl Export { track.pending = Some(pending); break; } - track.queue(name, pending, &mut self.jitter); + track.queue(name, pending, &mut self.jitter)?; } Poll::Ready(None) => { track.finished = true; @@ -1028,7 +1030,7 @@ impl Export { } // Nothing more can come in below the frames still held for their DTS. if track.finished && !waiting_for_header { - track.release(name, &mut self.jitter, true); + track.release(name, &mut self.jitter, true)?; } } Ok(()) @@ -1182,7 +1184,9 @@ impl Export { let kind = video_kind(config, name)?; let mut track = match old.remove(name) { Some(track) => track, - None => match ExportSource::for_video(&self.source, name, config, self.delay)?.map(|s| at_edge(s, self.replay)) { + None => match ExportSource::for_video(&self.source, name, config, self.delay)? + .map(|s| at_edge(s, self.replay)) + { Some(source) => Track::new(source, kind.clone()), None => continue, }, @@ -1195,7 +1199,9 @@ impl Export { let kind = audio_kind(config, name)?; let mut track = match old.remove(name) { Some(track) => track, - None => match ExportSource::for_audio(&self.source, name, config, self.delay)?.map(|s| at_edge(s, self.replay)) { + None => match ExportSource::for_audio(&self.source, name, config, self.delay)? + .map(|s| at_edge(s, self.replay)) + { Some(source) => Track::new(source, kind.clone()), None => continue, }, diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 1ddaf4951be..ad5c423a3cf 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -5989,7 +5989,12 @@ async fn a_skipped_group_keeps_the_clock() { { let mut cfg = AudioConfig::new(AAC { profile: 2 }, 48_000, 2); cfg.container = Container::Legacy; - catalog.modify().unwrap().audio.renditions.insert("a.aac".to_string(), cfg); + catalog + .modify() + .unwrap() + .audio + .renditions + .insert("a.aac".to_string(), cfg); } let delay = Duration::from_millis(500); diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index 6bef8c287b9..4dc89bdd701 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -66,9 +66,12 @@ pub(crate) enum Push { /// /// The clock follows the source's: a source clock running slower than ours would make /// every frame late in the end, and a faster one would hold more and more. So the clock -/// keeps the least slack a frame arrived with near the delay, by running its decode -/// timeline up to [`MAX_DRIFT`] faster or slower than ours, which also wears away a lead -/// the first frame anchored it with. +/// measures how far the source's runs off ours from the least slack frames arrive with, runs +/// its decode timeline at that rate, and pulls the least slack back to the delay, which also +/// wears away a lead the first frame anchored it with. The decode timeline is the output's +/// system clock, so it keeps to what ISO/IEC 13818-1 2.4.2.1 allows one: within [`MAX_DRIFT`] +/// of ours, changing by at most [`MAX_SLEW`]. A source past that falls behind or piles up +/// until it has used half the delay, and then [`Buffer::push`] fails. /// /// A zero delay holds nothing and drops nothing: each frame goes out as soon as it is /// read, ordered only among the frames read together. @@ -81,9 +84,8 @@ pub(crate) struct Buffer { horizon: Option, /// Whether a frame has gone out since the clock started. released: bool, - /// The least slack (deadline less arrival, in nanoseconds) a frame arrived with since the - /// clock was last steered, and the decode time it was steered at. - slack: Option<(i128, Timestamp)>, + /// What the clock is steered on, since it last started. + steer: Steer, timer: Option>>, dropped: u64, } @@ -91,12 +93,17 @@ pub(crate) struct Buffer { /// How often the clock is steered, in decode time: long enough that the least slack covers /// a group's worth of frames sent ahead by different amounts. const STEER: Duration = Duration::from_secs(2); -/// How quickly the clock steers its slack back to the delay: it closes the gap over this -/// long, and a source drifting at `r` keeps it `r` times this off. -const RESPONSE: Duration = Duration::from_secs(60); -/// The fastest the clock runs off ours, in parts per billion: far past the 30 ppm a source's -/// system clock may be off (ISO 13818-1 2.4.2.1), with ours as far off the other way. -const MAX_DRIFT: i128 = 500_000; +/// How far back the source's rate is measured, in decode time: long enough that arrival +/// jitter averages out of it. +const RATE_WINDOW: f64 = 600.0; +/// How quickly the clock pulls the least slack back to the delay: it closes the gap over this +/// many seconds, or slower where the slew limit could not stop it in time. +const RESPONSE: f64 = 600.0; +/// The furthest the clock runs off ours: 810 Hz of the 27 MHz system clock (ISO/IEC 13818-1 +/// 2.4.2.1). +const MAX_DRIFT: f64 = 810.0 / 27e6; +/// How fast that may change, per second: 0.075 Hz/s of 27 MHz (ISO/IEC 13818-1 2.4.2.1). +const MAX_SLEW: f64 = 0.075 / 27e6; /// The decode timeline's mapping onto ours: where a frame decoding at `base` is due, less the /// delay, and how much longer than the decode timeline ours runs. @@ -105,8 +112,22 @@ struct Clock { generation: u64, anchor: Instant, base: Timestamp, - /// In parts per billion. - drift: i128, + /// As a fraction of the decode timeline. + drift: f64, +} + +/// What the clock is steered on. +#[derive(Default)] +struct Steer { + /// The least slack (deadline less arrival, in seconds) a frame arrived with since the last + /// step, and the decode time the step began at. + least: Option<(f64, Timestamp)>, + /// Each step's decode time and the source's phase there, in seconds, over the last + /// [`RATE_WINDOW`]: how far the least slack is past the delay, less how far the drift has + /// moved the deadlines. Its slope is how much slower the source's clock runs than ours. + phase: VecDeque<(f64, f64)>, + /// How far the drift has moved the deadlines, in seconds. + steered: f64, } struct Track { @@ -129,19 +150,20 @@ impl Buffer { tracks: BTreeMap::new(), horizon: None, released: false, - slack: None, + steer: Steer::default(), timer: None, dropped: 0, } } - /// Queue a frame, or drop it if it cannot make its deadline. - pub fn push(&mut self, key: K, arrival: Arrival) -> Push { + /// Queue a frame, or drop it if it cannot make its deadline. Fails once the source's clock + /// runs further off ours than the clock may follow. + pub fn push(&mut self, key: K, arrival: Arrival) -> anyhow::Result { let clock = *self.clock.get_or_insert(Clock { generation: 0, anchor: arrival.arrived, base: arrival.decode, - drift: 0, + drift: 0.0, }); let mut generation = clock.generation; let mut deadline = self.deadline(&clock, arrival.decode); @@ -150,7 +172,8 @@ impl Buffer { let jumped = self.tracks.get(&key).is_some_and(|track| { let restarted = track.restart != arrival.restart && (track.generation == generation || !self.lands(deadline, arrival.arrived)); - let leapt = track.skip != arrival.skip && deadline.is_some_and(|deadline| deadline > self.bound(arrival.arrived)); + let leapt = + track.skip != arrival.skip && deadline.is_some_and(|deadline| deadline > self.bound(arrival.arrived)); restarted || leapt }); if jumped && self.released { @@ -163,10 +186,10 @@ impl Buffer { drift: clock.drift, }; self.clock = Some(clock); - self.slack = None; + self.steer = Steer::default(); deadline = self.deadline(&clock, arrival.decode); } else if self.released && !self.delay.is_zero() { - self.steer(deadline, &arrival); + self.steer(deadline, &arrival)?; } let track = self.tracks.entry(key).or_insert_with(|| Track { @@ -196,7 +219,7 @@ impl Buffer { if push != Push::Queued { self.dropped += 1; } - push + Ok(push) } /// Whether a frame arriving at `arrived` and due at `deadline` is on the clock: not late, @@ -211,33 +234,62 @@ impl Buffer { arrived.max(self.horizon.unwrap_or(arrived)) + self.delay } - /// Account for the slack a frame arrived with, and every [`STEER`] of decode time set the - /// clock's drift to close the gap between the least slack and the delay over [`RESPONSE`]. - fn steer(&mut self, deadline: Option, arrival: &Arrival) { + /// Account for the slack a frame arrived with, and every [`STEER`] of decode time step the + /// clock's drift toward the source's rate plus a pull back to the delay, as far as + /// [`MAX_SLEW`] allows. + fn steer(&mut self, deadline: Option, arrival: &Arrival) -> anyhow::Result<()> { let (Some(clock), Some(deadline)) = (self.clock, deadline) else { - return; + return Ok(()); }; let slack = match deadline >= arrival.arrived { - true => (deadline - arrival.arrived).as_nanos() as i128, - false => -((arrival.arrived - deadline).as_nanos() as i128), + true => (deadline - arrival.arrived).as_secs_f64(), + false => -(arrival.arrived - deadline).as_secs_f64(), }; - let (least, since) = self.slack.get_or_insert((slack, arrival.decode)); - *least = (*least).min(slack); + let (least, since) = self.steer.least.get_or_insert((slack, arrival.decode)); + *least = least.min(slack); if arrival.decode.as_nanos().saturating_sub(since.as_nanos()) < STEER.as_nanos() { - return; + return Ok(()); } - let gap = *least - self.delay.as_nanos() as i128; - let drift = (-gap * 1_000_000_000 / RESPONSE.as_nanos() as i128).clamp(-MAX_DRIFT, MAX_DRIFT); // Re-anchor where this frame decodes, so the new drift starts from there. - if let Some(anchor) = deadline.checked_sub(self.delay) { - self.clock = Some(Clock { - anchor, - base: arrival.decode, - drift, - ..clock - }); + let Some(anchor) = deadline.checked_sub(self.delay) else { + return Ok(()); + }; + let delay = self.delay.as_secs_f64(); + let gap = *least - delay; + self.steer.least = None; + + let now = arrival.decode.as_nanos() as f64 / 1e9; + let elapsed = now - clock.base.as_nanos() as f64 / 1e9; + self.steer.steered += elapsed * clock.drift; + let phase = &mut self.steer.phase; + phase.push_back((now, gap - self.steer.steered)); + while phase.front().is_some_and(|&(at, _)| now - at > RATE_WINDOW) { + phase.pop_front(); + } + let source = -slope(phase).unwrap_or(-clock.drift); + + // Pull the gap shut over [`RESPONSE`], no faster than the slew limit can stop on it. + let pull = (gap.abs() / RESPONSE).min((2.0 * MAX_SLEW * gap.abs()).sqrt()); + let target = source - pull.copysign(gap); + // Tracks interleave a little off decode order, so a step may decode before the last. + let step = MAX_SLEW * elapsed.max(0.0); + let drift = (clock.drift + (target - clock.drift).clamp(-step, step)).clamp(-MAX_DRIFT, MAX_DRIFT); + self.clock = Some(Clock { + anchor, + base: arrival.decode, + drift, + ..clock + }); + + // At the limit and still losing ground, with half the delay gone. + if drift == MAX_DRIFT.copysign(source) && source.abs() > MAX_DRIFT && gap * source.signum() < -delay / 2.0 { + anyhow::bail!( + "the source's clock runs {:.1} ppm off ours, past the {:.0} ppm the output's may follow", + source * 1e6, + MAX_DRIFT * 1e6 + ); } - self.slack = None; + Ok(()) } /// When a frame decoding at `decode` goes out on the current clock, once there is one. @@ -249,7 +301,7 @@ impl Buffer { fn deadline(&self, clock: &Clock, decode: Timestamp) -> Option { // Signed, since a frame can decode before the one that anchored the clock. let since = decode.as_nanos() as i128 - clock.base.as_nanos() as i128; - let offset = since + since * clock.drift / 1_000_000_000 + self.delay.as_nanos() as i128; + let offset = since + (since as f64 * clock.drift) as i128 + self.delay.as_nanos() as i128; let nanos = u64::try_from(offset.unsigned_abs()).ok()?; match offset >= 0 { true => clock.anchor.checked_add(Duration::from_nanos(nanos)), @@ -318,11 +370,23 @@ impl Buffer { self.tracks.clear(); self.horizon = None; self.released = false; - self.slack = None; + self.steer = Steer::default(); self.timer = None; } } +/// The least-squares slope of `points`, if they span any time. +fn slope(points: &VecDeque<(f64, f64)>) -> Option { + let n = points.len() as f64; + let (x, y) = points + .iter() + .fold((0.0, 0.0), |(x, y), (px, py)| (x + px / n, y + py / n)); + let (cov, var) = points.iter().fold((0.0, 0.0), |(cov, var), (px, py)| { + (cov + (px - x) * (py - y), var + (px - x).powi(2)) + }); + (var > 0.0).then(|| cov / var) +} + #[cfg(test)] mod tests { use super::*; @@ -372,15 +436,19 @@ mod tests { async fn releases_at_the_delay_in_decode_order() { let start = Instant::now(); let mut buffer = Buffer::new(DELAY); - assert_eq!(buffer.push(1, arrival(start, 0, "v0")), Push::Queued); - assert_eq!(buffer.push(1, arrival(start, 40, "v40")), Push::Queued); + assert_eq!(buffer.push(1, arrival(start, 0, "v0")).unwrap(), Push::Queued); + assert_eq!(buffer.push(1, arrival(start, 40, "v40")).unwrap(), Push::Queued); // Audio arrives later than the video, with an earlier decode time. assert_eq!( - buffer.push(2, arrival(start + Duration::from_millis(30), 20, "a20")), + buffer + .push(2, arrival(start + Duration::from_millis(30), 20, "a20")) + .unwrap(), Push::Queued ); assert_eq!( - buffer.push(2, arrival(start + Duration::from_millis(30), 40, "a40")), + buffer + .push(2, arrival(start + Duration::from_millis(30), 40, "a40")) + .unwrap(), Push::Queued ); @@ -412,11 +480,13 @@ mod tests { // One sees audio lag by 90ms, the other sees every frame on time. let skew = if key == 2 { 90 } else { 0 }; assert_eq!( - early.push(key, arrival(start + Duration::from_millis(decode), decode, item)), + early + .push(key, arrival(start + Duration::from_millis(decode), decode, item)) + .unwrap(), Push::Queued ); let arrived = start + Duration::from_millis(decode + skew); - assert_eq!(late.push(key, arrival(arrived, decode, item)), Push::Queued); + assert_eq!(late.push(key, arrival(arrived, decode, item)).unwrap(), Push::Queued); } tokio::time::advance(Duration::from_secs(1)).await; let order = due(&mut early); @@ -430,17 +500,17 @@ mod tests { async fn late_frames_are_dropped_until_a_sync_frame() { let start = Instant::now(); let mut buffer = Buffer::new(DELAY); - buffer.push(1, arrival(start, 0, "v0")); - buffer.push(2, arrival(start, 0, "a0")); + buffer.push(1, arrival(start, 0, "v0")).unwrap(); + buffer.push(2, arrival(start, 0, "a0")).unwrap(); let late = start + Duration::from_millis(200); let video = |decode, sync, item| Arrival { sync, ..arrival(late, decode, item) }; - assert_eq!(buffer.push(1, video(40, false, "v40")), Push::Late); - assert_eq!(buffer.push(1, video(200, false, "v200")), Push::Waiting); - assert_eq!(buffer.push(1, video(240, true, "v240")), Push::Queued); - assert_eq!(buffer.push(2, arrival(late, 180, "a180")), Push::Queued); + assert_eq!(buffer.push(1, video(40, false, "v40")).unwrap(), Push::Late); + assert_eq!(buffer.push(1, video(200, false, "v200")).unwrap(), Push::Waiting); + assert_eq!(buffer.push(1, video(240, true, "v240")).unwrap(), Push::Queued); + assert_eq!(buffer.push(2, arrival(late, 180, "a180")).unwrap(), Push::Queued); assert_eq!(buffer.dropped(), 2); tokio::time::advance(Duration::from_secs(1)).await; @@ -453,13 +523,19 @@ mod tests { let start = Instant::now(); let mut buffer = Buffer::new(DELAY); // The video's first group is stale and anchors the clock; the audio is live. - buffer.push(1, arrival(start, 0, "v0")); - buffer.push(2, arrival(start, 900, "a900")); + buffer.push(1, arrival(start, 0, "v0")).unwrap(); + buffer.push(2, arrival(start, 900, "a900")).unwrap(); // The video consumer skips to the live group. let skip = start + Duration::from_millis(5); - assert_eq!(buffer.push(1, after(1, arrival(skip, 900, "v900"))), Push::Queued); - assert_eq!(buffer.push(1, after(1, arrival(skip, 940, "v940"))), Push::Queued); - assert_eq!(buffer.push(2, arrival(skip, 940, "a940")), Push::Queued); + assert_eq!( + buffer.push(1, after(1, arrival(skip, 900, "v900"))).unwrap(), + Push::Queued + ); + assert_eq!( + buffer.push(1, after(1, arrival(skip, 940, "v940"))).unwrap(), + Push::Queued + ); + assert_eq!(buffer.push(2, arrival(skip, 940, "a940")).unwrap(), Push::Queued); tokio::time::advance(Duration::from_secs(2)).await; assert_eq!( @@ -474,21 +550,27 @@ mod tests { async fn every_track_joins_a_new_generation_at_its_next_frame() { let start = Instant::now(); let mut buffer = Buffer::new(DELAY); - buffer.push(1, arrival(start, 0, "v0")); - buffer.push(2, arrival(start, 0, "a0")); - buffer.push(3, arrival(start, 0, "d0")); + buffer.push(1, arrival(start, 0, "v0")).unwrap(); + buffer.push(2, arrival(start, 0, "a0")).unwrap(); + buffer.push(3, arrival(start, 0, "d0")).unwrap(); tokio::time::advance(DELAY).await; assert_eq!(due(&mut buffer), ["v0", "a0", "d0"]); // The publisher resumes a second later with its timeline five seconds on. let resume = start + Duration::from_secs(1); tokio::time::advance(resume - Instant::now()).await; - assert_eq!(buffer.push(1, after(1, arrival(resume, 5_000, "v5000"))), Push::Queued); + assert_eq!( + buffer.push(1, after(1, arrival(resume, 5_000, "v5000"))).unwrap(), + Push::Queued + ); // The audio crosses no restart of its own; the data crosses the same one. let late = resume + Duration::from_millis(10); - assert_eq!(buffer.push(2, arrival(late, 5_000, "a5000")), Push::Queued); - assert_eq!(buffer.push(3, after(1, arrival(late, 5_000, "d5000"))), Push::Queued); - assert_eq!(buffer.push(2, arrival(late, 5_020, "a5020")), Push::Queued); + assert_eq!(buffer.push(2, arrival(late, 5_000, "a5000")).unwrap(), Push::Queued); + assert_eq!( + buffer.push(3, after(1, arrival(late, 5_000, "d5000"))).unwrap(), + Push::Queued + ); + assert_eq!(buffer.push(2, arrival(late, 5_020, "a5020")).unwrap(), Push::Queued); tokio::time::advance(DELAY).await; assert_eq!(released(&mut buffer), [("v5000", 1), ("a5000", 1), ("d5000", 1)]); @@ -502,7 +584,7 @@ mod tests { async fn a_skip_keeps_the_clock() { let start = Instant::now(); let mut buffer = Buffer::new(DELAY); - buffer.push(1, arrival(start, 0, "a0")); + buffer.push(1, arrival(start, 0, "a0")).unwrap(); tokio::time::advance(DELAY).await; assert_eq!(due(&mut buffer), ["a0"]); @@ -513,8 +595,8 @@ mod tests { skip: 1, ..arrival(stalled, decode, item) }; - assert_eq!(buffer.push(1, skipped(200, "a200")), Push::Late); - assert_eq!(buffer.push(1, skipped(300, "a300")), Push::Queued); + assert_eq!(buffer.push(1, skipped(200, "a200")).unwrap(), Push::Late); + assert_eq!(buffer.push(1, skipped(300, "a300")).unwrap(), Push::Queued); tokio::time::advance(Duration::from_millis(100)).await; assert_eq!(released(&mut buffer), [("a300", 0)]); } @@ -526,7 +608,7 @@ mod tests { async fn a_skip_that_leaps_ahead_opens_a_generation() { let start = Instant::now(); let mut buffer = Buffer::new(DELAY); - buffer.push(1, arrival(start, 0, "a0")); + buffer.push(1, arrival(start, 0, "a0")).unwrap(); tokio::time::advance(DELAY).await; assert_eq!(due(&mut buffer), ["a0"]); @@ -535,7 +617,7 @@ mod tests { skip: 1, ..arrival(now, 5_000, "a5000") }; - assert_eq!(buffer.push(1, leapt), Push::Queued); + assert_eq!(buffer.push(1, leapt).unwrap(), Push::Queued); tokio::time::advance(DELAY).await; assert_eq!(released(&mut buffer), [("a5000", 1)]); } @@ -546,17 +628,17 @@ mod tests { async fn generations_go_out_in_turn() { let start = Instant::now(); let mut buffer = Buffer::new(DELAY); - buffer.push(1, arrival(start, 0, "v0")); - buffer.push(1, arrival(start, 40, "v40")); - buffer.push(2, arrival(start, 20, "a20")); + buffer.push(1, arrival(start, 0, "v0")).unwrap(); + buffer.push(1, arrival(start, 40, "v40")).unwrap(); + buffer.push(2, arrival(start, 20, "a20")).unwrap(); tokio::time::advance(DELAY).await; assert_eq!(due(&mut buffer), ["v0"]); // A backlog across a jump in the timeline, read in one go. let now = Instant::now(); - buffer.push(1, after(1, arrival(now, 5_000, "v5000"))); - buffer.push(2, arrival(now, 4_900, "a4900")); - buffer.push(2, arrival(now, 5_020, "a5020")); + buffer.push(1, after(1, arrival(now, 5_000, "v5000"))).unwrap(); + buffer.push(2, arrival(now, 4_900, "a4900")).unwrap(); + buffer.push(2, arrival(now, 5_020, "a5020")).unwrap(); tokio::time::advance(Duration::from_secs(1)).await; assert_eq!( released(&mut buffer), @@ -564,32 +646,72 @@ mod tests { ); } - /// A source whose clock runs off ours, slower or faster, neither goes late nor piles up: - /// two hours at 200 ppm is 1.44 s, over ten times the delay. + /// Feed `hours` of 5 fps frames from a source whose clock runs `ppm` slower than ours + /// through a 500 ms buffer, checking the clock stays within what 13818-1 allows, until a + /// push fails. Returns that error and the most frames the buffer held. + async fn drift(ppm: i128, hours: u64) -> (Option, usize) { + let start = Instant::now(); + let mut buffer = Buffer::new(Duration::from_millis(500)); + let mut most = 0; + let mut last = (0.0, ms(0)); + for k in 0..hours * 3_600 * 5 { + let decode = Duration::from_millis(k * 200); + let source = decode.as_nanos() as i128 * (1_000_000 + ppm) / 1_000_000; + let arrived = start + Duration::from_nanos(source as u64); + tokio::time::advance(arrived.saturating_duration_since(Instant::now())).await; + let frame = Arrival { + arrived, + decode: Timestamp::from_micros(decode.as_micros() as u64).unwrap(), + restart: 0, + skip: 0, + sync: true, + item: "v", + }; + match buffer.push(1, frame) { + Ok(push) => assert_eq!(push, Push::Queued, "{ppm} ppm: frame {k} was late"), + Err(err) => return (Some(err), most), + } + most = most.max(buffer.tracks[&1].queue.len()); + due(&mut buffer); + + let clock = buffer.clock.unwrap(); + let elapsed = (clock.base.as_nanos() - last.1.as_nanos()) as f64 / 1e9; + assert!(clock.drift.abs() <= MAX_DRIFT, "{ppm} ppm: drift {}", clock.drift); + assert!( + (clock.drift - last.0).abs() <= MAX_SLEW * elapsed * (1.0 + 1e-9), + "{ppm} ppm: drift slewed from {} to {} in {elapsed} s", + last.0, + clock.drift + ); + last = (clock.drift, clock.base); + } + let drift = buffer.clock.unwrap().drift; + assert!( + (drift * 1e6 - ppm as f64).abs() < 0.5, + "{ppm} ppm: the clock settled at {} ppm", + drift * 1e6 + ); + (None, most) + } + + /// A source whose clock runs off ours, slower or faster, by up to the 30 ppm 13818-1 allows + /// neither goes late nor piles up, and the clock settles at its rate. #[tokio::test(start_paused = true)] async fn the_clock_follows_a_drifting_source() { - for ppm in [200i128, -200] { - let start = Instant::now(); - let mut buffer = Buffer::new(DELAY); - let mut most = 0; - for k in 0..2 * 3_600 * 25u64 { - let decode = Duration::from_millis(k * 40); - let source = decode.as_nanos() as i128 * (1_000_000 + ppm) / 1_000_000; - let arrived = start + Duration::from_nanos(source as u64); - tokio::time::advance(arrived.saturating_duration_since(Instant::now())).await; - let frame = Arrival { - arrived, - decode: Timestamp::from_micros(decode.as_micros() as u64).unwrap(), - restart: 0, - skip: 0, - sync: true, - item: "v", - }; - assert_eq!(buffer.push(1, frame), Push::Queued, "{ppm} ppm: frame {k} was late"); - most = most.max(buffer.tracks[&1].queue.len()); - due(&mut buffer); - } - assert!(most <= 10, "{ppm} ppm: the buffer grew to {most} frames"); + for ppm in [25, -25] { + let (err, most) = drift(ppm, 10).await; + assert!(err.is_none(), "{ppm} ppm: {err:?}"); + assert!(most <= 5, "{ppm} ppm: the buffer grew to {most} frames"); + } + } + + /// A source further off than the clock may follow fails once it has used half the delay, + /// before any frame goes late. + #[tokio::test(start_paused = true)] + async fn a_source_past_the_drift_limit_fails() { + for ppm in [40, -40] { + let (err, _) = drift(ppm, 6).await; + assert!(err.is_some(), "{ppm} ppm: no error"); } } @@ -599,13 +721,13 @@ mod tests { async fn zero_delay_releases_on_arrival() { let start = Instant::now(); let mut buffer = Buffer::new(Duration::ZERO); - buffer.push(1, arrival(start, 0, "v0")); - buffer.push(1, arrival(start, 40, "v40")); - buffer.push(2, arrival(start, 20, "a20")); + buffer.push(1, arrival(start, 0, "v0")).unwrap(); + buffer.push(1, arrival(start, 40, "v40")).unwrap(); + buffer.push(2, arrival(start, 20, "a20")).unwrap(); assert_eq!(due(&mut buffer), ["v0", "a20", "v40"]); let late = start + Duration::from_secs(1); - assert_eq!(buffer.push(2, arrival(late, 40, "a40")), Push::Queued); + assert_eq!(buffer.push(2, arrival(late, 40, "a40")).unwrap(), Push::Queued); assert_eq!(due(&mut buffer), ["a40"]); assert_eq!(buffer.dropped(), 0); } @@ -614,13 +736,13 @@ mod tests { async fn clear_starts_a_fresh_clock() { let start = Instant::now(); let mut buffer = Buffer::new(DELAY); - buffer.push(1, arrival(start, 5_000, "old")); + buffer.push(1, arrival(start, 5_000, "old")).unwrap(); buffer.clear(); assert!(buffer.is_empty()); tokio::time::advance(Duration::from_secs(1)).await; let now = Instant::now(); - assert_eq!(buffer.push(1, arrival(now, 0, "new")), Push::Queued); + assert_eq!(buffer.push(1, arrival(now, 0, "new")).unwrap(), Push::Queued); tokio::time::advance(DELAY).await; assert_eq!(due(&mut buffer), ["new"]); } diff --git a/test/ts/README.md b/test/ts/README.md index ad5d2367c25..83074bdcc16 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -220,6 +220,12 @@ A constant-rate stream makes a fourth claim, graded by `pcr-schedule`: that the bytes between consecutive PCRs are the bytes the mux rate implies for that interval. +From a pipe, `pcr-rate` also grades the PCR clock's rate against the arrival +clock, as the least-squares slope over each time base after the start-up third: +within 30 ppm (`--rate-ppm`), the tolerance ISO 13818-1 2.4.2.1 gives a system +clock. `release` cannot see a clock running at a steady wrong rate, which keeps +its intervals and only adds to the drift. + ```bash # live: every domain, reading the exporter directly moq --connect http://localhost:4443 --broadcast live.hang export ts \ @@ -234,10 +240,11 @@ because every check is graded against the stream's **own** PCR values. If two consecutive PCRs are 25 ms apart in value then they must be ~25 ms apart in arrival, whatever clock rate the stream is running at. The price of that basis is the same one `compliance.py` pays: a PCR emitted at the wrong rate stays -internally consistent, so absolute rate is not what this grades. `pcr-schedule` -is the exception when it is given `--mux-rate`, which pins the rate the way -`duration-fidelity` does; without it, it estimates the rate from the capture and -grades only how evenly the bytes are laid over the PCRs. +internally consistent, so absolute rate is not what this grades, except against +the arrival clock in `pcr-rate`. `pcr-schedule` is the other exception when it is +given `--mux-rate`, which pins the rate the way `duration-fidelity` does; without +it, it estimates the rate from the capture and grades only how evenly the bytes +are laid over the PCRs. | Check | Severity | What it verifies | |---|---|---| diff --git a/test/ts/pcr-timing.py b/test/ts/pcr-timing.py index f835df10bcb..0a5543cc2fd 100755 --- a/test/ts/pcr-timing.py +++ b/test/ts/pcr-timing.py @@ -13,6 +13,8 @@ and a constant-rate stream makes a fourth, which `schedule` grades: that the bytes between consecutive PCRs are the bytes the mux rate implies for that interval. +`rate` (--live only) grades the clock's rate against arrival, within the 30 ppm +ISO 13818-1 2.4.2.1 allows. Every check is an invariant on the stream, not an assertion about how the stream was produced. `release` grades arrival against the PCR's *own* values rather than @@ -437,6 +439,48 @@ def check_release(scan, args): ) +def check_rate(scan, args): + """The PCR clock must run within 30 ppm of the clock its bytes are released on. + + ISO/IEC 13818-1 2.4.2.1 holds a system clock to 27 MHz +/- 810 Hz, 30 ppm. A sender + that steers its clock after a drifting source may use that much and no more, and a + receiver locks to the rate it sees the PCR arrive at. `release` cannot see this: a + clock running at a steady wrong rate keeps its intervals and only adds to the drift. + The rate is the least-squares slope of PCR value against arrival over each time base, + after the first third of the sample (the start-up fill), which averages the per-PCR + release jitter down to a few ppm. + """ + stamped = [i for i, (_, _, arrival, _) in enumerate(scan.pcr) if arrival is not None] + if len(stamped) < 3: + return ("pcr-rate", HARD, True, "not measured (no arrival stamps)", {}) + cut = len(stamped) // 3 + segments = [[]] + for i in stamped[cut:]: + if i in scan.new_base: + segments.append([]) + segments[-1].append((scan.pcr[i][2], scan.pcr[i][1] / (TICKS_PER_MS * 1000.0))) + rates = [] + for points in segments: + if len(points) < 3 or points[-1][0] - points[0][0] < 1.0: + continue + mx = statistics.fmean(x for x, _ in points) + my = statistics.fmean(y for _, y in points) + var = sum((x - mx) ** 2 for x, _ in points) + cov = sum((x - mx) * (y - my) for x, y in points) + rates.append((cov / var - 1.0) * 1e6) + if not rates: + return ("pcr-rate", HARD, False, "no gradable span (each time base under a second)", {}) + worst = max(rates, key=abs) + detail = {"ppm": [round(r, 2) for r in rates], "limit_ppm": args.rate_ppm} + return ( + "pcr-rate", + HARD, + abs(worst) <= args.rate_ppm, + f"PCR clock {worst:+.2f} ppm against arrival (limit ±{args.rate_ppm:g} ppm, ISO 13818-1 2.4.2.1)", + detail, + ) + + def check_position(scan, args): """A PCR packet must sit among the media bytes whose arrival it describes. @@ -579,7 +623,7 @@ def check_schedule(scan, args): # A file is graded for sync and continuity by compliance.py, through TSDuck's tsanalyze. # A pipe cannot be: tsp in front of it would rebuffer the very arrivals `release` stamps. LIVE_CHECKS = [check_sync, check_continuity] -CHECKS = [check_value_interval, check_release, check_position, check_schedule] +CHECKS = [check_value_interval, check_release, check_rate, check_position, check_schedule] def coincidence(scan, args): @@ -642,6 +686,12 @@ def main(): "hold; set it to the sender's latency budget (moq export ts --delay, " "itself 500ms by default)", ) + ap.add_argument( + "--rate-ppm", + type=float, + default=30.0, + help="how far the PCR clock may run off the arrival clock, in ppm (default 30, ISO 13818-1 2.4.2.1)", + ) ap.add_argument( "--adjacent-packets", type=int, default=1, help="packet gap at or below which two PCRs count as clustered" ) From e8556728cf93fa61596e37857f21d265703ef85f Mon Sep 17 00:00:00 2001 From: t0ms Date: Thu, 1 Oct 2026 19:29:34 +0100 Subject: [PATCH 27/40] test(moq-mux): mocked-time export tests for join skips, source drift, the system clock and 1+1 identity Live H.264 + AAC broadcast on the paused clock, frames written at the instant a source sends them, receivers polling on their own cadence. Two cases fail on 49efbc9a1 and are ignored with the reason: a mid-group joiner's system clock leaves ISO 13818-1's 30 ppm while it wears away the join's lead, and a 1+1 pair diverges after a skip. --- .../src/container/ts/export_timing_test.rs | 707 ++++++++++++++++++ rs/moq-mux/src/container/ts/mod.rs | 2 + 2 files changed, 709 insertions(+) create mode 100644 rs/moq-mux/src/container/ts/export_timing_test.rs diff --git a/rs/moq-mux/src/container/ts/export_timing_test.rs b/rs/moq-mux/src/container/ts/export_timing_test.rs new file mode 100644 index 00000000000..daf643310a4 --- /dev/null +++ b/rs/moq-mux/src/container/ts/export_timing_test.rs @@ -0,0 +1,707 @@ +//! Timing of a delayed [`Export`] on a live clock: what a receiver renders when it joins a +//! broadcast already running, when one track loses frames, when the source's clock runs off the +//! receiver's, and when two receivers render one broadcast for a 1+1 (SMPTE ST 2022-7) pair. +//! +//! Every case runs on the paused clock, with each frame written at the instant a live source +//! would send it. A receiver's view is measured from its own output: the lead of each PES (its +//! decode time less the PCR of the slot carrying it) and the instant each slot went out. + +use std::io::Cursor; +use std::time::Duration; + +use bytes::{Bytes, BytesMut}; +use hang::catalog::{AAC, AudioConfig, Container, H264, VideoConfig}; +use moq_net::Timestamp; +use mpeg2ts::ts::{ReadTsPacket, TsPacketReader, TsPayload}; +use tokio::time::Instant; + +use crate::catalog::hang::Container as HangContainer; +use crate::container::ts::Export; +use crate::container::ts::export::PCR_INTERVAL; +use crate::container::{Container as _, Frame, Producer}; + +const SPS: &[u8] = &[0x67, 0x42, 0xc0, 0x1f, 0xde]; +const PPS: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; + +/// 25 fps video. +const VIDEO_US: u64 = 40_000; +/// 48 kHz AAC, 1024 samples per frame. +const AUDIO_US: u64 = 21_333; +/// Video frames per group: 600 ms. +const GOP: u64 = 15; +/// Audio frames per group, roughly matching the video group duration. +const AUDIO_GROUP: u64 = 28; + +const DELAY: Duration = Duration::from_millis(500); +/// 34 packets in every 25 ms slot: room for the first slot's keyframe and tables, which have no +/// earlier slot to spread into, and a whole number of packets per slot, so that only +/// [`two_legs_pad_the_same_slots_at_a_fractional_rate`] depends on how a slot's share of a +/// fractional packet is carried. +const MUX_RATE: u64 = 34 * 188 * 8 * 40; + +/// The tick a late receiver joins at: eight frames into a group, so its subscription starts +/// with 320 ms of that group already published and delivered at once. +const JOIN: u64 = 3 * GOP + 8; + +const VIDEO_PID: u16 = 0x1001; +const AUDIO_PID: u16 = 0x1002; + +fn length_prefixed(nals: &[&[u8]]) -> Bytes { + let mut out = BytesMut::new(); + for nal in nals { + out.extend_from_slice(&(nal.len() as u32).to_be_bytes()); + out.extend_from_slice(nal); + } + out.freeze() +} + +/// A live H.264 + AAC broadcast whose frames are written on the source's clock. +struct Live { + _broadcast: moq_net::broadcast::Producer, + _catalog: crate::catalog::Producer, + /// Written group by group, so a group can be left open: see [`Self::run`]. + video: moq_net::track::Producer, + group: Option, + /// Whether the open group has stalled, and the stalled groups, kept open. + stalling: bool, + stalled: Vec, + audio: Producer, + source: crate::Source, + /// Media time per unit of the receivers' clock: above 1 the source runs fast. + scale: f64, + /// The multiplex rate the receivers pad to. + rate: u64, + /// How much later than its video the source sends audio of the same media time, in µs; + /// negative sends the video later. A TS source sends video well ahead of its decode time + /// and audio just in time, so its audio lags. + audio_lag: i64, + start: Instant, + tick: u64, + audio_index: u64, +} + +impl Live { + fn new(scale: f64, audio_lag: i64) -> Self { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + + let track = broadcast + .create_track( + broadcast.unique_name(".avc1"), + hang::container::track_info(hang::catalog::PRIORITY.video), + ) + .unwrap(); + let mut cfg = VideoConfig::new(H264 { + profile: 0x42, + constraints: 0xc0, + level: 0x1f, + inline: false, + }); + cfg.container = Container::Legacy; + cfg.description = + Some(crate::codec::h264::build_avcc(&[Bytes::from_static(SPS)], &[Bytes::from_static(PPS)]).unwrap()); + catalog + .modify() + .unwrap() + .video + .renditions + .insert(track.name().to_string(), cfg); + let video = track; + + let track = broadcast + .create_track( + broadcast.unique_name(".aac"), + hang::container::track_info(hang::catalog::PRIORITY.audio), + ) + .unwrap(); + let mut cfg = AudioConfig::new(AAC { profile: 2 }, 48_000, 2); + cfg.container = Container::Legacy; + catalog + .modify() + .unwrap() + .audio + .renditions + .insert(track.name().to_string(), cfg); + let audio = Producer::new(track, HangContainer::Legacy(crate::container::Kind::Audio)); + + Self { + source: crate::source::announced(&consumer), + _broadcast: broadcast, + _catalog: catalog, + video, + group: None, + stalling: false, + stalled: Vec::new(), + audio, + scale, + rate: MUX_RATE, + audio_lag, + start: Instant::now(), + tick: 0, + audio_index: 0, + } + } + + /// The receivers' instant at which the source's clock reads media time `micros`. + fn sent(&self, micros: u64) -> Instant { + self.start + Duration::from_secs_f64(micros as f64 / 1e6 / self.scale) + } + + /// When the source sends the video frame of media time `video` and the audio frame of + /// `audio`: the track that lags is sent late, the other on the source's clock. + fn send_times(&self, video: u64, audio: u64) -> (Instant, Instant) { + let lag = Duration::from_micros(self.audio_lag.unsigned_abs()); + let (v, a) = match self.audio_lag >= 0 { + true => (Duration::ZERO, lag), + false => (lag, Duration::ZERO), + }; + (self.sent(video) + v, self.sent(audio) + a) + } + + /// A receiver joining now. + async fn join(&self, every: Duration) -> Leg { + let export = Export::new(self.source.clone()) + .await + .unwrap() + .with_delay(DELAY) + .with_mux_rate(self.rate); + Leg { + export, + every, + next_poll: Instant::now(), + out: Vec::new(), + } + } + + /// Send every frame before video tick `until`, each at its instant on the source's clock, + /// polling the legs as they would poll on their own. + /// + /// A video tick `stall` accepts is never sent, and its group is left open: its tail is + /// still on its way when the next group starts, as when the relay's backlog for a joiner + /// arrives behind the live edge. The subscription then skips it as slow once the next + /// group is the delay newer, and counts a discontinuity. + async fn run(&mut self, until: u64, legs: &mut [&mut Leg], stall: impl Fn(u64) -> bool) { + while self.tick < until { + let video = self.tick * VIDEO_US; + let audio = self.audio_index * AUDIO_US; + let (video_at, audio_at) = self.send_times(video, audio); + advance(video_at.min(audio_at), legs).await; + if audio_at < video_at { + let index = self.audio_index; + self.audio + .write(Frame { + timestamp: Timestamp::from_micros(audio).unwrap(), + duration: None, + payload: Bytes::from_iter((0..180u16).map(|i| (i ^ index as u16) as u8)), + keyframe: index.is_multiple_of(AUDIO_GROUP), + }) + .unwrap(); + self.audio_index += 1; + } else { + let tick = self.tick; + let keyframe = tick.is_multiple_of(GOP); + if keyframe && let Some(group) = self.group.take() { + match std::mem::take(&mut self.stalling) { + true => self.stalled.push(group), + false => group.finish().unwrap(), + } + } + if keyframe { + self.group = Some(self.video.append_group().unwrap()); + } + if stall(tick) { + self.stalling = true; + } else if !self.stalling { + let slice = if keyframe { + vec![0x65u8; 3_000] + } else { + vec![0x41u8; 400] + }; + let group = self.group.as_mut().expect("the first tick is a keyframe"); + HangContainer::Legacy(crate::container::Kind::Video) + .write( + group, + &[Frame { + timestamp: Timestamp::from_micros(video).unwrap(), + duration: None, + payload: length_prefixed(&[&slice]), + keyframe, + }], + ) + .unwrap(); + } + self.tick += 1; + } + for leg in legs.iter_mut() { + leg.poll_now(); + } + } + } + + /// End the broadcast and let every leg render what it holds. + async fn finish(&mut self, legs: &mut [&mut Leg]) { + if let Some(group) = self.group.take() { + group.finish().unwrap(); + } + self.stalled.clear(); + self.video.finish().unwrap(); + self.audio.finish().unwrap(); + for leg in legs.iter_mut() { + leg.drain().await; + } + } +} + +/// One receiver: an exporter and what it rendered, with the instant each slot went out. +struct Leg { + export: Export, + /// How often it reads its subscription. Zero reads after every frame the source sends; a + /// coarser cadence stands for a receiver behind a burstier network or scheduler. + every: Duration, + next_poll: Instant, + out: Vec<(Instant, Frame)>, +} + +impl Leg { + /// When this leg next reads on its own. + fn wake(&self) -> Option { + match self.every.is_zero() { + true => self.export.next_due(), + false => Some(self.next_poll), + } + } + + fn poll_now(&mut self) { + if !self.every.is_zero() { + return; + } + self.poll(); + } + + fn poll(&mut self) { + let waiter = kio::Waiter::noop(); + let now = Instant::now(); + while let std::task::Poll::Ready(frame) = self.export.poll_next(&waiter) { + match frame.expect("exporter error") { + Some(frame) => self.out.push((now, frame)), + None => break, + } + } + } + + /// Read to the end of the broadcast. + async fn drain(&mut self) { + loop { + self.poll(); + if let Some(deadline) = self.export.next_due() { + tokio::time::sleep_until(deadline).await; + continue; + } + match tokio::time::timeout(Duration::from_millis(10), self.export.next()).await { + Ok(Ok(Some(frame))) => self.out.push((Instant::now(), frame)), + Ok(Ok(None)) => break, + Err(_) if self.export.next_due().is_none() => break, + Err(_) => {} + Ok(Err(err)) => panic!("exporter error: {err}"), + } + } + } + + /// The PES timing this leg rendered: one entry per PES start. + fn units(&self) -> Vec { + units(&self.out) + } +} + +/// Run the clock to `at`, letting every leg read whenever it would have on its own. +async fn advance(at: Instant, legs: &mut [&mut Leg]) { + while let Some(wake) = legs + .iter() + .filter_map(|leg| leg.wake()) + .filter(|wake| *wake <= at) + .min() + { + tokio::time::sleep_until(wake).await; + let now = Instant::now(); + for leg in legs.iter_mut() { + if leg.wake().is_some_and(|wake| wake <= now) { + leg.poll(); + if !leg.every.is_zero() { + while leg.next_poll <= now { + leg.next_poll += leg.every; + } + } + } + } + } + tokio::time::sleep_until(at).await; + // Read up to the instant the next frame arrives, so the arrival is timed exactly. + for leg in legs.iter_mut() { + leg.poll_now(); + } +} + +/// A PES start as a receiver sees it. +#[derive(Debug, Clone, Copy)] +struct Unit { + pid: u16, + /// Decode time (DTS, else PTS), 90 kHz, as carried. + decode: u64, + /// Decode time less the PCR of the slot carrying the PES start, in milliseconds: how long + /// the unit waits in the receiver's buffer after its first byte arrives. + lead: f64, + /// When the slot carrying it went out. + sent: Instant, +} + +const WRAP: i64 = 1 << 33; + +/// Signed difference of two 33-bit 90 kHz values. +fn diff90(a: u64, b: u64) -> i64 { + let d = (a as i64 - b as i64).rem_euclid(WRAP); + if d >= WRAP / 2 { d - WRAP } else { d } +} + +fn units(out: &[(Instant, Frame)]) -> Vec { + // One reader across the slots, since it needs the tables an earlier slot carried. + let bytes: Vec = out + .iter() + .flat_map(|(_, frame)| frame.payload.iter().copied()) + .collect(); + let slot_of: Vec = out + .iter() + .flat_map(|(sent, frame)| std::iter::repeat_n(*sent, frame.payload.len() / 188)) + .collect(); + let mut reader = TsPacketReader::new(Cursor::new(bytes)); + let mut units = Vec::new(); + let mut pcr: Option = None; + for sent in &slot_of { + let packet = reader.read_ts_packet().unwrap().expect("a packet per slot entry"); + if let Some(clock) = packet.adaptation_field.as_ref().and_then(|af| af.pcr) { + pcr = Some(clock.as_u64() / 300); + } + if let Some(TsPayload::PesStart(pes)) = &packet.payload + && let Some(pts) = pes.header.pts + && let Some(pcr) = pcr + { + let decode = pes.header.dts.unwrap_or(pts).as_u64(); + units.push(Unit { + pid: packet.header.pid.as_u16(), + decode, + lead: diff90(decode, pcr) as f64 / 90.0, + sent: *sent, + }); + } + } + units +} + +fn median(mut values: Vec) -> f64 { + assert!(!values.is_empty(), "no units to take a median of"); + values.sort_by(f64::total_cmp); + values[values.len() / 2] +} + +/// The median lead of one PID's units decoding at or after `from` (90 kHz). +fn lead(units: &[Unit], pid: u16, from: u64) -> f64 { + median( + units + .iter() + .filter(|unit| unit.pid == pid && diff90(unit.decode, from) >= 0) + .map(|unit| unit.lead) + .collect(), + ) +} + +/// The median of (slot sent − decode time) over one PID's units decoding at or after `from` +/// (90 kHz), in milliseconds against an arbitrary origin: when the receiver releases a unit +/// against when it decodes. +fn release(units: &[Unit], pid: u16, from: u64) -> f64 { + let origin = units[0].sent; + median( + units + .iter() + .filter(|unit| unit.pid == pid && diff90(unit.decode, from) >= 0) + .map(|unit| unit.sent.duration_since(origin).as_secs_f64() * 1e3 - unit.decode as f64 / 90.0) + .collect(), + ) +} + +/// How much later than the audio the receiver releases video of the same decode time, in +/// milliseconds, over units decoding at or after `from` (90 kHz). Zero, to a slot, on one clock. +fn video_behind_audio(units: &[Unit], from: u64) -> f64 { + release(units, VIDEO_PID, from) - release(units, AUDIO_PID, from) +} + +/// The decode time `micros` after the leg's first video unit, 90 kHz. +fn after_first_video(units: &[Unit], micros: u64) -> u64 { + let first = units.iter().find(|unit| unit.pid == VIDEO_PID).expect("a video unit"); + (first.decode + at90(micros)) % WRAP as u64 +} + +/// `micros` in 90 kHz units. +fn at90(micros: u64) -> u64 { + micros * 9 / 100 +} + +const SLOT_MS: f64 = PCR_INTERVAL.as_millis() as f64; + +// A receiver that joins mid-group is handed that group's published frames at once, so its clock +// starts behind the live edge. When its subscription then skips a slow group on one track, that +// track must stay on the others' clock. In the field it did not: the skipped video opened a +// generation of its own and ran 200 ms behind, a whole delay behind or 1.2 s ahead of the audio +// depending on the join, fixed for the run, and in the last two the decoder buffers of whichever +// track ran behind underflowed. +// +// The split needs the tracks to be sent out of step, as a TS source's are (video well ahead of +// its decode time, audio just in time); sent in step, the new generation's anchor is clamped onto +// the old clock and nothing shows. So each case runs with each track lagging in turn. + +/// Send lags, in µs, each case runs with: audio late, then video late. +const LAGS: [i64; 2] = [300_000, -300_000]; + +/// How far apart, on one clock, two runs' video-against-audio release may sit: a slot either way +/// for the grid, so a rewind that lands the video one slot over still passes. +const CLOCK_SLACK: f64 = 2.0 * SLOT_MS; + +/// Join mid-group with `lag`, render 7.2 s with the video ticks `stall` accepts stalled, and +/// return the leg's units. +async fn join(lag: i64, stall: impl Fn(u64) -> bool) -> Vec { + let mut live = Live::new(1.0, lag); + live.run(JOIN, &mut [], |_| false).await; + let mut leg = live.join(Duration::ZERO).await; + live.run(24 * GOP, &mut [&mut leg], stall).await; + assert_eq!(leg.export.dropped(), 0, "every frame arrived inside its deadline"); + leg.units() +} + +/// Assert a skip left the video where it is without one, against the audio, from 4 s after the +/// leg's first video unit. +async fn assert_one_clock(stall: impl Fn(u64) -> bool + Copy) { + let mut moved = Vec::new(); + for lag in LAGS { + let (clean, skipped) = (join(lag, |_| false).await, join(lag, stall).await); + let from = after_first_video(&clean, 4_000_000); + let (want, got) = (video_behind_audio(&clean, from), video_behind_audio(&skipped, from)); + if (got - want).abs() > CLOCK_SLACK { + moved.push(format!( + "{} sent {} ms late: video released {:+.1} ms against audio of the same decode time, \ + {:+.1} ms without the skip (PCR lead video {:.1} ms, audio {:.1} ms)", + if lag > 0 { "audio" } else { "video" }, + lag.abs() / 1_000, + got, + want, + lead(&skipped, VIDEO_PID, from), + lead(&skipped, AUDIO_PID, from), + )); + } + } + assert!( + moved.is_empty(), + "the skip put the video on its own clock:\n{}", + moved.join("\n") + ); +} + +/// The joined group stalls straight after the join, before anything has been released: its +/// tail never arrives, and the subscription skips to the next group. +#[tokio::test(start_paused = true)] +async fn a_skip_at_the_join_keeps_one_clock() { + assert_one_clock(|tick| (JOIN..4 * GOP).contains(&tick)).await; +} + +/// The same skip well after the first release: half a group of video stalls mid-stream. +#[tokio::test(start_paused = true)] +async fn a_skip_while_running_keeps_one_clock() { + assert_one_clock(|tick| (6 * GOP + 7..7 * GOP).contains(&tick)).await; +} + +// A source's clock is never the receiver's. A transport stream's 27 MHz may be off by 30 ppm, +// which walks a fixed anchor 108 ms an hour: a source running slow makes every frame late in +// turn once the walk passes the delay, and one running fast makes the buffer grow without bound. +// These cases run the source 400 ppm off, past what it may be off but inside the 500 ppm the +// jitter buffer steers by, so that five minutes of media walk an unsteered clock 120 ms. + +/// How far the instant each slot goes out may wander against the source over a run. A +/// tracker may lag; what this rules out is the delay walking with the source clock. +const DRIFT_SLACK: Duration = Duration::from_millis(100); + +/// Five minutes of media. +const DRIFT_TICKS: u64 = 300_000_000 / VIDEO_US; + +/// The spread of (slot sent − source's clock at the slot) after the first 10 s, and the drops. +async fn on_a_scaled_clock(scale: f64) -> (Duration, u64) { + let mut live = Live::new(scale, 0); + let mut leg = live.join(Duration::ZERO).await; + live.run(DRIFT_TICKS, &mut [&mut leg], |_| false).await; + + let first = leg.out.first().expect("output").1.timestamp.as_micros() as u64; + let held: Vec = leg + .out + .iter() + .filter(|(_, frame)| frame.timestamp.as_micros() as u64 >= first + 10_000_000) + .map(|(sent, frame)| sent.saturating_duration_since(live.sent(frame.timestamp.as_micros() as u64))) + .collect(); + assert!(held.len() > 1_000, "too little output to judge: {}", held.len()); + let spread = *held.iter().max().unwrap() - *held.iter().min().unwrap(); + (spread, leg.export.dropped()) +} + +#[tokio::test(start_paused = true)] +async fn a_slow_source_clock_keeps_the_delay() { + let (spread, dropped) = on_a_scaled_clock(0.9996).await; + assert_eq!(dropped, 0, "frames went late as the source fell behind the anchor"); + assert!(spread <= DRIFT_SLACK, "the delay walked by {spread:?}"); +} + +#[tokio::test(start_paused = true)] +async fn a_fast_source_clock_keeps_the_delay() { + let (spread, dropped) = on_a_scaled_clock(1.0004).await; + assert_eq!(dropped, 0); + assert!(spread <= DRIFT_SLACK, "the delay walked by {spread:?}"); +} + +// A 1+1 pair is two receivers of one broadcast whose outputs a seamless switch compares packet +// for packet. They join at different times and read with different cadences, and from the moment +// they overlap they must render the same bytes. The continuity counters are the known exception +// (they number what each process sent); every other byte has to match. + +/// Assert two legs rendered the same slots from media time `from` (µs, the slots' own +/// timestamps) to the end, continuity counters aside. +fn assert_same_packets(a: &Leg, b: &Leg, from: u64) { + let pick = |leg: &Leg| -> Vec { + leg.out + .iter() + .map(|(_, frame)| frame.clone()) + .filter(|frame| frame.timestamp.as_micros() as u64 >= from) + .collect() + }; + let (a, b) = (pick(a), pick(b)); + assert!(b.len() > 40, "not enough overlap to be worth comparing: {}", b.len()); + assert_eq!(a.len(), b.len(), "the legs rendered a different number of slots"); + for (a, b) in a.iter().zip(&b) { + assert_eq!(a.timestamp, b.timestamp, "compared slots must be the same slot"); + assert_eq!( + a.payload.len(), + b.payload.len(), + "slot at {:?} rendered to a different size", + a.timestamp + ); + for (offset, (x, y)) in a.payload.iter().zip(b.payload.iter()).enumerate() { + // Only the low nibble of byte 3 is the counter. + assert!( + x == y || (offset % 188 == 3 && (x ^ y) & 0xf0 == 0), + "slot at {:?} differs outside the continuity counter: offset {offset}, {x:#04x} vs {y:#04x}", + a.timestamp, + ); + } + } +} + +/// Two legs of one broadcast on `scale` with `lag`: one there from the start and reading after +/// every frame, the other joining mid-group and reading every 100 ms. Both see the same `stall`. +/// Returns them with the slot time 2 s after the second joined, from which they must agree. +async fn pair(scale: f64, lag: i64, ticks: u64, stall: impl Fn(u64) -> bool + Copy) -> (Leg, Leg, u64) { + pair_at(MUX_RATE, scale, lag, ticks, stall).await +} + +async fn pair_at(rate: u64, scale: f64, lag: i64, ticks: u64, stall: impl Fn(u64) -> bool + Copy) -> (Leg, Leg, u64) { + let mut live = Live::new(scale, lag); + live.rate = rate; + let mut a = live.join(Duration::ZERO).await; + live.run(JOIN, &mut [&mut a], stall).await; + let mut b = live.join(Duration::from_millis(100)).await; + live.run(ticks, &mut [&mut a, &mut b], stall).await; + live.finish(&mut [&mut a, &mut b]).await; + let joined = b.out.first().expect("the joiner rendered").1.timestamp.as_micros() as u64; + (a, b, joined + 2_000_000) +} + +#[tokio::test(start_paused = true)] +async fn two_legs_render_the_same_packets() { + for lag in LAGS { + let (a, b, from) = pair(1.0, lag, 12 * GOP, |_| false).await; + assert_same_packets(&a, &b, from); + } +} + +/// Most multiplex rates are not a whole number of packets per slot (10 Mb/s is 166.2), so each +/// slot carries 166 or 167. Which slots carry the extra packet has to follow the media grid, not +/// when the leg started. +#[tokio::test(start_paused = true)] +async fn two_legs_pad_the_same_slots_at_a_fractional_rate() { + let (a, b, from) = pair_at(2_000_000, 1.0, 0, 12 * GOP, |_| false).await; + assert_same_packets(&a, &b, from); +} + +/// A slow group reaches both legs. Both have to skip it the same way. +#[tokio::test(start_paused = true)] +#[ignore = "fails on 49efbc9a1: after the skip each leg anchors its new generation on its own reads"] +async fn two_legs_render_a_skip_the_same_way() { + for lag in LAGS { + let (a, b, from) = pair(1.0, lag, 12 * GOP, |tick| (6 * GOP + 7..7 * GOP).contains(&tick)).await; + assert_same_packets(&a, &b, from); + } +} + +/// Whatever follows the source's clock has to follow it identically on both legs, though they +/// joined at different points on it. +#[tokio::test(start_paused = true)] +async fn two_legs_render_a_drifting_source_the_same_way() { + for scale in [0.9996, 1.0004] { + let (a, b, from) = pair(scale, 0, DRIFT_TICKS * 3 / 5, |_| false).await; + assert_same_packets(&a, &b, from); + } +} + +// A receiver locks its decoder's clock to the PCR, so the output's system clock is held to +// ISO 13818-1 2.4.2.1: 27 MHz within 810 Hz (30 ppm), changing by at most 0.075 Hz/s. Steering it, +// to follow the source or to wear away a lead, has to stay inside both. + +/// A receiver that joins a running broadcast mid-group keeps its output's system clock within +/// 30 ppm of the source's and slews it no faster than 0.075 Hz/s, over 30 s windows of slots. +#[tokio::test(start_paused = true)] +#[ignore = "fails on 49efbc9a1: the clock steers at 500 ppm to wear away the join's lead"] +async fn a_joiner_keeps_the_system_clock_in_tolerance() { + const WINDOW: u64 = 30_000_000; + let mut live = Live::new(1.0, 0); + live.run(JOIN, &mut [], |_| false).await; + let mut leg = live.join(Duration::ZERO).await; + live.run(JOIN + 300_000_000 / VIDEO_US, &mut [&mut leg], |_| false) + .await; + + // The instant each slot first went out, by its PCR time. + let mut first = std::collections::BTreeMap::new(); + for (sent, frame) in &leg.out { + first.entry(frame.timestamp.as_micros() as u64).or_insert(*sent); + } + let slots: Vec<(u64, Instant)> = first.into_iter().collect(); + // (window end in seconds of PCR time, the system clock's offset in ppm) + let mut rates = Vec::new(); + let mut from = 0; + for to in 0..slots.len() { + if slots[to].0 < slots[from].0 + WINDOW { + continue; + } + let pcr = (slots[to].0 - slots[from].0) as f64 / 1e6; + let wall = (slots[to].1 - slots[from].1).as_secs_f64(); + rates.push((slots[to].0 as f64 / 1e6, (pcr / wall - 1.0) * 1e6)); + from = to; + } + assert!(rates.len() >= 8, "too little output to judge: {} windows", rates.len()); + for (at, ppm) in &rates { + assert!( + ppm.abs() <= 30.0, + "the system clock ran {ppm:+.1} ppm off over the 30 s to {at:.0} s" + ); + } + for pair in rates.windows(2) { + let slew = (pair[1].1 - pair[0].1).abs() * 27.0 / (pair[1].0 - pair[0].0); + assert!( + slew <= 0.075, + "the system clock slewed {slew:.3} Hz/s at {:.0} s", + pair[1].0 + ); + } +} diff --git a/rs/moq-mux/src/container/ts/mod.rs b/rs/moq-mux/src/container/ts/mod.rs index b7fbb7443b1..181adcec497 100644 --- a/rs/moq-mux/src/container/ts/mod.rs +++ b/rs/moq-mux/src/container/ts/mod.rs @@ -41,4 +41,6 @@ pub mod stats; #[cfg(test)] mod export_test; #[cfg(test)] +mod export_timing_test; +#[cfg(test)] mod import_test; From 4f36c8b09adff8caf15f7ff2b76e6669e7cbe1bc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 15:12:10 -0700 Subject: [PATCH 28/40] fix(moq-mux): acquire the TS export's clock and recover it from floors Implements quest/m1/release-clock-recovery.md (from #4670). Acquire before release: at the start and at each new generation, the jitter buffer holds frames until a track starts its next group, or one of them falls due, and anchors the clock on the freshest frame (the one read least behind its decode time). What of a joined group is older than the delay is dropped without counting as late, so a mid-group joiner runs at the delay from its first output instead of carrying the group's age. Freshness is judged on when a frame was read, so frames read together do not look fresher than they were. Clock recovery: each 2 s window gives a floor, the most slack any frame arrived with, which queueing only lowers. The upper envelope of the floors over 10 min gives the source's rate (its slope at the mean) and the slack now, so a spell of queueing shorter than half the window moves neither. Steering stays within 30 ppm and 0.075 Hz/s. Export::stats reports the measured drift and how often it was out of tolerance. t0ms's drift tests ran a 400 ppm source the old controller followed; they now run 30 ppm sources and check the measured rate, and a 400 ppm source is counted. a_joiner_keeps_the_system_clock_in_tolerance passes. Co-Authored-By: t0ms Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 30 +- rs/moq-mux/src/container/ts/export.rs | 41 +- rs/moq-mux/src/container/ts/export_test.rs | 9 +- .../src/container/ts/export_timing_test.rs | 76 +- rs/moq-mux/src/jitter.rs | 709 +++++++++++++----- 5 files changed, 656 insertions(+), 209 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index f92c69f907a..743723041e6 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -362,19 +362,23 @@ groups fetchable, which the [HLS gateway](/bin/hls) depends on. `export --max-ag stalled group before skipping. Raising the first never delays playback. `export ts` takes `--delay` (default 500 ms) instead, and works like an SRT -receiver's latency. Every frame is muxed that long after its decode time, -with the clock started when the first frame arrived, so the output keeps the -source's pace and muxes all tracks in decode order whatever their arrival -skew: two exporters of one broadcast emit them in one order. A frame that -arrives later than its deadline is dropped, and video then resumes at its next -keyframe. The clock follows the source's, so a source whose clock runs a little -fast or slow neither goes late nor piles up over a long run, and a lead it -started with wears away. The output's PCR is that clock, so it keeps to what -ISO/IEC 13818-1 allows a system clock: within 30 ppm of the exporter's, its -rate changing by at most 0.075 Hz/s. Catching up with a source near that limit -takes hours, and one further off fails the export once it has used half the -delay. Each 25 ms -slice of the output is written when the clock reaches it. The delay is also how +receiver's latency. Every frame is muxed that long after its decode time, on +a clock that keeps the source's pace and muxes all tracks in decode order +whatever their arrival skew: two exporters of one broadcast emit them in one +order. The export joins at the newest group and holds its output until a +track starts its next group, or one of the frames it holds falls due, then +starts the clock on the freshest frame it saw. What of the joined group is +older than the delay is dropped, so a joiner runs at the delay from its first +output. A frame that arrives later than its deadline is dropped, and video +then resumes at its next keyframe. The clock follows the source's, measured +from the frames that arrive least delayed, so a source whose clock runs a +little fast or slow neither goes late nor piles up over a long run, and a +spell of network queueing does not move it. The output's PCR is that clock, +so it keeps to what ISO/IEC 13818-1 allows a system clock: within 30 ppm of +the exporter's, its rate changing by at most 0.075 Hz/s. Catching up with a +source near that limit takes hours; one further off is counted in the +export's stats, and fails the export once it has used half the delay. Each +25 ms slice of the output is written when the clock reaches it. The delay is also how stale a group may get before it is skipped. `--delay 0` holds nothing and drops nothing, writing frames in arrival order as they come. diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 96d10739f86..cd25204c77b 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -89,6 +89,21 @@ fn sanitize_mux_rate(rate: u64) -> Option { (1..=MAX_MUX_RATE).contains(&rate).then_some(rate) } +/// How an [`Export`]'s release clock is keeping up with the source. See [`Export::stats`]. +#[derive(Clone, Debug, Default, PartialEq)] +#[non_exhaustive] +pub struct ExportStats { + /// Frames dropped for arriving after their deadline, and the frames a video track then + /// dropped waiting for its next keyframe. + pub dropped: u64, + /// How fast the source's clock runs against ours, in parts per million (positive runs + /// fast), as last measured. `None` until there is a measurement. + pub drift: Option, + /// How many measurements found the source's clock further off ours than the 30 ppm an + /// MPEG-TS system clock may be (ISO/IEC 13818-1 2.4.2.1), which the output cannot follow. + pub out_of_tolerance: u64, +} + /// Subscribe to a broadcast and produce an MPEG-TS byte stream. /// /// Use [`next`](Self::next) to pull one [`Frame`] per PCR grid slot: its `payload` @@ -176,6 +191,8 @@ struct Pending { skip: u64, /// The earliest the frame could have arrived: when its source was last found empty. arrived: web_async::time::Instant, + /// When it was read: the latest it could have arrived. + read: web_async::time::Instant, } /// A frame waiting in the jitter buffer, with what it needs from the moment it was read. @@ -312,6 +329,7 @@ impl Track { restart, skip, arrived, + read, } = pending; let dts = dts.filter(|&dts| dts != to_ticks(frame.timestamp)); let decode = dts @@ -319,6 +337,7 @@ impl Track { .unwrap_or(frame.timestamp); let arrival = Arrival { arrived, + read, decode, restart, skip, @@ -734,16 +753,20 @@ impl Export { } /// Read each track from the oldest group the delay still reaches instead of the live - /// edge, for a test that writes a broadcast whole before exporting it. + /// edge, and anchor the clock on the first frame instead of acquiring it, for a test that + /// writes a broadcast whole before exporting it. #[cfg(test)] pub(super) fn with_replay(mut self) -> Self { self.replay = true; + self.jitter = jitter::Buffer::new(self.delay).replay(); self } /// Mux each frame this long after its decode time, like an SRT receiver's TSBPD. /// - /// The clock starts at the first frame's arrival, so the output keeps the source's + /// The clock is acquired before anything goes out: frames are held until a track starts + /// a new group, or for the delay at most, and the freshest of them sets the clock, so a + /// joiner runs at the delay from its first output. The output then keeps the source's /// pace and muxes every track in `(DTS, PID)` order whatever the arrival skew between /// them. A frame that arrives after its deadline is dropped, and a video track that /// dropped one resumes at its next keyframe. The delay is also each source's staleness @@ -757,6 +780,10 @@ impl Export { pub fn with_delay(mut self, delay: Duration) -> Self { self.delay = delay; self.jitter = jitter::Buffer::new(delay); + #[cfg(test)] + if self.replay { + self.jitter = jitter::Buffer::new(delay).replay(); + } self.schedule = Schedule::new(delay); self.schedule.set_rate(self.mux_rate); self @@ -1004,6 +1031,7 @@ impl Export { restart: track.source.restarts(), skip: track.source.skips(), arrived: track.empty, + read: web_async::time::Instant::now(), }; // A new timeline must reach the reset before tune-in alignment can drop it. if let Some(start) = video_start @@ -1272,6 +1300,15 @@ impl Export { self.jitter.dropped() } + /// How the release clock is keeping up with the source. + pub fn stats(&self) -> ExportStats { + ExportStats { + dropped: self.jitter.dropped(), + drift: self.jitter.drift().map(|drift| drift * 1e6), + out_of_tolerance: self.jitter.out_of_tolerance(), + } + } + /// When the next queued frame is due, or the next grid slot while media is queued or /// every source has finished and the tail runs out. #[cfg(test)] diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index ad5c423a3cf..a060ea0e3af 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -5307,7 +5307,8 @@ async fn a_heavy_passage_fills_the_slots_before_it() { .await .unwrap() .with_mux_rate(2_000_000) - .with_delay(Duration::from_millis(500)); + .with_delay(Duration::from_millis(500)) + .with_replay(); let frames = drain_frames(&mut export).await; let ts: Vec = frames.iter().flat_map(|f| f.payload.iter().copied()).collect(); assert_eq!(video_pes_timing(&ts).len(), 100, "every frame went out"); @@ -5826,7 +5827,11 @@ async fn declared_reorder_sizes_the_decode_clock_from_the_first_frame() { &[], ); let max_age = Duration::from_millis(500); - let (mut late, mut whole) = (rig.export(max_age).await, rig.export(max_age).await); + // Written as fast as it is read, so the broadcast is all there at once. + let (mut late, mut whole) = ( + rig.export(max_age).await.with_replay(), + rig.export(max_age).await.with_replay(), + ); let (mut out_late, mut out_whole) = (Vec::new(), Vec::new()); for tick in 0..TICKS / 2 { while rig.audio_before(tick + 2) { diff --git a/rs/moq-mux/src/container/ts/export_timing_test.rs b/rs/moq-mux/src/container/ts/export_timing_test.rs index daf643310a4..4501286579e 100644 --- a/rs/moq-mux/src/container/ts/export_timing_test.rs +++ b/rs/moq-mux/src/container/ts/export_timing_test.rs @@ -16,7 +16,7 @@ use mpeg2ts::ts::{ReadTsPacket, TsPacketReader, TsPayload}; use tokio::time::Instant; use crate::catalog::hang::Container as HangContainer; -use crate::container::ts::Export; +use crate::container::ts::{Export, ExportStats}; use crate::container::ts::export::PCR_INTERVAL; use crate::container::{Container as _, Frame, Producer}; @@ -520,21 +520,24 @@ async fn a_skip_while_running_keeps_one_clock() { // A source's clock is never the receiver's. A transport stream's 27 MHz may be off by 30 ppm, // which walks a fixed anchor 108 ms an hour: a source running slow makes every frame late in // turn once the walk passes the delay, and one running fast makes the buffer grow without bound. -// These cases run the source 400 ppm off, past what it may be off but inside the 500 ppm the -// jitter buffer steers by, so that five minutes of media walk an unsteered clock 120 ms. +// The output's clock may follow it only as fast as 13818-1 lets a system clock change, so +// reaching a source 30 ppm off takes hours (the jitter buffer's own tests run a day); these +// cases run twenty minutes of a source at the limit, long enough to measure its rate. -/// How far the instant each slot goes out may wander against the source over a run. A -/// tracker may lag; what this rules out is the delay walking with the source clock. -const DRIFT_SLACK: Duration = Duration::from_millis(100); +/// How far the instant each slot goes out may wander against the source over a run: what +/// reaching a source at the limit costs under the slew limit (30 ppm squared over twice +/// 0.075 Hz/s of 27 MHz), and a slot. +const DRIFT_SLACK: Duration = Duration::from_millis(162 + 25); -/// Five minutes of media. -const DRIFT_TICKS: u64 = 300_000_000 / VIDEO_US; +/// Twenty minutes of media. +const DRIFT_TICKS: u64 = 1_200_000_000 / VIDEO_US; -/// The spread of (slot sent − source's clock at the slot) after the first 10 s, and the drops. -async fn on_a_scaled_clock(scale: f64) -> (Duration, u64) { +/// The spread of (slot sent − source's clock at the slot) after the first 10 s over `ticks` +/// of media, the drops, and the export's stats at the end. +async fn on_a_scaled_clock(scale: f64, ticks: u64) -> (Duration, u64, ExportStats) { let mut live = Live::new(scale, 0); let mut leg = live.join(Duration::ZERO).await; - live.run(DRIFT_TICKS, &mut [&mut leg], |_| false).await; + live.run(ticks, &mut [&mut leg], |_| false).await; let first = leg.out.first().expect("output").1.timestamp.as_micros() as u64; let held: Vec = leg @@ -545,21 +548,34 @@ async fn on_a_scaled_clock(scale: f64) -> (Duration, u64) { .collect(); assert!(held.len() > 1_000, "too little output to judge: {}", held.len()); let spread = *held.iter().max().unwrap() - *held.iter().min().unwrap(); - (spread, leg.export.dropped()) + (spread, leg.export.dropped(), leg.export.stats()) } #[tokio::test(start_paused = true)] async fn a_slow_source_clock_keeps_the_delay() { - let (spread, dropped) = on_a_scaled_clock(0.9996).await; + let (spread, dropped, stats) = on_a_scaled_clock(1.0 - 30e-6, DRIFT_TICKS).await; assert_eq!(dropped, 0, "frames went late as the source fell behind the anchor"); assert!(spread <= DRIFT_SLACK, "the delay walked by {spread:?}"); + let drift = stats.drift.expect("a measurement"); + assert!((drift + 30.0).abs() < 0.5, "measured {drift} ppm"); } #[tokio::test(start_paused = true)] async fn a_fast_source_clock_keeps_the_delay() { - let (spread, dropped) = on_a_scaled_clock(1.0004).await; + let (spread, dropped, stats) = on_a_scaled_clock(1.0 + 30e-6, DRIFT_TICKS).await; assert_eq!(dropped, 0); assert!(spread <= DRIFT_SLACK, "the delay walked by {spread:?}"); + let drift = stats.drift.expect("a measurement"); + assert!((drift - 30.0).abs() < 0.5, "measured {drift} ppm"); +} + +/// A source 400 ppm off, past what the output's clock may follow, is measured and counted. +#[tokio::test(start_paused = true)] +async fn a_source_past_the_limit_is_counted() { + let (_, _, stats) = on_a_scaled_clock(1.0004, 300_000_000 / VIDEO_US).await; + let drift = stats.drift.expect("a measurement"); + assert!((drift - 400.0).abs() < 1.0, "measured {drift} ppm"); + assert!(stats.out_of_tolerance > 0, "not counted"); } // A 1+1 pair is two receivers of one broadcast whose outputs a seamless switch compares packet @@ -637,7 +653,7 @@ async fn two_legs_pad_the_same_slots_at_a_fractional_rate() { /// A slow group reaches both legs. Both have to skip it the same way. #[tokio::test(start_paused = true)] -#[ignore = "fails on 49efbc9a1: after the skip each leg anchors its new generation on its own reads"] +#[ignore = "1+1 (ST 2022-7) packet identity across a skip is deferred to an m2 follow-up"] async fn two_legs_render_a_skip_the_same_way() { for lag in LAGS { let (a, b, from) = pair(1.0, lag, 12 * GOP, |tick| (6 * GOP + 7..7 * GOP).contains(&tick)).await; @@ -650,7 +666,7 @@ async fn two_legs_render_a_skip_the_same_way() { #[tokio::test(start_paused = true)] async fn two_legs_render_a_drifting_source_the_same_way() { for scale in [0.9996, 1.0004] { - let (a, b, from) = pair(scale, 0, DRIFT_TICKS * 3 / 5, |_| false).await; + let (a, b, from) = pair(scale, 0, 180_000_000 / VIDEO_US, |_| false).await; assert_same_packets(&a, &b, from); } } @@ -662,7 +678,6 @@ async fn two_legs_render_a_drifting_source_the_same_way() { /// A receiver that joins a running broadcast mid-group keeps its output's system clock within /// 30 ppm of the source's and slews it no faster than 0.075 Hz/s, over 30 s windows of slots. #[tokio::test(start_paused = true)] -#[ignore = "fails on 49efbc9a1: the clock steers at 500 ppm to wear away the join's lead"] async fn a_joiner_keeps_the_system_clock_in_tolerance() { const WINDOW: u64 = 30_000_000; let mut live = Live::new(1.0, 0); @@ -705,3 +720,30 @@ async fn a_joiner_keeps_the_system_clock_in_tolerance() { ); } } + +/// A receiver that joins mid-group runs at the delay from its first output: every slot goes +/// out a fixed lag after the source sent its media (the delay before release, and the delay +/// the schedule sends ahead of the decode time), rather than that plus how old the group it +/// joined on was. +#[tokio::test(start_paused = true)] +async fn a_joiner_runs_at_the_delay_from_its_first_output() { + let mut live = Live::new(1.0, 0); + live.run(JOIN, &mut [], |_| false).await; + let mut leg = live.join(Duration::ZERO).await; + live.run(JOIN + 10_000_000 / VIDEO_US, &mut [&mut leg], |_| false).await; + assert_eq!(leg.export.dropped(), 0, "every frame arrived inside its deadline"); + + let held: Vec = leg + .out + .iter() + .map(|(sent, frame)| sent.saturating_duration_since(live.sent(frame.timestamp.as_micros() as u64))) + .collect(); + assert!(held.len() > 100, "too little output to judge: {}", held.len()); + let slot = Duration::from_millis(SLOT_MS as u64); + for (i, held) in held.iter().enumerate() { + assert!( + *held >= 2 * DELAY && *held <= 2 * DELAY + 2 * slot, + "slot {i} went out {held:?} after the source sent it, against a {DELAY:?} delay" + ); + } +} diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index 4dc89bdd701..5cfb0fd82c4 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -11,8 +11,12 @@ use web_async::time::{Instant, Sleep}; /// A frame handed to [`Buffer::push`]. pub(crate) struct Arrival { - /// When the frame was read from its source. + /// The earliest the frame could have arrived, which its deadline is judged on, so a + /// caller that reads late does not make it late. pub arrived: Instant, + /// When it was read: the latest it could have arrived, which its freshness is judged on, + /// so frames read together do not look fresher than they are. + pub read: Instant, /// When the frame decodes, nondecreasing within a track. pub decode: Timestamp, /// How many times the source had restarted its timeline when the frame was read. @@ -47,38 +51,52 @@ pub(crate) enum Push { /// Holds each track's frames until a fixed delay past their decode time, like an SRT /// receiver's TSBPD, in decode order across tracks. /// -/// Every track runs on one clock, anchored at the first frame's arrival. Under it the -/// deadline order is the decode order, so two buffers that saw the same frames arrive with -/// different skew, each within its deadline, emit them in the same order. A frame that -/// arrives past its deadline would break that order, so it is dropped and counted. +/// Every track runs on one clock. Under it the deadline order is the decode order, so two +/// buffers that saw the same frames arrive with different skew, each within its deadline, +/// emit them in the same order. A frame that arrives past its deadline would break that +/// order, so it is dropped and counted. +/// +/// The clock is acquired before anything goes out: frames are held until a track starts a +/// new group, or for a delay at most, and the clock is anchored on the freshest of them, the +/// one that arrived least behind its decode time, so that frame is due a delay after it +/// arrived. A joiner is handed the group it joined at once, already partly old, and +/// anchoring on its first frame would carry that lag for the whole run. What the anchor +/// makes due before the acquisition ended is dropped instead, without counting as late. /// /// A skipped group leaves the timeline where it was, so it keeps the clock: frames it made /// late are dropped like any other. A source that restarts its timeline (a declared -/// marker) has moved it, so once frames have gone out the restart opens a new generation -/// with its own clock, anchored at that frame though never ahead of a deadline already -/// given out. So does a skip after which a frame would be held more than a delay past -/// everything queued: its timeline jumped ahead, as when the skipped groups held a marker -/// the publisher shed. Every track's next frame runs on it: one crossing the same restart joins it -/// when the frame lands on its clock, neither late nor held more than a delay past +/// marker) has moved it, so once frames have gone out the restart opens a new generation, +/// whose clock is acquired the same way, though never ahead of a deadline already given +/// out. So does a skip after which a frame would be held more than a delay past everything +/// queued: its timeline jumped ahead, as when the skipped groups held a marker the +/// publisher shed. Every track's next frame runs on it: one crossing the same restart joins +/// it when the frame lands on its clock, neither late nor held more than a delay past /// everything queued, and opens another otherwise. Before anything has gone out there is /// nothing to keep in step with, so a restart there keeps the clock. No two tracks are /// ever on different clocks. /// /// The clock follows the source's: a source clock running slower than ours would make /// every frame late in the end, and a faster one would hold more and more. So the clock -/// measures how far the source's runs off ours from the least slack frames arrive with, runs -/// its decode timeline at that rate, and pulls the least slack back to the delay, which also -/// wears away a lead the first frame anchored it with. The decode timeline is the output's -/// system clock, so it keeps to what ISO/IEC 13818-1 2.4.2.1 allows one: within [`MAX_DRIFT`] -/// of ours, changing by at most [`MAX_SLEW`]. A source past that falls behind or piles up -/// until it has used half the delay, and then [`Buffer::push`] fails. +/// measures the source's rate and runs its decode timeline at it, holding the freshest +/// frames a delay. Each [`STEER`] of decode time gives a floor, the most slack a frame +/// arrived with, which queueing and retransmission only ever lower; the upper envelope of +/// the floors over [`RATE_WINDOW`] gives the rate (its slope) and the slack now, so a +/// spell of queueing shorter than half the window moves neither. The decode timeline is +/// the output's system clock, so it keeps to what ISO/IEC 13818-1 2.4.2.1 allows one: +/// within [`MAX_DRIFT`] of ours, changing by at most [`MAX_SLEW`]. A source past that is +/// counted, and once it has used half the delay [`Buffer::push`] fails. /// /// A zero delay holds nothing and drops nothing: each frame goes out as soon as it is /// read, ordered only among the frames read together. pub(crate) struct Buffer { delay: Duration, + /// Anchor each clock on the first frame rather than acquire it: a test exporting a + /// broadcast it wrote whole, every frame of which arrives at once. + replay: bool, /// The clock every frame is pushed under. clock: Option, + /// The next generation's frames, held while its clock is acquired. + acquire: Option>, tracks: BTreeMap>, /// The latest deadline given out. horizon: Option, @@ -88,15 +106,17 @@ pub(crate) struct Buffer { steer: Steer, timer: Option>>, dropped: u64, + /// Steps at which the source's clock ran further off ours than the clock may follow. + out_of_tolerance: u64, } -/// How often the clock is steered, in decode time: long enough that the least slack covers -/// a group's worth of frames sent ahead by different amounts. +/// How often the clock is steered, in decode time: long enough that the floor covers a +/// group's worth of frames sent ahead by different amounts. const STEER: Duration = Duration::from_secs(2); -/// How far back the source's rate is measured, in decode time: long enough that arrival -/// jitter averages out of it. +/// How far back the floors are kept, in decode time: a spell of queueing shorter than half +/// of it moves neither the rate nor the slack. const RATE_WINDOW: f64 = 600.0; -/// How quickly the clock pulls the least slack back to the delay: it closes the gap over this +/// How quickly the clock pulls the slack back to the delay: it closes the gap over this /// many seconds, or slower where the slew limit could not stop it in time. const RESPONSE: f64 = 600.0; /// The furthest the clock runs off ours: 810 Hz of the 27 MHz system clock (ISO/IEC 13818-1 @@ -116,18 +136,69 @@ struct Clock { drift: f64, } +/// A generation whose clock is being acquired. +struct Acquire { + generation: u64, + /// When its first frame was read; it ends a delay later at the latest. + since: Instant, + /// Its frames, in arrival order. + frames: Vec<(K, Arrival)>, + /// Its tracks, and whether each is partway through a group, so its next sync frame + /// starts the next one. + tracks: BTreeMap, +} + +impl Acquire { + /// The freshest frame: the one read least behind its decode time. + fn fresh(&self) -> &Arrival { + let behind = |frame: &Arrival| since(self.since, frame.read) - frame.decode.as_nanos() as i128; + self.frames + .iter() + .map(|(_, frame)| frame) + .min_by_key(|frame| behind(frame)) + .expect("an acquisition holds a frame") + } + + /// When it ends: once a frame that could still go out on time falls due on the freshest + /// frame's clock, since waiting longer would drop it, and a delay after it began at the + /// latest. + fn end(&self, delay: Duration) -> Instant { + let fresh = self.fresh(); + let due = |frame: &Arrival| { + let ahead = frame.decode.as_nanos() as i128 - fresh.decode.as_nanos() as i128; + offset(fresh.read + delay, ahead) + }; + self.frames + .iter() + .filter_map(|(_, frame)| due(frame).filter(|due| *due >= frame.read)) + .chain([self.since + delay]) + .min() + .expect("a bound") + } +} + /// What the clock is steered on. #[derive(Default)] struct Steer { - /// The least slack (deadline less arrival, in seconds) a frame arrived with since the last - /// step, and the decode time the step began at. - least: Option<(f64, Timestamp)>, - /// Each step's decode time and the source's phase there, in seconds, over the last - /// [`RATE_WINDOW`]: how far the least slack is past the delay, less how far the drift has - /// moved the deadlines. Its slope is how much slower the source's clock runs than ours. + /// This step's floor. + floor: Option, + /// Each step's floor, as its decode time and phase in seconds, over the last + /// [`RATE_WINDOW`]. Its slope is how much faster the source's clock runs than ours. phase: VecDeque<(f64, f64)>, /// How far the drift has moved the deadlines, in seconds. steered: f64, + /// The source's clock rate off ours, as last measured: positive runs fast. + source: Option, +} + +/// The frame with the most slack in a step: the one queued least on its way. +struct Floor { + /// Its slack past the delay, less how far the drift has moved its deadline, in seconds. + phase: f64, + /// When it decodes, in seconds. + at: f64, + /// When the step's first frame decodes. + began: Timestamp, } struct Track { @@ -138,88 +209,177 @@ struct Track { skip: u64, /// The generation of the last frame. generation: u64, - /// A frame was dropped, so later ones are too until the next sync frame. - waiting: bool, + /// Frames are dropped until the next sync frame, and why. + waiting: Option, +} + +/// Why a track waits for a sync frame. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Wait { + /// A frame missed its deadline: what it takes with it counts as dropped. + Late, + /// The acquisition dropped what was due before it ended, which is not a loss. + Acquired, } impl Buffer { pub fn new(delay: Duration) -> Self { Self { delay, + replay: false, clock: None, + acquire: None, tracks: BTreeMap::new(), horizon: None, released: false, steer: Steer::default(), timer: None, dropped: 0, + out_of_tolerance: 0, } } + /// Anchor each clock on its first frame instead of acquiring it, for a test that writes a + /// broadcast whole before exporting it. + #[cfg(test)] + pub fn replay(mut self) -> Self { + self.replay = true; + self + } + /// Queue a frame, or drop it if it cannot make its deadline. Fails once the source's clock /// runs further off ours than the clock may follow. pub fn push(&mut self, key: K, arrival: Arrival) -> anyhow::Result { - let clock = *self.clock.get_or_insert(Clock { - generation: 0, - anchor: arrival.arrived, - base: arrival.decode, - drift: 0.0, - }); - let mut generation = clock.generation; - let mut deadline = self.deadline(&clock, arrival.decode); - // A track still on an older generation may be crossing the restart another track - // opened the newest one for. - let jumped = self.tracks.get(&key).is_some_and(|track| { - let restarted = track.restart != arrival.restart - && (track.generation == generation || !self.lands(deadline, arrival.arrived)); - let leapt = - track.skip != arrival.skip && deadline.is_some_and(|deadline| deadline > self.bound(arrival.arrived)); - restarted || leapt - }); - if jumped && self.released { - generation += 1; - let after = self.horizon.and_then(|horizon| horizon.checked_sub(self.delay)); - let clock = Clock { - generation, - anchor: after.map_or(arrival.arrived, |after| after.max(arrival.arrived)), - base: arrival.decode, - drift: clock.drift, - }; - self.clock = Some(clock); - self.steer = Steer::default(); - deadline = self.deadline(&clock, arrival.decode); - } else if self.released && !self.delay.is_zero() { + let held = self.acquire.as_ref().is_some_and(|acquire| acquire.tracks.contains_key(&key)); + let Some(clock) = self.clock.filter(|_| !held && !(self.released && self.jumps(&key, &arrival))) else { + self.hold(key, arrival); + return Ok(Push::Queued); + }; + let deadline = self.deadline(&clock, arrival.decode); + if self.released && !self.delay.is_zero() { self.steer(deadline, &arrival)?; } + Ok(self.queue(key, arrival, clock.generation, deadline, false)) + } + + /// Whether a frame moves its track off the clock: a restart, unless the track is crossing + /// one another track already opened a generation for and the frame lands on its clock, or + /// a skip that leapt ahead. + fn jumps(&self, key: &K, arrival: &Arrival) -> bool { + let (Some(clock), Some(track)) = (self.clock, self.tracks.get(key)) else { + return false; + }; + let deadline = self.deadline(&clock, arrival.decode); + let restarted = track.restart != arrival.restart + && (track.generation == clock.generation || !self.lands(deadline, arrival.arrived)); + let leapt = track.skip != arrival.skip && deadline.is_some_and(|deadline| deadline > self.bound(arrival.arrived)); + restarted || leapt + } + + /// Hold a frame for the generation being acquired, starting one if none is, and anchor it + /// once a track starts a new group. Without a delay, or replaying, it anchors at once. + fn hold(&mut self, key: K, arrival: Arrival) { + let generation = self.clock.map_or(0, |clock| clock.generation + 1); + let acquire = self.acquire.get_or_insert_with(|| Acquire { + generation, + since: arrival.read, + frames: Vec::new(), + tracks: BTreeMap::new(), + }); + let mid = acquire.tracks.entry(key.clone()).or_default(); + let grouped = arrival.sync && *mid; + *mid |= !arrival.sync; + let now = arrival.read; + acquire.frames.push((key, arrival)); + if grouped || self.replay || self.delay.is_zero() || acquire.end(self.delay) <= now { + self.acquired(); + } + } + + /// Anchor the acquired generation's clock on its freshest frame (its first, replaying or + /// without a delay), and queue what it held, dropping what that made due before it was + /// read: what a joiner is handed of its group that is older than the delay. + fn acquired(&mut self) { + let Some(acquire) = self.acquire.take() else { + return; + }; + let acquiring = !self.replay && !self.delay.is_zero(); + let fresh = match acquiring { + true => acquire.fresh(), + false => &acquire.frames.first().expect("an acquisition holds a frame").1, + }; + let mut clock = Clock { + generation: acquire.generation, + anchor: match acquiring { + true => fresh.read, + false => fresh.arrived, + }, + base: fresh.decode, + drift: self.clock.map_or(0.0, |clock| clock.drift), + }; + // Never ahead of a deadline already given out. + let kept = acquire + .frames + .iter() + .filter_map(|(_, frame)| match self.judge(&clock, frame, acquiring) { + (deadline, false) => deadline, + (_, true) => None, + }) + .min(); + if let (Some(horizon), Some(kept)) = (self.horizon, kept) + && kept < horizon + { + clock.anchor += horizon - kept; + } + self.clock = Some(clock); + self.steer = Steer::default(); + for (key, frame) in acquire.frames { + let (deadline, stale) = self.judge(&clock, &frame, acquiring); + self.queue(key, frame, clock.generation, deadline, stale); + } + } + /// A held frame's deadline on `clock`, and whether it was due before it was read, when + /// `acquiring`. + fn judge(&self, clock: &Clock, frame: &Arrival, acquiring: bool) -> (Option, bool) { + let deadline = self.deadline(clock, frame.decode); + (deadline, acquiring && deadline.is_none_or(|deadline| deadline < frame.read)) + } + + /// Queue a frame on `generation` due at `deadline`, unless it is `stale` or cannot make it. + fn queue(&mut self, key: K, arrival: Arrival, generation: u64, deadline: Option, stale: bool) -> Push { let track = self.tracks.entry(key).or_insert_with(|| Track { queue: VecDeque::new(), restart: arrival.restart, skip: arrival.skip, generation, - waiting: false, + waiting: None, }); track.restart = arrival.restart; track.skip = arrival.skip; track.generation = generation; let push = match deadline { - _ if track.waiting && !arrival.sync => Push::Waiting, + _ if stale => { + track.waiting = Some(Wait::Acquired); + Push::Waiting + } + _ if track.waiting.is_some() && !arrival.sync => Push::Waiting, Some(deadline) if arrival.arrived <= deadline || self.delay.is_zero() => { - track.waiting = false; + track.waiting = None; track.queue.push_back((generation, deadline, arrival.item)); self.horizon = self.horizon.max(Some(deadline)); Push::Queued } // Past the deadline, or so far off the clock that no instant holds it. _ => { - track.waiting = true; + track.waiting = Some(Wait::Late); Push::Late } }; - if push != Push::Queued { + if push != Push::Queued && track.waiting == Some(Wait::Late) { self.dropped += 1; } - Ok(push) + push } /// Whether a frame arriving at `arrived` and due at `deadline` is on the clock: not late, @@ -241,36 +401,51 @@ impl Buffer { let (Some(clock), Some(deadline)) = (self.clock, deadline) else { return Ok(()); }; - let slack = match deadline >= arrival.arrived { - true => (deadline - arrival.arrived).as_secs_f64(), - false => -(arrival.arrived - deadline).as_secs_f64(), - }; - let (least, since) = self.steer.least.get_or_insert((slack, arrival.decode)); - *least = least.min(slack); - if arrival.decode.as_nanos().saturating_sub(since.as_nanos()) < STEER.as_nanos() { + let delay = self.delay.as_secs_f64(); + let now = arrival.decode.as_nanos() as f64 / 1e9; + let elapsed = now - clock.base.as_nanos() as f64 / 1e9; + // The source's phase at this frame: its slack past the delay, less how far the drift + // has moved its deadline. + let slack = since(arrival.read, deadline) as f64 / 1e9; + let phase = slack - delay - (self.steer.steered + elapsed * clock.drift); + let floor = self.steer.floor.get_or_insert(Floor { + phase, + at: now, + began: arrival.decode, + }); + if phase > floor.phase { + (floor.phase, floor.at) = (phase, now); + } + if arrival.decode.as_nanos().saturating_sub(floor.began.as_nanos()) < STEER.as_nanos() { return Ok(()); } // Re-anchor where this frame decodes, so the new drift starts from there. let Some(anchor) = deadline.checked_sub(self.delay) else { return Ok(()); }; - let delay = self.delay.as_secs_f64(); - let gap = *least - delay; - self.steer.least = None; - - let now = arrival.decode.as_nanos() as f64 / 1e9; - let elapsed = now - clock.base.as_nanos() as f64 / 1e9; + let floor = self.steer.floor.take().expect("a floor"); self.steer.steered += elapsed * clock.drift; - let phase = &mut self.steer.phase; - phase.push_back((now, gap - self.steer.steered)); - while phase.front().is_some_and(|&(at, _)| now - at > RATE_WINDOW) { - phase.pop_front(); + let phases = &mut self.steer.phase; + phases.push_back((floor.at, floor.phase)); + while phases.front().is_some_and(|&(at, _)| now - at > RATE_WINDOW) { + phases.pop_front(); } - let source = -slope(phase).unwrap_or(-clock.drift); + let Some((rate, at)) = envelope(phases, now) else { + self.clock = Some(Clock { + anchor, + base: arrival.decode, + ..clock + }); + return Ok(()); + }; + // The phase rises as the source's clock runs fast, so frames arrive ever earlier. + let slow = -rate; + let gap = at + self.steer.steered; + self.steer.source = Some(rate); // Pull the gap shut over [`RESPONSE`], no faster than the slew limit can stop on it. let pull = (gap.abs() / RESPONSE).min((2.0 * MAX_SLEW * gap.abs()).sqrt()); - let target = source - pull.copysign(gap); + let target = slow - pull.copysign(gap); // Tracks interleave a little off decode order, so a step may decode before the last. let step = MAX_SLEW * elapsed.max(0.0); let drift = (clock.drift + (target - clock.drift).clamp(-step, step)).clamp(-MAX_DRIFT, MAX_DRIFT); @@ -281,13 +456,17 @@ impl Buffer { ..clock }); - // At the limit and still losing ground, with half the delay gone. - if drift == MAX_DRIFT.copysign(source) && source.abs() > MAX_DRIFT && gap * source.signum() < -delay / 2.0 { - anyhow::bail!( - "the source's clock runs {:.1} ppm off ours, past the {:.0} ppm the output's may follow", - source * 1e6, - MAX_DRIFT * 1e6 - ); + // To a hundredth of a ppm, so a source right at the limit is not out by rounding. + if (slow.abs() * 1e8).round() > (MAX_DRIFT * 1e8).round() { + self.out_of_tolerance += 1; + // At the limit and still losing ground, with half the delay gone. + if drift == MAX_DRIFT.copysign(slow) && gap * slow.signum() < -delay / 2.0 { + anyhow::bail!( + "the source's clock runs {:+.1} ppm off ours, past the {:.0} ppm the output's may follow", + rate * 1e6, + MAX_DRIFT * 1e6 + ); + } } Ok(()) } @@ -301,12 +480,10 @@ impl Buffer { fn deadline(&self, clock: &Clock, decode: Timestamp) -> Option { // Signed, since a frame can decode before the one that anchored the clock. let since = decode.as_nanos() as i128 - clock.base.as_nanos() as i128; - let offset = since + (since as f64 * clock.drift) as i128 + self.delay.as_nanos() as i128; - let nanos = u64::try_from(offset.unsigned_abs()).ok()?; - match offset >= 0 { - true => clock.anchor.checked_add(Duration::from_nanos(nanos)), - false => clock.anchor.checked_sub(Duration::from_nanos(nanos)), - } + offset( + clock.anchor, + since + (since as f64 * clock.drift) as i128 + self.delay.as_nanos() as i128, + ) } /// The generation, deadline and track of the frame that goes out next: generation by @@ -321,52 +498,83 @@ impl Buffer { .min() } + /// When the acquisition under way ends, unless a track starts a new group first. + fn acquired_by(&self) -> Option { + self.acquire.as_ref().map(|acquire| acquire.end(self.delay)) + } + /// The next frame whose deadline has come, in [`Self::front`] order. pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll> { - let Some((_, deadline, key)) = self.front() else { - self.timer = None; - return Poll::Pending; - }; - let key = key.clone(); - if !self.delay.is_zero() && Instant::now() < deadline { + loop { + let now = Instant::now(); + let acquired = self.acquired_by(); + if acquired.is_some_and(|end| now >= end) { + self.acquired(); + continue; + } + let front = self.front().map(|(_, deadline, key)| (deadline, key.clone())); + if let Some((deadline, key)) = &front + && (self.delay.is_zero() || now >= *deadline) + { + let (generation, _, item) = self.tracks.get_mut(key).and_then(|t| t.queue.pop_front()).unwrap(); + self.released = true; + return Poll::Ready(Ready { + track: key.clone(), + generation, + item, + }); + } + let Some(wake) = front.map(|(deadline, _)| deadline).into_iter().chain(acquired).min() else { + self.timer = None; + return Poll::Pending; + }; let timer = self .timer - .get_or_insert_with(|| Box::pin(web_async::time::sleep_until(deadline))); - if timer.deadline() != deadline { - timer.as_mut().reset(deadline); + .get_or_insert_with(|| Box::pin(web_async::time::sleep_until(wake))); + if timer.deadline() != wake { + timer.as_mut().reset(wake); } if waiter.poll_future(timer.as_mut()).is_pending() { return Poll::Pending; } } - let (generation, _, item) = self.tracks.get_mut(&key).and_then(|t| t.queue.pop_front()).unwrap(); - self.released = true; - Poll::Ready(Ready { - track: key, - generation, - item, - }) } - /// When the next queued frame is due. + /// When the next queued frame is due, or the acquisition under way ends. #[cfg(test)] pub fn next_deadline(&self) -> Option { - self.front().map(|(_, deadline, _)| deadline) + self.front() + .map(|(_, deadline, _)| deadline) + .into_iter() + .chain(self.acquired_by()) + .min() } /// Whether no frame is waiting. pub fn is_empty(&self) -> bool { - self.tracks.values().all(|track| track.queue.is_empty()) + self.acquire.is_none() && self.tracks.values().all(|track| track.queue.is_empty()) } - /// How many frames were dropped, late or waiting for a sync frame. + /// How many frames were dropped, late or waiting for a sync frame after one was. pub fn dropped(&self) -> u64 { self.dropped } + /// The source's clock rate off ours, as last measured, as a fraction: positive runs fast. + pub fn drift(&self) -> Option { + self.steer.source + } + + /// How many times the source's clock was measured further off ours than the clock may + /// follow. + pub fn out_of_tolerance(&self) -> u64 { + self.out_of_tolerance + } + /// Drop every queued frame and the clock, so the next frame starts it afresh. pub fn clear(&mut self) { self.clock = None; + self.acquire = None; self.tracks.clear(); self.horizon = None; self.released = false; @@ -375,16 +583,43 @@ impl Buffer { } } -/// The least-squares slope of `points`, if they span any time. -fn slope(points: &VecDeque<(f64, f64)>) -> Option { - let n = points.len() as f64; - let (x, y) = points - .iter() - .fold((0.0, 0.0), |(x, y), (px, py)| (x + px / n, y + py / n)); - let (cov, var) = points.iter().fold((0.0, 0.0), |(cov, var), (px, py)| { - (cov + (px - x) * (py - y), var + (px - x).powi(2)) - }); - (var > 0.0).then(|| cov / var) +/// `nanos` after `at`, or before it if negative, if an instant holds it. +fn offset(at: Instant, nanos: i128) -> Option { + let span = Duration::from_nanos(u64::try_from(nanos.unsigned_abs()).ok()?); + match nanos >= 0 { + true => at.checked_add(span), + false => at.checked_sub(span), + } +} + +/// How long after `from` the instant `to` is, in nanoseconds, negative if before. +fn since(from: Instant, to: Instant) -> i128 { + match to >= from { + true => (to - from).as_nanos() as i128, + false => -((from - to).as_nanos() as i128), + } +} + +/// The upper envelope of `points`, sorted by time: the slope of its edge over their mean +/// time, and where that edge's line is at `now`. A point below the envelope, as queueing +/// leaves the floor, moves neither unless it lasts past the mean. +fn envelope(points: &VecDeque<(f64, f64)>, now: f64) -> Option<(f64, f64)> { + let mut hull: Vec<(f64, f64)> = Vec::with_capacity(points.len()); + for &(x, y) in points { + while let [.., (x1, y1), (x2, y2)] = hull[..] + && (y2 - y1) * (x - x1) <= (y - y1) * (x2 - x1) + { + hull.pop(); + } + hull.push((x, y)); + } + let mean = points.iter().map(|(x, _)| x).sum::() / points.len() as f64; + let ((x1, y1), (x2, y2)) = hull + .windows(2) + .map(|edge| (edge[0], edge[1])) + .find(|((x1, _), (x2, _))| *x1 <= mean && mean <= *x2 && x2 > x1)?; + let slope = (y2 - y1) / (x2 - x1); + Some((slope, y1 + slope * (now - x1))) } #[cfg(test)] @@ -400,6 +635,7 @@ mod tests { fn arrival(arrived: Instant, decode: u64, item: &'static str) -> Arrival<&'static str> { Arrival { arrived, + read: arrived, decode: ms(decode), restart: 0, skip: 0, @@ -435,7 +671,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn releases_at_the_delay_in_decode_order() { let start = Instant::now(); - let mut buffer = Buffer::new(DELAY); + let mut buffer = Buffer::new(DELAY).replay(); assert_eq!(buffer.push(1, arrival(start, 0, "v0")).unwrap(), Push::Queued); assert_eq!(buffer.push(1, arrival(start, 40, "v40")).unwrap(), Push::Queued); // Audio arrives later than the video, with an earlier decode time. @@ -467,7 +703,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn arrival_skew_does_not_change_the_order() { let start = Instant::now(); - let (mut early, mut late) = (Buffer::new(DELAY), Buffer::new(DELAY)); + let (mut early, mut late) = (Buffer::new(DELAY).replay(), Buffer::new(DELAY).replay()); let frames = [ (1, 0, "v0"), (1, 40, "v40"), @@ -499,7 +735,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn late_frames_are_dropped_until_a_sync_frame() { let start = Instant::now(); - let mut buffer = Buffer::new(DELAY); + let mut buffer = Buffer::new(DELAY).replay(); buffer.push(1, arrival(start, 0, "v0")).unwrap(); buffer.push(2, arrival(start, 0, "a0")).unwrap(); let late = start + Duration::from_millis(200); @@ -521,7 +757,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn a_skip_before_the_first_release_keeps_one_clock() { let start = Instant::now(); - let mut buffer = Buffer::new(DELAY); + let mut buffer = Buffer::new(DELAY).replay(); // The video's first group is stale and anchors the clock; the audio is live. buffer.push(1, arrival(start, 0, "v0")).unwrap(); buffer.push(2, arrival(start, 900, "a900")).unwrap(); @@ -549,7 +785,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn every_track_joins_a_new_generation_at_its_next_frame() { let start = Instant::now(); - let mut buffer = Buffer::new(DELAY); + let mut buffer = Buffer::new(DELAY).replay(); buffer.push(1, arrival(start, 0, "v0")).unwrap(); buffer.push(2, arrival(start, 0, "a0")).unwrap(); buffer.push(3, arrival(start, 0, "d0")).unwrap(); @@ -627,7 +863,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn generations_go_out_in_turn() { let start = Instant::now(); - let mut buffer = Buffer::new(DELAY); + let mut buffer = Buffer::new(DELAY).replay(); buffer.push(1, arrival(start, 0, "v0")).unwrap(); buffer.push(1, arrival(start, 40, "v40")).unwrap(); buffer.push(2, arrival(start, 20, "a20")).unwrap(); @@ -646,21 +882,56 @@ mod tests { ); } - /// Feed `hours` of 5 fps frames from a source whose clock runs `ppm` slower than ours - /// through a 500 ms buffer, checking the clock stays within what 13818-1 allows, until a - /// push fails. Returns that error and the most frames the buffer held. - async fn drift(ppm: i128, hours: u64) -> (Option, usize) { + /// What a run against a drifting source saw. + struct Run { + /// The error the run ended on, if it failed. + err: Option, + /// The largest gap between a frame's slack and the delay, in seconds, from `settled` + /// hours on. + worst: f64, + /// The source's rate the buffer measured at the end, and before `step` began. + drift: Option, + before: Option, + /// The clock's drift at the end, and before `step` began. + clock: f64, + clock_before: f64, + out_of_tolerance: u64, + } + + /// Feed `hours` of 2 fps frames from a source whose clock runs `ppm` slower than ours + /// through a 500 ms buffer, each frame queued `queueing(hour)` extra on its way, checking + /// that no frame goes late and the clock stays within what 13818-1 allows, until a push + /// fails. + async fn drift(ppm: f64, hours: f64, settled: f64, queueing: impl Fn(f64) -> Duration) -> Run { + let delay = Duration::from_millis(500); let start = Instant::now(); - let mut buffer = Buffer::new(Duration::from_millis(500)); - let mut most = 0; - let mut last = (0.0, ms(0)); - for k in 0..hours * 3_600 * 5 { - let decode = Duration::from_millis(k * 200); - let source = decode.as_nanos() as i128 * (1_000_000 + ppm) / 1_000_000; - let arrived = start + Duration::from_nanos(source as u64); + let mut buffer = Buffer::new(delay); + let mut run = Run { + err: None, + worst: 0.0, + drift: None, + before: None, + clock: 0.0, + clock_before: 0.0, + out_of_tolerance: 0, + }; + let mut last: Option<(f64, Timestamp)> = None; + let mut stepped = false; + for k in 0..(hours * 3_600.0 * 2.0) as u64 { + let decode = Duration::from_millis(k * 500); + let hour = decode.as_secs_f64() / 3_600.0; + let queued = queueing(hour); + stepped |= !queued.is_zero(); + if !stepped { + run.before = buffer.drift(); + run.clock_before = buffer.clock.map_or(0.0, |clock| clock.drift); + } + let source = decode.as_secs_f64() * (1.0 + ppm * 1e-6); + let arrived = start + Duration::from_secs_f64(source) + queued; tokio::time::advance(arrived.saturating_duration_since(Instant::now())).await; let frame = Arrival { arrived, + read: arrived, decode: Timestamp::from_micros(decode.as_micros() as u64).unwrap(), restart: 0, skip: 0, @@ -669,52 +940,140 @@ mod tests { }; match buffer.push(1, frame) { Ok(push) => assert_eq!(push, Push::Queued, "{ppm} ppm: frame {k} was late"), - Err(err) => return (Some(err), most), + Err(err) => { + run.err = Some(err); + break; + } + } + if hour >= settled + && queued.is_zero() + && let Some(deadline) = buffer.at(Timestamp::from_micros(decode.as_micros() as u64).unwrap()) + { + let slack = since(arrived, deadline) as f64 / 1e9; + run.worst = run.worst.max((slack - delay.as_secs_f64()).abs()); } - most = most.max(buffer.tracks[&1].queue.len()); due(&mut buffer); - let clock = buffer.clock.unwrap(); - let elapsed = (clock.base.as_nanos() - last.1.as_nanos()) as f64 / 1e9; + let Some(clock) = buffer.clock else { + continue; + }; assert!(clock.drift.abs() <= MAX_DRIFT, "{ppm} ppm: drift {}", clock.drift); - assert!( - (clock.drift - last.0).abs() <= MAX_SLEW * elapsed * (1.0 + 1e-9), - "{ppm} ppm: drift slewed from {} to {} in {elapsed} s", - last.0, - clock.drift - ); - last = (clock.drift, clock.base); + if let Some((drift, base)) = last { + let elapsed = (clock.base.as_nanos() as f64 - base.as_nanos() as f64) / 1e9; + assert!( + (clock.drift - drift).abs() <= MAX_SLEW * elapsed.max(0.0) * (1.0 + 1e-9), + "{ppm} ppm: drift slewed from {drift} to {} in {elapsed} s", + clock.drift + ); + } + last = Some((clock.drift, clock.base)); } - let drift = buffer.clock.unwrap().drift; - assert!( - (drift * 1e6 - ppm as f64).abs() < 0.5, - "{ppm} ppm: the clock settled at {} ppm", - drift * 1e6 - ); - (None, most) + assert_eq!(buffer.dropped(), 0, "{ppm} ppm: frames went late"); + run.drift = buffer.drift(); + run.clock = buffer.clock.map_or(0.0, |clock| clock.drift); + run.out_of_tolerance = buffer.out_of_tolerance(); + run } - /// A source whose clock runs off ours, slower or faster, by up to the 30 ppm 13818-1 allows - /// neither goes late nor piles up, and the clock settles at its rate. + /// A source whose clock runs off ours by up to the 30 ppm 13818-1 allows loses no frame + /// over a day, and settles at its rate. Within it the freshest frames are held the delay to + /// 10 ms once converged; at the limit the clock cannot run past the source's to win back + /// what reaching its rate under the slew limit cost, 30 ppm squared over twice the slew + /// (162 ms), so it holds there. #[tokio::test(start_paused = true)] - async fn the_clock_follows_a_drifting_source() { - for ppm in [25, -25] { - let (err, most) = drift(ppm, 10).await; - assert!(err.is_none(), "{ppm} ppm: {err:?}"); - assert!(most <= 5, "{ppm} ppm: the buffer grew to {most} frames"); + async fn the_clock_follows_a_drifting_source_for_a_day() { + for (ppm, bound) in [(30.0, 0.170), (-30.0, 0.170), (25.0, 0.010), (-25.0, 0.010)] { + let run = drift(ppm, 24.0, 16.0, |_| Duration::ZERO).await; + assert!(run.err.is_none(), "{ppm} ppm: {:?}", run.err); + assert!(run.worst <= bound, "{ppm} ppm: held {} s off the delay", run.worst); + let drift = run.drift.expect("a measurement") * 1e6; + assert!((drift + ppm).abs() < 0.1, "{ppm} ppm: measured {drift} ppm"); + assert!((run.clock * 1e6 - ppm).abs() < 0.5, "{ppm} ppm: the clock settled at {}", run.clock * 1e6); + assert_eq!(run.out_of_tolerance, 0, "{ppm} ppm"); } } - /// A source further off than the clock may follow fails once it has used half the delay, - /// before any frame goes late. + /// A spell of queueing lowers the floor only while it lasts, so four minutes of 200 ms + /// more moves neither the measured rate nor the clock. + #[tokio::test(start_paused = true)] + async fn a_step_in_queueing_does_not_move_the_estimate() { + let spell = 6.0..6.0 + 4.0 / 60.0; + let run = drift(10.0, 6.5, 6.5, |hour| match spell.contains(&hour) { + true => Duration::from_millis(200), + false => Duration::ZERO, + }) + .await; + assert!(run.err.is_none(), "{:?}", run.err); + let (before, after) = (run.before.unwrap() * 1e6, run.drift.unwrap() * 1e6); + assert!((after - before).abs() < 0.01, "the estimate moved from {before} to {after} ppm"); + let (before, after) = (run.clock_before * 1e6, run.clock * 1e6); + assert!((after - before).abs() < 0.01, "the clock moved from {before} to {after} ppm"); + } + + /// A source further off than the clock may follow is counted, and fails once it has used + /// half the delay, before any frame goes late. #[tokio::test(start_paused = true)] async fn a_source_past_the_drift_limit_fails() { - for ppm in [40, -40] { - let (err, _) = drift(ppm, 6).await; - assert!(err.is_some(), "{ppm} ppm: no error"); + for ppm in [40.0, -40.0] { + let run = drift(ppm, 6.0, 6.0, |_| Duration::ZERO).await; + assert!(run.err.is_some(), "{ppm} ppm: no error"); + assert!(run.out_of_tolerance > 0, "{ppm} ppm: not counted"); } } + /// A joiner is handed its group at once. The clock is anchored on the freshest frame, so + /// what is older than the delay is dropped without counting as late, and the next group + /// goes out a delay after it arrived. + #[tokio::test(start_paused = true)] + async fn a_joiner_anchors_on_its_freshest_frame() { + let start = Instant::now(); + let mut buffer = Buffer::new(DELAY); + let video = |arrived, decode, sync, item| Arrival { + sync, + ..arrival(arrived, decode, item) + }; + // Joined 200 ms into a group, twice the delay. + buffer.push(1, video(start, 0, true, "v0")).unwrap(); + for (decode, item) in [(40, "v40"), (80, "v80"), (120, "v120"), (160, "v160"), (200, "v200")] { + buffer.push(1, video(start, decode, false, item)).unwrap(); + } + tokio::time::advance(Duration::from_millis(40)).await; + let next = start + Duration::from_millis(40); + buffer.push(1, video(next, 240, true, "v240")).unwrap(); + buffer.push(1, video(next, 280, false, "v280")).unwrap(); + assert!(due(&mut buffer).is_empty()); + assert_eq!(buffer.next_deadline(), Some(next + DELAY)); + tokio::time::advance(DELAY).await; + assert_eq!(due(&mut buffer), ["v240"]); + assert_eq!(buffer.dropped(), 0, "trimming the join is not a loss"); + } + + /// A frame the acquisition holds that is still in time goes out on time: the + /// acquisition ends when the first of them falls due rather than drop it. + #[tokio::test(start_paused = true)] + async fn the_acquisition_ends_before_a_held_frame_is_due() { + let start = Instant::now(); + let mut buffer = Buffer::new(DELAY); + // Joined 60 ms into a group: its keyframe is due 40 ms from now. + let key = Arrival { + sync: true, + ..arrival(start, 0, "v0") + }; + buffer.push(1, key).unwrap(); + buffer + .push( + 1, + Arrival { + sync: false, + ..arrival(start, 60, "v60") + }, + ) + .unwrap(); + assert_eq!(buffer.next_deadline(), Some(start + Duration::from_millis(40))); + tokio::time::advance(Duration::from_millis(40)).await; + assert_eq!(due(&mut buffer), ["v0"]); + } + /// Zero holds nothing: frames read together go out at once in decode order, and a /// frame behind the clock still goes out. #[tokio::test(start_paused = true)] From 016789ac004ff6ee24ac1d7aff6cbfa33a65864b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 15:20:22 -0700 Subject: [PATCH 29/40] fix(moq-mux): drop cut AC-3 frames, end without a missed deadline, model teletext From t0ms's re-grade of the broadcast capture: - A passed-through AC-3 sync frame shorter than its header says, as a source stopped mid-frame leaves in its last PES, is dropped rather than passed on (it underflowed B in compliance.py). - Once every source has ended, a unit that misses its deadline is the tail going out late, not a failure: the export no longer exits with "missed a decode deadline" after the publisher's tracks have closed. - Teletext as DVB private data gets its EN 300 472 transport buffer: the schedule drains it at 6.75 Mb/s, and compliance.py grades its 480-byte TB (a tstd-controls case overflows it where 512 bytes would not). Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export.rs | 72 +++++++++++++++++++--- rs/moq-mux/src/container/ts/export_test.rs | 49 +++++++++++---- rs/moq-mux/src/container/ts/schedule.rs | 30 ++++++++- test/ts/compliance.py | 19 ++++-- test/ts/tstd-controls.py | 19 +++++- 5 files changed, 160 insertions(+), 29 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index cd25204c77b..6ed0489535a 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -888,6 +888,12 @@ impl Export { self.fill(waiter)?; } + // Once every source has ended, the tail goes out late if it must: the end of the + // stream is not a missed deadline. + if !self.tracks.is_empty() && self.tracks.values().all(|track| track.finished) { + self.schedule.end(); + } + // 4. Mux each frame the jitter buffer lets go (the first carries the buffered // PAT/PMT), then lay out every grid slot whose time has come ([`Self::lay_due`]). loop { @@ -1690,6 +1696,13 @@ impl Export { .then(|| ac3_frames(frame.timestamp, &es_payload)) .flatten() .unwrap_or_else(|| vec![(frame.timestamp, es_payload.as_slice())]); + if frames.is_empty() { + // Nothing left of the PES but the tables muxed ahead of it. + return match out.is_empty() { + true => Ok(()), + false => self.push_unit(name, pid, decode, out, false), + }; + } let mut at = decode; for (k, (timestamp, payload)) in frames.into_iter().enumerate() { if k > 0 { @@ -2238,8 +2251,12 @@ fn audio_buffer(config: &AudioConfig, kind: &Kind) -> Buffer { /// The DVB AC-3 descriptor's tag (ETSI EN 300 468 6.2.1), marking AC-3 as private data. const AC3_DESCRIPTOR: u8 = 0x6a; +/// The DVB teletext descriptor's tag (ETSI EN 300 468 6.2.43). +const TELETEXT_DESCRIPTOR: u8 = 0x56; + /// A verbatim stream's buffers, for the kinds its descriptors identify: AC-3 as DVB private -/// data (ATSC A/52 Annex A 5.4). Any other stream has none the schedule knows. +/// data (ATSC A/52 Annex A 5.4), and teletext, whose transport buffer drains at 6.75 Mb/s +/// (ETSI EN 300 472 5). Any other stream has none the schedule knows. fn verbatim_buffer(kind: &Kind, descriptors: &[catalog::Descriptor]) -> Option { let Kind::Verbatim { stream_type: 0x06, @@ -2249,29 +2266,40 @@ fn verbatim_buffer(kind: &Kind, descriptors: &[catalog::Descriptor]) -> Option Some(Buffer { rate: 2_000_000, size: Some(5_696), - }) + }), + TELETEXT_DESCRIPTOR => Some(Buffer { + rate: 6_750_000, + size: None, + }), + _ => None, + }) } /// The AC-3 sync frames `payload` holds, each with its presentation time counted on from -/// `start`, or `None` unless it is two or more whole frames. +/// `start`, or `None` if it does not parse as AC-3. A trailing frame shorter than its header +/// says is dropped, as a source cut off mid-frame leaves one: no decoder can use it. fn ac3_frames(start: Timestamp, payload: &[u8]) -> Option> { let mut frames = Vec::new(); let (mut rest, mut samples) = (payload, 0u64); while !rest.is_empty() { let header = (crate::codec::ac3::DESCRIPTOR.parse)(rest).ok()?; - let (frame, tail) = rest.split_at_checked(header.len)?; + let Some((frame, tail)) = rest.split_at_checked(header.len) else { + tracing::warn!( + missing = header.len - rest.len(), + "dropped an AC-3 sync frame cut short of its length" + ); + return Some(frames); + }; let offset = Timestamp::from_micros(samples * 1_000_000 / u64::from(header.sample_rate.max(1))).ok()?; frames.push((start.checked_add(offset).ok()?, frame)); samples += header.samples; rest = tail; } - (frames.len() > 1).then_some(frames) + Some(frames) } /// Build the Annex-B elementary-stream payload for one video frame: rewrite the @@ -2543,12 +2571,36 @@ mod tests { use moq_net::Timestamp; - use super::{DecodeClock, PSI_INTERVAL, due, is_complete_section, si_due, slot}; + use super::{Buffer, DecodeClock, Kind, PSI_INTERVAL, due, is_complete_section, si_due, slot, verbatim_buffer}; fn ms(value: u64) -> Timestamp { Timestamp::from_millis(value).unwrap() } + /// Teletext passed through as DVB private data takes its transport buffer's rate from + /// EN 300 472, so the schedule spreads it no faster than a receiver passes it on; private + /// data nothing identifies has no buffers. + #[test] + fn teletext_drains_at_its_own_rate() { + let kind = Kind::Verbatim { + stream_type: 0x06, + framing: super::catalog::Framing::Pes, + stream_id: None, + }; + let descriptor = |tag| super::catalog::Descriptor { + tag, + data: bytes::Bytes::from_static(b"eng\x09\x00"), + }; + assert_eq!( + verbatim_buffer(&kind, &[descriptor(0x56)]), + Some(Buffer { + rate: 6_750_000, + size: None, + }) + ); + assert_eq!(verbatim_buffer(&kind, &[descriptor(0x0a)]), None); + } + /// Push a decode-order PTS stream (90 kHz) through the decode clock under a catalog /// `jitter` and a declared reorder `floor`, flushing at the end, and return the DTS per /// frame. diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index a060ea0e3af..d98c3ad1b5b 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -1055,11 +1055,44 @@ async fn export_pcr_backfills_a_coarse_cadence() { /// is due and decoded on its own. #[tokio::test(start_paused = true)] async fn export_splits_a_multi_frame_ac3_pes() { - const AC3_PID: u16 = 0x104; - // A 32 kb/s 48 kHz 2/0 sync frame: 128 bytes, 1,536 samples (32 ms). + let pes = exported_ac3(ac3_sync().repeat(3)).await; + assert_eq!(pes.len(), 3, "one PES a sync frame: {pes:?}"); + for (k, &(pts, bytes)) in pes.iter().enumerate() { + assert_eq!(bytes, 128, "frame {k} carries one sync frame"); + assert_eq!( + pts, + pes[0].0 + k as u64 * 2_880, + "frame {k} is presented 32 ms after the last" + ); + } +} + +/// A sync frame cut short of the length its header gives, as a source stopped mid-frame +/// leaves at the end of its last PES, is no use to a decoder, so the export drops it rather +/// than pass the PES on whole: after whole frames, or alone. +#[tokio::test(start_paused = true)] +async fn export_drops_a_trailing_partial_ac3_frame() { + let mut payload = ac3_sync().repeat(2); + payload.extend_from_slice(&ac3_sync()[..60]); + let pes = exported_ac3(payload).await; + assert_eq!(pes.len(), 2, "the whole frames, each its own PES: {pes:?}"); + assert!(pes.iter().all(|&(_, bytes)| bytes == 128), "{pes:?}"); + + let pes = exported_ac3(ac3_sync()[..60].to_vec()).await; + assert!(pes.is_empty(), "a lone cut frame goes nowhere: {pes:?}"); +} + +/// A 32 kb/s 48 kHz 2/0 AC-3 sync frame: 128 bytes, 1,536 samples (32 ms). +fn ac3_sync() -> Vec { let mut sync = vec![0x0b, 0x77, 0x00, 0x00, 0x00, 0x40, 0x40]; sync.resize(128, 0x00); + sync +} +/// Export one PES of `payload` on an AC-3 passthrough PID beside bbb's media, and return the +/// PTS of every PES on that PID and how many bytes each carries. +async fn exported_ac3(payload: Vec) -> Vec<(u64, usize)> { + const AC3_PID: u16 = 0x104; let data = include_bytes!("test_data/bbb.ts"); let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); @@ -1091,7 +1124,7 @@ async fn export_splits_a_multi_frame_ac3_pes() { ac3.write(Frame { timestamp: Timestamp::from_millis(1_410).unwrap(), duration: None, - payload: Bytes::from(sync.repeat(3)), + payload: Bytes::from(payload), keyframe: true, }) .unwrap(); @@ -1133,15 +1166,7 @@ async fn export_splits_a_multi_frame_ac3_pes() { last.1 += body.len(); } } - assert_eq!(pes.len(), 3, "one PES a sync frame: {pes:?}"); - for (k, &(pts, bytes)) in pes.iter().enumerate() { - assert_eq!(bytes, 128, "frame {k} carries one sync frame"); - assert_eq!( - pts, - pes[0].0 + k as u64 * 2_880, - "frame {k} is presented 32 ms after the last" - ); - } + pes } /// Full SCTE-35 round-trip: import `bbb.ts` (real H.264 + AAC) into a broadcast diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index 9f300afaa8c..0d3324835e8 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -240,6 +240,9 @@ pub(super) struct Schedule { buffers: HashMap, /// Units sent into a decoder buffer: their PID, due slot and bytes held. decoding: VecDeque<(u16, u128, usize)>, + /// Every source has ended, so a unit that misses its deadline is the tail going out late + /// rather than an output that cannot keep up. + ended: bool, } impl Schedule { @@ -253,9 +256,15 @@ impl Schedule { last: None, buffers: HashMap::new(), decoding: VecDeque::new(), + ended: false, } } + /// Every source has ended: what is queued still goes out, late if it must. + pub fn end(&mut self) { + self.ended = true; + } + /// Pad to `rate` bits per second from the next slot on, or stop padding. pub fn set_rate(&mut self, rate: Option) { self.rate = rate; @@ -307,6 +316,7 @@ impl Schedule { self.next = None; self.horizon = None; self.last = None; + self.ended = false; } /// The first slot `unit` may go out in: a window ahead of its due slot, or only the slot @@ -363,7 +373,9 @@ impl Schedule { Some(rate) => { let allowed = (packets_before(index + 1, rate) - packets_before(index, rate)).max(1); let take = self.admit(index, allowed as usize - 1); - if let Some(unit) = self.missed(index, &take) { + if let Some(unit) = self.missed(index, &take) + && !self.ended + { anyhow::bail!( "MPEG-TS output missed a decode deadline on PID {} at {rate} b/s; raise the delay or the multiplex rate", unit.pid, @@ -567,6 +579,22 @@ mod tests { assert!(err.is_some(), "a burst past the window must fail the export"); } + /// Once every source has ended, a unit the window cannot carry by its due slot is the + /// tail going out late, not a failure. + #[test] + fn a_burst_too_big_for_the_window_goes_out_late_at_the_end() { + let mut schedule = Schedule::new(Duration::from_millis(50)); + schedule.set_rate(Some(RATE)); + schedule.set_buffer(1, VIDEO); + schedule.push(1, ms(1_000), unit(1, 200), true); + schedule.end(); + let sent: usize = drain(&mut schedule) + .into_iter() + .map(|(_, per_pid, _)| per_pid.get(&1).copied().unwrap_or(0)) + .sum(); + assert_eq!(sent, 200, "the whole unit goes out"); + } + /// The unit due first goes first, whatever order the PIDs were pushed in. #[test] fn the_earliest_deadline_goes_first() { diff --git a/test/ts/compliance.py b/test/ts/compliance.py index 188a6f052f3..ccf7ec68579 100755 --- a/test/ts/compliance.py +++ b/test/ts/compliance.py @@ -348,6 +348,8 @@ def check_service_descriptors(analysis: dict) -> Check: # parameters are transcribed from the specs named beside each table. TB_SIZE = 512 +# ETSI EN 300 472 5: teletext's transport buffer is smaller than the 512 bytes of 2.4.2.3. +TELETEXT_TB_SIZE = 480 # 2.4.2.6: TB must empty at least once a second. TB_EMPTY_S = 1.0 # PTS/DTS are 33 bits at 90 kHz. @@ -444,7 +446,9 @@ class Params: label: str rx: float + tb: float = TB_SIZE # Video: MB and EB with the leak rate between them. Audio: the main buffer B alone. + # Teletext: neither, only its transport buffer. mb: float = 0.0 eb: float = 0.0 rbx: float = 0.0 @@ -773,6 +777,8 @@ def stream_kind(stream: Stream) -> str | None: return STREAM_KINDS[kind] if kind == 0x06 and tags & {"DVB_AC3_descriptor", "AC3_descriptor"}: return "ac3-dvb" + if kind == 0x06 and "teletext_descriptor" in tags: + return "teletext" if kind == 0x06 and f"registration:{int.from_bytes(b'Opus', 'big')}" in tags: return "opus" if kind == 0x06 and tags & {"DVB_enhanced_AC3_descriptor", "enhanced_AC3_descriptor"}: @@ -789,6 +795,9 @@ def stream_params(kind: str, stream: Stream) -> tuple[Params, object]: return avc_params(stream.sps), None if kind == "hevc": return hevc_params(stream.sps), None + if kind == "teletext": + # ETSI EN 300 472 5: a 480-byte transport buffer drained at 6.75 Mb/s. + return Params(label="teletext", rx=6_750_000, tb=TELETEXT_TB_SIZE), None if kind == "adts": adts_frame(es, 0) config = ((es[2] & 0x01) << 2) | (es[3] >> 6) @@ -945,8 +954,8 @@ def simulate( busy_since = start leave = max(end, max(leave, start) + 188 * 8 / params.rx) fill = (leave - end) * params.rx / 8 - grade.peak("TB", fill, TB_SIZE) - if fill > TB_SIZE + 0.5: + grade.peak("TB", fill, params.tb) + if fill > params.tb + 0.5: grade.flag("TB overflow") if busy_since is not None and leave - busy_since > TB_EMPTY_S: grade.flag("TB not emptied within 1 s") @@ -1030,7 +1039,7 @@ def leak(until: float) -> None: grade.peak("MB", mb_header + mb_payload, params.mb) if mb_header + mb_payload > params.mb + 0.5: grade.flag("MB overflow") - else: + elif params.b: if header: headers.append((offset, header)) b_header += header @@ -1096,7 +1105,9 @@ def check_tstd(ts_path: str, packet_size: int, scan: Scan) -> Check: raise Refused("the HRD-scheduled MB to EB transfer (H.222.0 2.14.3.1) is not modelled") stream.sps = read_sps(ts_path, stream.pid, 7 if kinds[stream.pid] == "avc" else 33) params, parse = stream_params(kinds[stream.pid], stream) - if parse is None: + if kinds[stream.pid] == "teletext": + units = [] + elif parse is None: units = video_units(stream, kinds[stream.pid]) else: units = access_units(stream, parse) diff --git a/test/ts/tstd-controls.py b/test/ts/tstd-controls.py index ccebb6f8f07..0aa369365fc 100755 --- a/test/ts/tstd-controls.py +++ b/test/ts/tstd-controls.py @@ -26,6 +26,8 @@ two AUs, one PES an AU without its own timestamp refused adaptation burst four payload-less packets after an AU TB overflow (they cost TB) duplicate packet one AU's first packet sent twice (2.4.3.3) must pass (TB only, not MB) + teletext burst eight teletext packets back to back TB overflow (EN 300 472's 480 B, + where 2.4.2.3's 512 B holds them) A model that passes everything, or fails everything, cannot tell these apart. """ @@ -45,7 +47,7 @@ # The Kyrion SPS (High@4.0, NAL HRD 1.935 Mb/s CBR, 755 kbit CPB), so the synthetic # streams get the same declared buffers. SPS = bytes.fromhex("6764 0028 acd1 0078 044f de03 6a02 0202 8000 01f4 8000 7530 7500 0762 0002 e11f af7f 072a 1629 92") -VIDEO_PID, PMT_PID = 0x100, 0x1000 +VIDEO_PID, TELETEXT_PID, PMT_PID = 0x100, 0x101, 0x1000 RATE = 10_000_000 SLOT_S = 188 * 8 / RATE @@ -96,7 +98,12 @@ def stamp(prefix: int, ticks: int) -> bytes: def synthetic(path: str, layout: str) -> None: """Write a compliant 2 s stream, bent into `layout` at one access unit.""" pat = section(0x00, 1, (1).to_bytes(2, "big") + (0xE000 | PMT_PID).to_bytes(2, "big")) - pmt = section(0x02, 1, bytes([0xE1, 0x00, 0xF0, 0x00, 0x1B, 0xE1, 0x00, 0xF0, 0x00])) + streams = bytes([0x1B, 0xE1, 0x00, 0xF0, 0x00]) + if layout == "teletext burst": + # Private data with a teletext_descriptor (EN 300 468 6.2.43): English, initial page 100. + teletext = bytes([0x56, 0x05]) + b"eng" + bytes([0x09, 0x00]) + streams += bytes([0x06, 0xE1, 0x01, 0xF0, len(teletext)]) + teletext + pmt = section(0x02, 1, bytes([0xE1, 0x00, 0xF0, 0x00]) + streams) cc: dict[int, int] = {} out: list[bytes] = [] @@ -134,6 +141,13 @@ def pes(units: list[bytes], dts_s: float) -> None: if layout == "adaptation burst" and k == 20: for _ in range(4): emit(VIDEO_PID, adaptation=b"\x00") + if layout == "teletext burst" and k == 20: + # One PES of eight packets, sent back to back: at 10 Mb/s against a 6.75 Mb/s + # drain each adds 61 bytes to the transport buffer, 489 by the eighth. + ticks = round((1.0 + 0.5 + k * 0.04) * compliance.PTS_HZ) + data = bytes.fromhex("000001bd 05be 8480 05") + stamp(2, ticks) + b"\x10" + b"\xff" * (8 * 184 - 15) + for at in range(0, len(data), 184): + emit(TELETEXT_PID, data[at : at + 184], pusi=at == 0) slots = round(0.04 / SLOT_S) while len(out) - begin < slots: if len(out) - begin == slots // 2: @@ -182,6 +196,7 @@ def build(path: str) -> str: ("two AUs, one PES", built("two AUs, one PES"), "several access units"), ("adaptation burst", built("adaptation burst"), {"TB overflow"}), ("duplicate packet", built("duplicate packet"), "pass"), + ("teletext burst", built("teletext burst"), {"TB overflow"}), ] From 6f6123a7f41a73925e41ff3802ab7ef42af3c6cd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 15:30:16 -0700 Subject: [PATCH 30/40] style(moq-mux): format, and keep the export drift cases under nextest's slow mark Co-Authored-By: Claude Opus 5.5 --- .../src/container/ts/export_timing_test.rs | 8 ++--- rs/moq-mux/src/jitter.rs | 34 +++++++++++++++---- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export_timing_test.rs b/rs/moq-mux/src/container/ts/export_timing_test.rs index 4501286579e..256f538968f 100644 --- a/rs/moq-mux/src/container/ts/export_timing_test.rs +++ b/rs/moq-mux/src/container/ts/export_timing_test.rs @@ -16,8 +16,8 @@ use mpeg2ts::ts::{ReadTsPacket, TsPacketReader, TsPayload}; use tokio::time::Instant; use crate::catalog::hang::Container as HangContainer; -use crate::container::ts::{Export, ExportStats}; use crate::container::ts::export::PCR_INTERVAL; +use crate::container::ts::{Export, ExportStats}; use crate::container::{Container as _, Frame, Producer}; const SPS: &[u8] = &[0x67, 0x42, 0xc0, 0x1f, 0xde]; @@ -522,15 +522,15 @@ async fn a_skip_while_running_keeps_one_clock() { // turn once the walk passes the delay, and one running fast makes the buffer grow without bound. // The output's clock may follow it only as fast as 13818-1 lets a system clock change, so // reaching a source 30 ppm off takes hours (the jitter buffer's own tests run a day); these -// cases run twenty minutes of a source at the limit, long enough to measure its rate. +// cases run five minutes of a source at the limit, long enough to measure its rate. /// How far the instant each slot goes out may wander against the source over a run: what /// reaching a source at the limit costs under the slew limit (30 ppm squared over twice /// 0.075 Hz/s of 27 MHz), and a slot. const DRIFT_SLACK: Duration = Duration::from_millis(162 + 25); -/// Twenty minutes of media. -const DRIFT_TICKS: u64 = 1_200_000_000 / VIDEO_US; +/// Five minutes of media. +const DRIFT_TICKS: u64 = 300_000_000 / VIDEO_US; /// The spread of (slot sent − source's clock at the slot) after the first 10 s over `ticks` /// of media, the drops, and the export's stats at the end. diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index 5cfb0fd82c4..deef8b3430e 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -250,8 +250,14 @@ impl Buffer { /// Queue a frame, or drop it if it cannot make its deadline. Fails once the source's clock /// runs further off ours than the clock may follow. pub fn push(&mut self, key: K, arrival: Arrival) -> anyhow::Result { - let held = self.acquire.as_ref().is_some_and(|acquire| acquire.tracks.contains_key(&key)); - let Some(clock) = self.clock.filter(|_| !held && !(self.released && self.jumps(&key, &arrival))) else { + let held = self + .acquire + .as_ref() + .is_some_and(|acquire| acquire.tracks.contains_key(&key)); + let Some(clock) = self + .clock + .filter(|_| !held && !(self.released && self.jumps(&key, &arrival))) + else { self.hold(key, arrival); return Ok(Push::Queued); }; @@ -272,7 +278,8 @@ impl Buffer { let deadline = self.deadline(&clock, arrival.decode); let restarted = track.restart != arrival.restart && (track.generation == clock.generation || !self.lands(deadline, arrival.arrived)); - let leapt = track.skip != arrival.skip && deadline.is_some_and(|deadline| deadline > self.bound(arrival.arrived)); + let leapt = + track.skip != arrival.skip && deadline.is_some_and(|deadline| deadline > self.bound(arrival.arrived)); restarted || leapt } @@ -343,7 +350,10 @@ impl Buffer { /// `acquiring`. fn judge(&self, clock: &Clock, frame: &Arrival, acquiring: bool) -> (Option, bool) { let deadline = self.deadline(clock, frame.decode); - (deadline, acquiring && deadline.is_none_or(|deadline| deadline < frame.read)) + ( + deadline, + acquiring && deadline.is_none_or(|deadline| deadline < frame.read), + ) } /// Queue a frame on `generation` due at `deadline`, unless it is `stale` or cannot make it. @@ -988,7 +998,11 @@ mod tests { assert!(run.worst <= bound, "{ppm} ppm: held {} s off the delay", run.worst); let drift = run.drift.expect("a measurement") * 1e6; assert!((drift + ppm).abs() < 0.1, "{ppm} ppm: measured {drift} ppm"); - assert!((run.clock * 1e6 - ppm).abs() < 0.5, "{ppm} ppm: the clock settled at {}", run.clock * 1e6); + assert!( + (run.clock * 1e6 - ppm).abs() < 0.5, + "{ppm} ppm: the clock settled at {}", + run.clock * 1e6 + ); assert_eq!(run.out_of_tolerance, 0, "{ppm} ppm"); } } @@ -1005,9 +1019,15 @@ mod tests { .await; assert!(run.err.is_none(), "{:?}", run.err); let (before, after) = (run.before.unwrap() * 1e6, run.drift.unwrap() * 1e6); - assert!((after - before).abs() < 0.01, "the estimate moved from {before} to {after} ppm"); + assert!( + (after - before).abs() < 0.01, + "the estimate moved from {before} to {after} ppm" + ); let (before, after) = (run.clock_before * 1e6, run.clock * 1e6); - assert!((after - before).abs() < 0.01, "the clock moved from {before} to {after} ppm"); + assert!( + (after - before).abs() < 0.01, + "the clock moved from {before} to {after} ppm" + ); } /// A source further off than the clock may follow is counted, and fails once it has used From 559a35244aaf284e68164381067b1b7e1c9c04c8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 15:54:08 -0700 Subject: [PATCH 31/40] refactor(moq-mux): name the export's stats ts::export::Stats The export module is public so its Stats sits beside the import's ts::Stats; ts::Export is re-exported as before. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export.rs | 10 +++++----- rs/moq-mux/src/container/ts/export_timing_test.rs | 5 +++-- rs/moq-mux/src/container/ts/mod.rs | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 6ed0489535a..749019b9130 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -3,7 +3,7 @@ //! [`Export`] subscribes to a MoQ broadcast and produces MPEG-TS: PAT/PMT program //! tables and PES packets, packetized into 188-byte TS packets, with the PCR //! riding its own adaptation-field-only packets on a fixed media-time grid. -//! Output is sliced on that grid rather than per media frame ([`Schedule`]): +//! Output is sliced on that grid rather than per media frame (the schedule): //! each [`Frame`] is one slot's clock packet plus the bytes belonging to it, //! stamped at the slot boundary, so the clock a receiver recovers from byte //! position agrees with the values, and a pacing caller releases each slot at @@ -13,7 +13,7 @@ //! so tracks interleave in `(DTS, PID)` order whatever the arrival skew between them, //! and the output keeps the source's pace. //! -//! Video flows through [`ExportSource`], which normalizes every H.264/H.265 +//! Video flows through `ExportSource`, which normalizes every H.264/H.265 //! source to length-prefixed NALU plus a resolved avcC/hvcC (parsing in-band //! avc3/hev1 parameter sets out of the bitstream, or taking the catalog //! `description` for out-of-band avc1/hvc1). The muxer then does one @@ -92,7 +92,7 @@ fn sanitize_mux_rate(rate: u64) -> Option { /// How an [`Export`]'s release clock is keeping up with the source. See [`Export::stats`]. #[derive(Clone, Debug, Default, PartialEq)] #[non_exhaustive] -pub struct ExportStats { +pub struct Stats { /// Frames dropped for arriving after their deadline, and the frames a video track then /// dropped waiting for its next keyframe. pub dropped: u64, @@ -1307,8 +1307,8 @@ impl Export { } /// How the release clock is keeping up with the source. - pub fn stats(&self) -> ExportStats { - ExportStats { + pub fn stats(&self) -> Stats { + Stats { dropped: self.jitter.dropped(), drift: self.jitter.drift().map(|drift| drift * 1e6), out_of_tolerance: self.jitter.out_of_tolerance(), diff --git a/rs/moq-mux/src/container/ts/export_timing_test.rs b/rs/moq-mux/src/container/ts/export_timing_test.rs index 256f538968f..0cd4d7af435 100644 --- a/rs/moq-mux/src/container/ts/export_timing_test.rs +++ b/rs/moq-mux/src/container/ts/export_timing_test.rs @@ -16,8 +16,9 @@ use mpeg2ts::ts::{ReadTsPacket, TsPacketReader, TsPayload}; use tokio::time::Instant; use crate::catalog::hang::Container as HangContainer; +use crate::container::ts::Export; use crate::container::ts::export::PCR_INTERVAL; -use crate::container::ts::{Export, ExportStats}; +use crate::container::ts::export::Stats; use crate::container::{Container as _, Frame, Producer}; const SPS: &[u8] = &[0x67, 0x42, 0xc0, 0x1f, 0xde]; @@ -534,7 +535,7 @@ const DRIFT_TICKS: u64 = 300_000_000 / VIDEO_US; /// The spread of (slot sent − source's clock at the slot) after the first 10 s over `ticks` /// of media, the drops, and the export's stats at the end. -async fn on_a_scaled_clock(scale: f64, ticks: u64) -> (Duration, u64, ExportStats) { +async fn on_a_scaled_clock(scale: f64, ticks: u64) -> (Duration, u64, Stats) { let mut live = Live::new(scale, 0); let mut leg = live.join(Duration::ZERO).await; live.run(ticks, &mut [&mut leg], |_| false).await; diff --git a/rs/moq-mux/src/container/ts/mod.rs b/rs/moq-mux/src/container/ts/mod.rs index 181adcec497..2be3c4d49c1 100644 --- a/rs/moq-mux/src/container/ts/mod.rs +++ b/rs/moq-mux/src/container/ts/mod.rs @@ -19,7 +19,7 @@ //! only the newest group. mod adts; -mod export; +pub mod export; mod import; mod mux_rate; mod programs; @@ -32,7 +32,7 @@ mod si; mod catalog; pub use catalog::{Catalog, Descriptor, Ext, Framing, Mpegts, Program, SiEntry, Track, Verbatim}; -pub use export::*; +pub use export::Export; pub use import::*; pub use programs::Programs; From 7a75ff34aa5a141b8ea204d2120d6182a28e8481 Mon Sep 17 00:00:00 2001 From: t0ms Date: Fri, 2 Oct 2026 10:58:23 +0100 Subject: [PATCH 32/40] test(moq-mux): a track sent later than the delay loses nothing Joined at the live edge with one track sent 700 ms behind the other, the export should drop no frame. At 559a35244 it drops 543-556 when the later track is audio and 285-293 when it is video: the acquisition anchors on the frame with the most slack across all tracks, and the floor steers on it, which leaves the other track the delay less the difference. --- .../src/container/ts/export_timing_test.rs | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/rs/moq-mux/src/container/ts/export_timing_test.rs b/rs/moq-mux/src/container/ts/export_timing_test.rs index 0cd4d7af435..b910365ed26 100644 --- a/rs/moq-mux/src/container/ts/export_timing_test.rs +++ b/rs/moq-mux/src/container/ts/export_timing_test.rs @@ -518,6 +518,44 @@ async fn a_skip_while_running_keeps_one_clock() { assert_one_clock(|tick| (6 * GOP + 7..7 * GOP).contains(&tick)).await; } +// A TS source sends its video up to a second ahead of its decode time and its audio just in time, +// and a passed-through AC-3 PES of nine sync frames arrives only once its 288 ms are complete, so +// audio of a given decode time can reach the receiver more than a delay after the video. A clock +// anchored, or steered, on the frame with the most slack gives the other track the delay less that +// difference, and every one of its frames then goes late. + +/// How much later than the other track one is sent: past [`DELAY`]. +const LONG_LAG: i64 = 700_000; + +/// A receiver joining at a group boundary, or mid-group, with one track sent [`LONG_LAG`] after +/// the other, renders every frame that arrives once its tracks are all running. +#[tokio::test(start_paused = true)] +async fn a_track_sent_later_than_the_delay_loses_nothing() { + let mut lost = Vec::new(); + for lag in [LONG_LAG, -LONG_LAG] { + for join in [3 * GOP, JOIN] { + let mut live = Live::new(1.0, lag); + live.run(join, &mut [], |_| false).await; + let mut leg = live.join(Duration::ZERO).await; + live.run(join + 20 * GOP, &mut [&mut leg], |_| false).await; + let dropped = leg.export.dropped(); + if dropped > 0 { + lost.push(format!( + "{} sent {} ms late, joined {} frames into a group: {dropped} dropped", + if lag > 0 { "audio" } else { "video" }, + lag.abs() / 1_000, + join % GOP, + )); + } + } + } + assert!( + lost.is_empty(), + "frames went late on a clean source:\n{}", + lost.join("\n") + ); +} + // A source's clock is never the receiver's. A transport stream's 27 MHz may be off by 30 ppm, // which walks a fixed anchor 108 ms an hour: a source running slow makes every frame late in // turn once the walk passes the delay, and one running fast makes the buffer grow without bound. From 01fedaedd9d44ac4ffad40e385674e1947af9f7b Mon Sep 17 00:00:00 2001 From: t0ms Date: Fri, 2 Oct 2026 10:58:23 +0100 Subject: [PATCH 33/40] fix(moq-mux): anchor and steer on the track sent latest, after hearing from every track Scratch, offered for discussion. Each track's freshest frame bounds its slack; the acquisition anchors on the track with the least of those, and each steering step's floor is the least of the per-track floors. The acquisition waits, two delays at most, until every catalogued track has delivered a frame, since the one not yet heard from may be the one sent latest. --- rs/moq-mux/src/container/ts/export.rs | 1 + rs/moq-mux/src/jitter.rs | 102 +++++++++++++++++++------- 2 files changed, 76 insertions(+), 27 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 749019b9130..0591f9f7ba2 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -1261,6 +1261,7 @@ impl Export { track.kind = kind; self.refresh(name, track, pids[name], entry.descriptors.clone()); } + self.jitter.expect(self.tracks.values().map(|track| track.pid)); Ok(()) } diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index deef8b3430e..3ce52b94377 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -1,7 +1,7 @@ //! A jitter buffer that releases several tracks' frames in one decode order, each a fixed //! delay after its decode time. -use std::collections::{BTreeMap, VecDeque}; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::pin::Pin; use std::task::Poll; use std::time::Duration; @@ -100,10 +100,13 @@ pub(crate) struct Buffer { tracks: BTreeMap>, /// The latest deadline given out. horizon: Option, + /// The tracks an acquisition waits to hear from, bounded, so that it anchors on whichever + /// of them is sent latest against its decode time. + expect: BTreeSet, /// Whether a frame has gone out since the clock started. released: bool, /// What the clock is steered on, since it last started. - steer: Steer, + steer: Steer, timer: Option>>, dropped: u64, /// Steps at which the source's clock ran further off ours than the clock may follow. @@ -148,14 +151,22 @@ struct Acquire { tracks: BTreeMap, } -impl Acquire { - /// The freshest frame: the one read least behind its decode time. +impl Acquire { + /// The frame to anchor on: each track's freshest frame (the one read least behind its + /// decode time) is that track's slack less its queueing, and of those, the one with the least + /// slack, so the track sent latest against its decode time still has the delay. fn fresh(&self) -> &Arrival { let behind = |frame: &Arrival| since(self.since, frame.read) - frame.decode.as_nanos() as i128; - self.frames - .iter() - .map(|(_, frame)| frame) - .min_by_key(|frame| behind(frame)) + let mut freshest: BTreeMap<&K, &Arrival> = BTreeMap::new(); + for (key, frame) in &self.frames { + let best = freshest.entry(key).or_insert(frame); + if behind(frame) < behind(best) { + *best = frame; + } + } + freshest + .into_values() + .max_by_key(|frame| behind(frame)) .expect("an acquisition holds a frame") } @@ -178,10 +189,9 @@ impl Acquire { } /// What the clock is steered on. -#[derive(Default)] -struct Steer { +struct Steer { /// This step's floor. - floor: Option, + floor: Option>, /// Each step's floor, as its decode time and phase in seconds, over the last /// [`RATE_WINDOW`]. Its slope is how much faster the source's clock runs than ours. phase: VecDeque<(f64, f64)>, @@ -191,12 +201,23 @@ struct Steer { source: Option, } -/// The frame with the most slack in a step: the one queued least on its way. -struct Floor { - /// Its slack past the delay, less how far the drift has moved its deadline, in seconds. - phase: f64, - /// When it decodes, in seconds. - at: f64, +impl Default for Steer { + fn default() -> Self { + Self { + floor: None, + phase: VecDeque::new(), + steered: 0.0, + source: None, + } + } +} + +/// Each track's frame with the most slack in a step: the one queued least on its way. The +/// step's floor is the track with the least of those, the one sent latest against its decode time. +struct Floor { + /// Per track, its slack past the delay, less how far the drift has moved its deadline, and + /// when it decodes, in seconds. + tracks: BTreeMap, /// When the step's first frame decodes. began: Timestamp, } @@ -231,6 +252,7 @@ impl Buffer { acquire: None, tracks: BTreeMap::new(), horizon: None, + expect: BTreeSet::new(), released: false, steer: Steer::default(), timer: None, @@ -239,6 +261,11 @@ impl Buffer { } } + /// The tracks an acquisition should hear from before it anchors. + pub fn expect(&mut self, keys: impl IntoIterator) { + self.expect = keys.into_iter().collect(); + } + /// Anchor each clock on its first frame instead of acquiring it, for a test that writes a /// broadcast whole before exporting it. #[cfg(test)] @@ -263,7 +290,7 @@ impl Buffer { }; let deadline = self.deadline(&clock, arrival.decode); if self.released && !self.delay.is_zero() { - self.steer(deadline, &arrival)?; + self.steer(&key, deadline, &arrival)?; } Ok(self.queue(key, arrival, clock.generation, deadline, false)) } @@ -298,7 +325,9 @@ impl Buffer { *mid |= !arrival.sync; let now = arrival.read; acquire.frames.push((key, arrival)); - if grouped || self.replay || self.delay.is_zero() || acquire.end(self.delay) <= now { + let acquire = self.acquire.as_ref().expect("just held"); + let heard = self.heard(acquire); + if (grouped && heard) || self.replay || self.delay.is_zero() || self.ends(acquire) <= now { self.acquired(); } } @@ -407,7 +436,7 @@ impl Buffer { /// Account for the slack a frame arrived with, and every [`STEER`] of decode time step the /// clock's drift toward the source's rate plus a pull back to the delay, as far as /// [`MAX_SLEW`] allows. - fn steer(&mut self, deadline: Option, arrival: &Arrival) -> anyhow::Result<()> { + fn steer(&mut self, key: &K, deadline: Option, arrival: &Arrival) -> anyhow::Result<()> { let (Some(clock), Some(deadline)) = (self.clock, deadline) else { return Ok(()); }; @@ -418,13 +447,13 @@ impl Buffer { // has moved its deadline. let slack = since(arrival.read, deadline) as f64 / 1e9; let phase = slack - delay - (self.steer.steered + elapsed * clock.drift); - let floor = self.steer.floor.get_or_insert(Floor { - phase, - at: now, + let floor = self.steer.floor.get_or_insert_with(|| Floor { + tracks: BTreeMap::new(), began: arrival.decode, }); - if phase > floor.phase { - (floor.phase, floor.at) = (phase, now); + let best = floor.tracks.entry(key.clone()).or_insert((phase, now)); + if phase > best.0 { + *best = (phase, now); } if arrival.decode.as_nanos().saturating_sub(floor.began.as_nanos()) < STEER.as_nanos() { return Ok(()); @@ -434,9 +463,14 @@ impl Buffer { return Ok(()); }; let floor = self.steer.floor.take().expect("a floor"); + let (phase, at) = floor + .tracks + .into_values() + .min_by(|a, b| a.0.total_cmp(&b.0)) + .expect("a floor holds a track"); self.steer.steered += elapsed * clock.drift; let phases = &mut self.steer.phase; - phases.push_back((floor.at, floor.phase)); + phases.push_back((at, phase)); while phases.front().is_some_and(|&(at, _)| now - at > RATE_WINDOW) { phases.pop_front(); } @@ -510,7 +544,21 @@ impl Buffer { /// When the acquisition under way ends, unless a track starts a new group first. fn acquired_by(&self) -> Option { - self.acquire.as_ref().map(|acquire| acquire.end(self.delay)) + self.acquire.as_ref().map(|acquire| self.ends(acquire)) + } + + /// Whether every track it expects has delivered a frame to `acquire`. + fn heard(&self, acquire: &Acquire) -> bool { + self.expect.iter().all(|key| acquire.tracks.contains_key(key)) + } + + /// When `acquire` ends: a track not yet heard from may be the one sent latest, so until + /// every expected track has delivered it waits, two delays at most. + fn ends(&self, acquire: &Acquire) -> Instant { + match self.heard(acquire) { + true => acquire.end(self.delay), + false => acquire.since + 2 * self.delay, + } } /// The next frame whose deadline has come, in [`Self::front`] order. From fe7cec10657957da3bd51c884dde6552c7495317 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 10:04:23 -0700 Subject: [PATCH 34/40] fix(moq-mux): release equal decode times by track, not by nanosecond deadline Two frames of one decode time queued either side of a steering step were due nanoseconds apart, so the release order, and with it the TS bytes, followed each exporter's clock state rather than the media (t0ms's 1+1 trace). Release generation by generation, then in decode order, ties by track. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/jitter.rs | 103 +++++++++++++++++++++++++++++++-------- 1 file changed, 84 insertions(+), 19 deletions(-) diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index 3ce52b94377..42873d83f1f 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -56,10 +56,14 @@ pub(crate) enum Push { /// emit them in the same order. A frame that arrives past its deadline would break that /// order, so it is dropped and counted. /// -/// The clock is acquired before anything goes out: frames are held until a track starts a -/// new group, or for a delay at most, and the clock is anchored on the freshest of them, the -/// one that arrived least behind its decode time, so that frame is due a delay after it -/// arrived. A joiner is handed the group it joined at once, already partly old, and +/// The clock is acquired before anything goes out: frames are held until every expected +/// track has delivered (two delays at most) and a track starts a new group, or for a delay +/// at most once all have. Each track's freshest frame, the one that arrived least behind its +/// decode time, shows how far ahead of its decode time the source sends it; the clock is +/// anchored on the track sent latest, so its freshest frame is due a delay after it arrived +/// and every other track has at least the delay. A TS source sends video most of a second +/// ahead and audio just in time, and a clock anchored on the video would make every audio +/// frame late. A joiner is handed the group it joined at once, already partly old, and /// anchoring on its first frame would carry that lag for the whole run. What the anchor /// makes due before the acquisition ended is dropped instead, without counting as late. /// @@ -78,8 +82,9 @@ pub(crate) enum Push { /// The clock follows the source's: a source clock running slower than ours would make /// every frame late in the end, and a faster one would hold more and more. So the clock /// measures the source's rate and runs its decode timeline at it, holding the freshest -/// frames a delay. Each [`STEER`] of decode time gives a floor, the most slack a frame -/// arrived with, which queueing and retransmission only ever lower; the upper envelope of +/// frames of the track sent latest a delay. Each [`STEER`] of decode time gives a floor, the +/// most slack a frame of that track arrived with (the least across tracks of each one's +/// most), which queueing and retransmission only ever lower; the upper envelope of /// the floors over [`RATE_WINDOW`] gives the rate (its slope) and the slack now, so a /// spell of queueing shorter than half the window moves neither. The decode timeline is /// the output's system clock, so it keeps to what ISO/IEC 13818-1 2.4.2.1 allows one: @@ -223,8 +228,8 @@ struct Floor { } struct Track { - /// Frames in arrival order, each with its generation and deadline. - queue: VecDeque<(u64, Instant, T)>, + /// Frames in arrival order, each with its generation, decode time and deadline. + queue: VecDeque<(u64, Timestamp, Instant, T)>, /// The source's restart and skip counters at the last frame. restart: u64, skip: u64, @@ -261,7 +266,8 @@ impl Buffer { } } - /// The tracks an acquisition should hear from before it anchors. + /// The tracks an acquisition waits to hear from, two delays at most, before it anchors: any + /// of them may be the one sent latest against its decode time. pub fn expect(&mut self, keys: impl IntoIterator) { self.expect = keys.into_iter().collect(); } @@ -405,7 +411,9 @@ impl Buffer { _ if track.waiting.is_some() && !arrival.sync => Push::Waiting, Some(deadline) if arrival.arrived <= deadline || self.delay.is_zero() => { track.waiting = None; - track.queue.push_back((generation, deadline, arrival.item)); + track + .queue + .push_back((generation, arrival.decode, deadline, arrival.item)); self.horizon = self.horizon.max(Some(deadline)); Push::Queued } @@ -530,16 +538,19 @@ impl Buffer { ) } - /// The generation, deadline and track of the frame that goes out next: generation by - /// generation, earliest deadline first, ties by track. - fn front(&self) -> Option<(u64, Instant, &K)> { + /// The deadline and track of the frame that goes out next: generation by generation, then + /// in decode order, ties by track. Not in deadline order: two frames of one decode time + /// queued either side of a steering step can be due nanoseconds apart, which would make + /// the order depend on when each was read rather than on the media. + fn front(&self) -> Option<(Instant, &K)> { self.tracks .iter() .filter_map(|(key, track)| { - let (generation, deadline, _) = track.queue.front()?; - Some((*generation, *deadline, key)) + let (generation, decode, deadline, _) = track.queue.front()?; + Some(((*generation, decode.as_nanos(), key), *deadline)) }) - .min() + .min_by(|a, b| a.0.cmp(&b.0)) + .map(|((_, _, key), deadline)| (deadline, key)) } /// When the acquisition under way ends, unless a track starts a new group first. @@ -570,11 +581,11 @@ impl Buffer { self.acquired(); continue; } - let front = self.front().map(|(_, deadline, key)| (deadline, key.clone())); + let front = self.front().map(|(deadline, key)| (deadline, key.clone())); if let Some((deadline, key)) = &front && (self.delay.is_zero() || now >= *deadline) { - let (generation, _, item) = self.tracks.get_mut(key).and_then(|t| t.queue.pop_front()).unwrap(); + let (generation, _, _, item) = self.tracks.get_mut(key).and_then(|t| t.queue.pop_front()).unwrap(); self.released = true; return Poll::Ready(Ready { track: key.clone(), @@ -602,7 +613,7 @@ impl Buffer { #[cfg(test)] pub fn next_deadline(&self) -> Option { self.front() - .map(|(_, deadline, _)| deadline) + .map(|(deadline, _)| deadline) .into_iter() .chain(self.acquired_by()) .min() @@ -1116,6 +1127,60 @@ mod tests { assert_eq!(buffer.dropped(), 0, "trimming the join is not a loss"); } + /// A source that sends one track well ahead of its decode time and another just in time + /// (a TS mux's video and audio) has the clock anchored on the one sent latest, so both + /// keep the delay: anchored on the video, every audio frame would be late. + #[tokio::test(start_paused = true)] + async fn a_joiner_anchors_on_the_track_sent_latest() { + let start = Instant::now(); + let mut buffer = Buffer::new(DELAY); + buffer.expect([1, 2]); + // Video is sent 300 ms ahead of its decode time, audio as it decodes. + buffer.push(1, arrival(start, 300, "v300")).unwrap(); + tokio::time::advance(Duration::from_millis(20)).await; + buffer + .push(2, arrival(start + Duration::from_millis(20), 20, "a20")) + .unwrap(); + tokio::time::advance(Duration::from_millis(100)).await; + assert_eq!(due(&mut buffer), ["a20"], "the audio keeps the delay"); + assert_eq!(buffer.next_deadline(), Some(start + Duration::from_millis(400))); + assert_eq!(buffer.dropped(), 0); + } + + /// The acquisition waits to hear from every track it expects, since the silent one may be + /// the one sent latest, but two delays at most. + #[tokio::test(start_paused = true)] + async fn the_acquisition_waits_two_delays_at_most_for_a_silent_track() { + let start = Instant::now(); + let mut buffer = Buffer::new(DELAY); + buffer.expect([1, 2]); + buffer.push(1, arrival(start, 0, "v0")).unwrap(); + assert_eq!(buffer.next_deadline(), Some(start + 2 * DELAY)); + tokio::time::advance(2 * DELAY).await; + assert_eq!(due(&mut buffer), ["v0"]); + } + + /// Frames of one decode time go out by track, even when a steering step between their + /// pushes put their deadlines a nanosecond apart: the order is the media's, not the + /// moment each was read. + #[tokio::test(start_paused = true)] + async fn a_steering_step_does_not_reorder_a_tie() { + let start = Instant::now(); + let order = |first: (u16, &'static str), second: (u16, &'static str)| { + let mut buffer = Buffer::new(DELAY).replay(); + buffer.push(1, arrival(start, 0, "v0")).unwrap(); + buffer.push(first.0, arrival(start, 100, first.1)).unwrap(); + buffer.clock.as_mut().unwrap().anchor += Duration::from_nanos(1); + buffer.push(second.0, arrival(start, 100, second.1)).unwrap(); + buffer + }; + let mut video_first = order((1, "v100"), (2, "a100")); + let mut audio_first = order((2, "a100"), (1, "v100")); + tokio::time::advance(Duration::from_secs(1)).await; + assert_eq!(due(&mut video_first), ["v0", "v100", "a100"]); + assert_eq!(due(&mut audio_first), ["v0", "v100", "a100"]); + } + /// A frame the acquisition holds that is still in time goes out on time: the /// acquisition ends when the first of them falls due rather than drop it. #[tokio::test(start_paused = true)] From e6d9c2226adc75814c55e6fd1edc50fe37f08944 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 7 Oct 2026 14:50:22 -0700 Subject: [PATCH 35/40] fix(moq-mux): a track reaching a new generation without a marker joins its acquisition While a generation is being acquired, a frame held more than a delay past everything queued joins the acquisition instead of queuing far ahead on the old clock, which held every track back. The acquisition then hears from that track and anchors on it when it is the one sent latest. An acquisition waits only on audio, video and the passthrough streams the schedule models (DVB AC-3, teletext), not sparse PES such as subtitles or ID3. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 2 +- rs/moq-mux/src/container/ts/export.rs | 18 +++++------ rs/moq-mux/src/jitter.rs | 44 +++++++++++++++++++++++++-- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index f91a47bb19c..f72ad192a80 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -484,7 +484,7 @@ receiver's latency. Every frame is muxed that long after its decode time, on a clock that keeps the source's pace and muxes all tracks in decode order whatever their arrival skew: two exporters of one broadcast emit them in one order. The export joins at the newest group and holds its output until it -has heard from every audio, video and PES track (two delays at most) and a +has heard from every audio, video, DVB AC-3 and teletext track (two delays at most) and a track starts its next group, or one of the frames it holds falls due. It then starts the clock on the track the source sends latest against its decode time, from that track's freshest frame: a broadcast TS sends video most of a second diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 20f7f806aac..73c673f1f81 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -1308,17 +1308,13 @@ impl Export { } // The clock anchors on whichever track a source sends latest against its decode time // (audio just in time while video runs most of a second ahead), so it waits to hear - // from each. Not from sections: SCTE-35 and the like are sparse, and would hold every - // join for the full bound. - let continuous = self.tracks.values().filter(|track| { - !matches!( - track.kind, - Kind::Verbatim { - framing: catalog::Framing::Section, - .. - } - ) - }); + // from each continuous one: audio, video, and the passthrough streams the schedule + // models (DVB AC-3, teletext). Not from sparse ones such as SCTE-35, subtitles or ID3, + // which would hold every join for the full bound. + let continuous = self + .tracks + .values() + .filter(|track| !matches!(track.kind, Kind::Verbatim { .. }) || track.buffer.is_some()); self.jitter.expect(continuous.map(|track| track.pid)); Ok(()) } diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index 42873d83f1f..9d191b25847 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -303,7 +303,9 @@ impl Buffer { /// Whether a frame moves its track off the clock: a restart, unless the track is crossing /// one another track already opened a generation for and the frame lands on its clock, or - /// a skip that leapt ahead. + /// a frame held more than a delay past everything queued after a skip, or while a + /// generation is acquired. The latter is a track reaching the new timeline without a + /// marker of its own, which belongs to that generation, not far ahead on the old clock. fn jumps(&self, key: &K, arrival: &Arrival) -> bool { let (Some(clock), Some(track)) = (self.clock, self.tracks.get(key)) else { return false; @@ -311,8 +313,8 @@ impl Buffer { let deadline = self.deadline(&clock, arrival.decode); let restarted = track.restart != arrival.restart && (track.generation == clock.generation || !self.lands(deadline, arrival.arrived)); - let leapt = - track.skip != arrival.skip && deadline.is_some_and(|deadline| deadline > self.bound(arrival.arrived)); + let leapt = (track.skip != arrival.skip || self.acquire.is_some()) + && deadline.is_some_and(|deadline| deadline > self.bound(arrival.arrived)); restarted || leapt } @@ -883,6 +885,42 @@ mod tests { assert_eq!(released(&mut buffer), [("a5020", 1)]); } + /// Not replaying, the same: a track that crosses no restart of its own, sent behind the + /// one that did, joins the generation being acquired once its frame is ahead of the old + /// clock, rather than queuing seconds ahead on it and holding every track back. The new + /// clock anchors on it, the track sent latest, so it keeps the delay. + #[tokio::test(start_paused = true)] + async fn a_track_without_the_restart_joins_the_acquisition() { + let start = Instant::now(); + let mut buffer = Buffer::new(DELAY); + buffer.expect([1, 2]); + buffer.push(1, arrival(start, 0, "v0")).unwrap(); + buffer.push(2, arrival(start, 0, "a0")).unwrap(); + tokio::time::advance(DELAY).await; + assert_eq!(due(&mut buffer), ["v0", "a0"]); + + // The publisher resumes a second later with its timeline five seconds on, and sends + // the audio further behind the video than the delay. + let resume = start + Duration::from_secs(1); + tokio::time::advance(resume - Instant::now()).await; + assert_eq!( + buffer.push(1, after(1, arrival(resume, 5_000, "v5000"))).unwrap(), + Push::Queued + ); + let lag = resume + DELAY + Duration::from_millis(50); + tokio::time::advance(lag - Instant::now()).await; + assert_eq!(buffer.push(2, arrival(lag, 5_000, "a5000")).unwrap(), Push::Queued); + tokio::time::advance(Duration::from_millis(20)).await; + let next = Instant::now(); + assert_eq!(buffer.push(2, arrival(next, 5_020, "a5020")).unwrap(), Push::Queued); + + tokio::time::advance(lag + DELAY - Instant::now()).await; + assert_eq!(released(&mut buffer), [("v5000", 1), ("a5000", 1)]); + tokio::time::advance(Duration::from_millis(20)).await; + assert_eq!(released(&mut buffer), [("a5020", 1)]); + assert_eq!(buffer.dropped(), 0); + } + /// A skipped group whose timeline carried on keeps the clock: a frame it made late is /// dropped, and the next on-time frame goes out on the same generation. #[tokio::test(start_paused = true)] From d7323b085dba610d547c37c39d9d2d2477ab53ca Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 7 Oct 2026 14:55:38 -0700 Subject: [PATCH 36/40] docs(upgrade): the TS export's release counters join ts::stats::Export Co-Authored-By: Claude Opus 5.5 --- doc/setup/upgrade.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/doc/setup/upgrade.md b/doc/setup/upgrade.md index 2de9ea652bf..f0afd0ceec0 100644 --- a/doc/setup/upgrade.md +++ b/doc/setup/upgrade.md @@ -58,9 +58,8 @@ These land with the next breaking release, not the 2026-09-23 train. written that long after its decode time, on a clock that follows the source's; `--max-age` and `--latency-max` are refused with the new flag named. In Rust, moq-mux's `ts::Export::with_max_age` is `with_delay`, and - `ts::Export::stats` returns `ts::stats::Export` (its `streams` rows, plus the - late drops, measured drift and out-of-tolerance count) instead of - `ts::Stats`; pass it to `ts::stats::Log` with `.into()`. + `ts::stats::Export` gains the late drops, measured drift and + out-of-tolerance count beside its `streams` rows. It is no longer `Eq`. - **moq-mux has no clock translators.** `clock::Anchor`, `clock::Lane`, and `SourceMap` (#4667) are gone, along with the importers' `live()`. Publish the source's own timestamps and let the catalog clock map them to wall time; @@ -150,8 +149,7 @@ These land with the next breaking release, not the 2026-09-23 train. - **moq-mux TS stats live in `ts::stats`.** `ts::Stats` is `ts::stats::Snapshot` and `ts::StreamStats` is `ts::stats::Stream`, whose `track` is an owned `String`. `ts::Export::stats` returns - `ts::stats::Export`, which carries only `streams`; feed it to - `stats::Log` with `.into()`. `ts::MultipleProgramsError` is + `ts::stats::Export`; feed it to `stats::Log` with `.into()`. `ts::MultipleProgramsError` is `#[non_exhaustive]`: recover it by downcast and read `programs`. - **@moq/publish drops `OpusConfig.usedtx`.** Chromium's DTX output shifts the audio timeline, so Opus DTX is always off (the WebCodecs default). Remove the From ed6dd465a3530886f114aa59a5bf31b32e0a9ea7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 7 Oct 2026 16:02:50 -0700 Subject: [PATCH 37/40] fix(moq-mux): a new generation keeps each track's lead from the last How far ahead of its decode time a source sends each track belongs to its mux, so the steering steps record each track's slack against the others. A re-acquisition no longer waits to hear from a track with a known lead, and anchors late enough that the track sent latest keeps the delay when it crosses. A track that has not reached the restart stays on the previous generation while its frames land there, instead of going late on the new clock. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 9 +-- rs/moq-mux/src/jitter.rs | 139 ++++++++++++++++++++++++++++++++++----- 2 files changed, 129 insertions(+), 19 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index f72ad192a80..6912d3a8bf9 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -129,10 +129,11 @@ SI matches the selection by DVB `service_id`, which is assumed to equal the PAT moq --connect https://relay.example.com/anon --broadcast event.hang import ts --program all < mux.ts ``` -MPEG-TS export restarts its clock and table cadence after a declared marker, -discarding the old mux buffer. The first new clock packet signals the break and -stdout pacing re-anchors. Every rendition joins the new program generation; -no track is fenced across the marker. +MPEG-TS export starts a new clock after a declared marker, and the first new +clock packet signals the break. Each track goes out on the old clock until it +reaches the marker, then joins the new one. The new clock keeps each track's +lead over the others, so a track the source sends later than the rest crosses +without losing frames. No track is fenced across the marker. MPEG-TS export frames AAC as ADTS, which labels only the AAC Main, LC, SSR, and LTP profiles. HE-AAC and HE-AACv2 go out as their AAC-LC core, and decoders diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index 9d191b25847..ae0301c0f7e 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -73,11 +73,16 @@ pub(crate) enum Push { /// whose clock is acquired the same way, though never ahead of a deadline already given /// out. So does a skip after which a frame would be held more than a delay past everything /// queued: its timeline jumped ahead, as when the skipped groups held a marker the -/// publisher shed. Every track's next frame runs on it: one crossing the same restart joins -/// it when the frame lands on its clock, neither late nor held more than a delay past -/// everything queued, and opens another otherwise. Before anything has gone out there is -/// nothing to keep in step with, so a restart there keeps the clock. No two tracks are -/// ever on different clocks. +/// publisher shed. How far ahead of its decode time the source sends each track belongs to +/// its mux and survives a restart, so the new clock does not wait to hear from a track the +/// previous one measured: it is anchored late enough that the track sent latest keeps the +/// delay when it gets there. A track that has +/// not reached the restart stays on the previous generation while its frames land there. +/// One crossing the same restart joins the new generation when the frame lands on its +/// clock, neither late nor held more than a delay past everything queued, and opens another +/// otherwise. Before anything has gone out there is nothing to keep in step with, so a +/// restart there keeps the clock. Generations go out in turn, so no two tracks are ever on +/// different clocks at once. /// /// The clock follows the source's: a source clock running slower than ours would make /// every frame late in the end, and a faster one would hold more and more. So the clock @@ -100,6 +105,8 @@ pub(crate) struct Buffer { replay: bool, /// The clock every frame is pushed under. clock: Option, + /// The generation before it, for a track the restart has not reached yet. + previous: Option, /// The next generation's frames, held while its clock is acquired. acquire: Option>, tracks: BTreeMap>, @@ -112,6 +119,11 @@ pub(crate) struct Buffer { released: bool, /// What the clock is steered on, since it last started. steer: Steer, + /// Each track's slack in the last steering step, in seconds, against the others'. How far + /// ahead of its decode time a source sends each track belongs to its mux, not to one + /// generation, so a new generation keeps the track sent latest at the delay even before + /// that track reaches it. + leads: BTreeMap, timer: Option>>, dropped: u64, /// Steps at which the source's clock ran further off ours than the clock may follow. @@ -160,7 +172,7 @@ impl Acquire { /// The frame to anchor on: each track's freshest frame (the one read least behind its /// decode time) is that track's slack less its queueing, and of those, the one with the least /// slack, so the track sent latest against its decode time still has the delay. - fn fresh(&self) -> &Arrival { + fn fresh(&self) -> (&K, &Arrival) { let behind = |frame: &Arrival| since(self.since, frame.read) - frame.decode.as_nanos() as i128; let mut freshest: BTreeMap<&K, &Arrival> = BTreeMap::new(); for (key, frame) in &self.frames { @@ -170,8 +182,8 @@ impl Acquire { } } freshest - .into_values() - .max_by_key(|frame| behind(frame)) + .into_iter() + .max_by_key(|(_, frame)| behind(frame)) .expect("an acquisition holds a frame") } @@ -179,7 +191,7 @@ impl Acquire { /// frame's clock, since waiting longer would drop it, and a delay after it began at the /// latest. fn end(&self, delay: Duration) -> Instant { - let fresh = self.fresh(); + let (_, fresh) = self.fresh(); let due = |frame: &Arrival| { let ahead = frame.decode.as_nanos() as i128 - fresh.decode.as_nanos() as i128; offset(fresh.read + delay, ahead) @@ -254,12 +266,14 @@ impl Buffer { delay, replay: false, clock: None, + previous: None, acquire: None, tracks: BTreeMap::new(), horizon: None, expect: BTreeSet::new(), released: false, steer: Steer::default(), + leads: BTreeMap::new(), timer: None, dropped: 0, out_of_tolerance: 0, @@ -294,6 +308,10 @@ impl Buffer { self.hold(key, arrival); return Ok(Push::Queued); }; + if let Some(previous) = self.lagging(&key, &arrival) { + let deadline = self.deadline(&previous, arrival.decode); + return Ok(self.queue(key, arrival, previous.generation, deadline, false)); + } let deadline = self.deadline(&clock, arrival.decode); if self.released && !self.delay.is_zero() { self.steer(&key, deadline, &arrival)?; @@ -301,6 +319,18 @@ impl Buffer { Ok(self.queue(key, arrival, clock.generation, deadline, false)) } + /// The previous generation's clock, for a frame of a track the restart that opened the + /// current one has not reached yet: still on the previous generation, crossing no restart, + /// and on that clock. + fn lagging(&self, key: &K, arrival: &Arrival) -> Option { + let previous = self.previous?; + let track = self.tracks.get(key)?; + let lags = track.generation == previous.generation + && track.restart == arrival.restart + && self.lands(self.deadline(&previous, arrival.decode), arrival.arrived); + lags.then_some(previous) + } + /// Whether a frame moves its track off the clock: a restart, unless the track is crossing /// one another track already opened a generation for and the frame lands on its clock, or /// a frame held more than a delay past everything queued after a skip, or while a @@ -348,14 +378,17 @@ impl Buffer { return; }; let acquiring = !self.replay && !self.delay.is_zero(); - let fresh = match acquiring { + let (key, fresh) = match acquiring { true => acquire.fresh(), - false => &acquire.frames.first().expect("an acquisition holds a frame").1, + false => { + let (key, frame) = acquire.frames.first().expect("an acquisition holds a frame"); + (key, frame) + } }; let mut clock = Clock { generation: acquire.generation, anchor: match acquiring { - true => fresh.read, + true => fresh.read + self.unheard_lag(&acquire, key), false => fresh.arrived, }, base: fresh.decode, @@ -375,7 +408,7 @@ impl Buffer { { clock.anchor += horizon - kept; } - self.clock = Some(clock); + self.previous = self.clock.replace(clock); self.steer = Steer::default(); for (key, frame) in acquire.frames { let (deadline, stale) = self.judge(&clock, &frame, acquiring); @@ -473,6 +506,9 @@ impl Buffer { return Ok(()); }; let floor = self.steer.floor.take().expect("a floor"); + for (key, (phase, _)) in &floor.tracks { + self.leads.insert(key.clone(), *phase); + } let (phase, at) = floor .tracks .into_values() @@ -560,9 +596,30 @@ impl Buffer { self.acquire.as_ref().map(|acquire| self.ends(acquire)) } - /// Whether every track it expects has delivered a frame to `acquire`. + /// Whether every track it expects has delivered a frame to `acquire`, or, once a clock has + /// run, has a lead from it that says how much later than the others it is sent. fn heard(&self, acquire: &Acquire) -> bool { - self.expect.iter().all(|key| acquire.tracks.contains_key(key)) + let known = |key: &K| self.clock.is_some() && self.leads.contains_key(key); + self.expect + .iter() + .all(|key| acquire.tracks.contains_key(key) || known(key)) + } + + /// How much later than `fresh`'s track the latest-sent expected track that `acquire` has + /// not heard from is sent, by the leads the previous generation measured: anchoring that + /// much later keeps that track at the delay when it reaches the new generation. + fn unheard_lag(&self, acquire: &Acquire, fresh: &K) -> Duration { + let Some(lead) = self.leads.get(fresh) else { + return Duration::ZERO; + }; + let lag = self + .expect + .iter() + .filter(|key| !acquire.tracks.contains_key(key)) + .filter_map(|key| self.leads.get(key)) + .map(|other| lead - other) + .fold(0.0, f64::max); + Duration::from_secs_f64(lag) } /// When `acquire` ends: a track not yet heard from may be the one sent latest, so until @@ -645,6 +702,8 @@ impl Buffer { /// Drop every queued frame and the clock, so the next frame starts it afresh. pub fn clear(&mut self) { self.clock = None; + self.previous = None; + self.leads.clear(); self.acquire = None; self.tracks.clear(); self.horizon = None; @@ -921,6 +980,56 @@ mod tests { assert_eq!(buffer.dropped(), 0); } + /// A track sent further behind than the new generation's acquisition lasts crosses the + /// restart after the new clock is set. The new clock keeps the lead the previous + /// generation measured for it, so its frames land there, and the frames it sends before + /// crossing stay on the previous generation: nothing is dropped and no third generation + /// opens. + #[tokio::test(start_paused = true)] + async fn a_track_crossing_after_the_new_clock_keeps_its_lead() { + // The audio is sent this far behind the video, more than two delays. + const LAG: u64 = 300; + let start = Instant::now(); + let mut buffer = Buffer::new(DELAY); + buffer.expect([1, 2]); + + // (read, track, restarts, decode), every 100 ms of decode time: four seconds, so the + // clock is steered and the leads measured, then a restart onto a timeline 15 seconds on. + let mut frames = Vec::new(); + for step in 0..40 { + frames.push((step * 100, 1, 0, 1_000 + step * 100)); + } + for step in 0..=(LAG / 100 + 39) { + frames.push((step * 100, 2, 0, 1_000 - LAG + step * 100)); + } + for step in 0..20 { + let decode = 20_000 + step * 100; + frames.push((4_000 + step * 100, 1, 1, decode)); + frames.push((4_000 + step * 100 + LAG, 2, 1, decode)); + } + frames.sort(); + + let mut out = Vec::new(); + for (read, key, restart, decode) in frames { + let at = start + Duration::from_millis(read); + tokio::time::advance(at - Instant::now()).await; + out.extend(released(&mut buffer).into_iter().map(|(_, generation)| generation)); + let frame = after(restart, arrival(at, decode, "frame")); + assert_eq!( + buffer.push(key, frame).unwrap(), + Push::Queued, + "track {key} at {decode}" + ); + } + tokio::time::advance(Duration::from_secs(2)).await; + out.extend(released(&mut buffer).into_iter().map(|(_, generation)| generation)); + + assert_eq!(buffer.dropped(), 0); + assert_eq!(out.iter().filter(|&&generation| generation == 0).count(), 83); + assert_eq!(out.iter().filter(|&&generation| generation == 1).count(), 40); + assert_eq!(out.len(), 123, "a third generation opened, or a frame never went out"); + } + /// A skipped group whose timeline carried on keeps the clock: a frame it made late is /// dropped, and the next on-time frame goes out on the same generation. #[tokio::test(start_paused = true)] From 78b7afdc2bfdc256af93e30b3b33270efd565982 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 7 Oct 2026 16:43:27 -0700 Subject: [PATCH 38/40] fix(moq-mux): keep the previous generation ahead of the new one's first frame A lagging track's old-clock frame could be queued after the new generation's first frame was due or released, breaking the generation order the TS export relies on. Queue it on the previous clock only while it is due no later than the current generation's earliest deadline, and stop once that generation has released a frame. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 8 ++-- doc/setup/upgrade.md | 3 +- rs/moq-mux/src/jitter.rs | 81 +++++++++++++++++++++++++++------------- 3 files changed, 63 insertions(+), 29 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 6912d3a8bf9..438c0f4f59e 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -131,9 +131,11 @@ moq --connect https://relay.example.com/anon --broadcast event.hang import ts -- MPEG-TS export starts a new clock after a declared marker, and the first new clock packet signals the break. Each track goes out on the old clock until it -reaches the marker, then joins the new one. The new clock keeps each track's -lead over the others, so a track the source sends later than the rest crosses -without losing frames. No track is fenced across the marker. +reaches the marker, then joins the new one, unless an old-clock frame would go +out after the new clock's first, which is dropped. Once the export has measured +each track's lead over the others (two seconds of media), the new clock keeps +it, so a track the source sends later than the rest crosses without losing +frames. No track is fenced across the marker. MPEG-TS export frames AAC as ADTS, which labels only the AAC Main, LC, SSR, and LTP profiles. HE-AAC and HE-AACv2 go out as their AAC-LC core, and decoders diff --git a/doc/setup/upgrade.md b/doc/setup/upgrade.md index f0afd0ceec0..b03c2abcf3b 100644 --- a/doc/setup/upgrade.md +++ b/doc/setup/upgrade.md @@ -58,7 +58,8 @@ These land with the next breaking release, not the 2026-09-23 train. written that long after its decode time, on a clock that follows the source's; `--max-age` and `--latency-max` are refused with the new flag named. In Rust, moq-mux's `ts::Export::with_max_age` is `with_delay`, and - `ts::stats::Export` gains the late drops, measured drift and + `ts::stats::Export` gains the dropped-frame count (late frames, and the + video frames then dropped waiting for a keyframe), measured drift and out-of-tolerance count beside its `streams` rows. It is no longer `Eq`. - **moq-mux has no clock translators.** `clock::Anchor`, `clock::Lane`, and `SourceMap` (#4667) are gone, along with the importers' `live()`. Publish the diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index ae0301c0f7e..aa24f6c5f5d 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -77,7 +77,8 @@ pub(crate) enum Push { /// its mux and survives a restart, so the new clock does not wait to hear from a track the /// previous one measured: it is anchored late enough that the track sent latest keeps the /// delay when it gets there. A track that has -/// not reached the restart stays on the previous generation while its frames land there. +/// not reached the restart stays on the previous generation while its frames land there and +/// go out before the new generation's first frame; one that would follow it drops as late. /// One crossing the same restart joins the new generation when the frame lands on its /// clock, neither late nor held more than a delay past everything queued, and opens another /// otherwise. Before anything has gone out there is nothing to keep in step with, so a @@ -105,7 +106,8 @@ pub(crate) struct Buffer { replay: bool, /// The clock every frame is pushed under. clock: Option, - /// The generation before it, for a track the restart has not reached yet. + /// The generation before it, for a track the restart has not reached yet, until the + /// current generation's first frame goes out. previous: Option, /// The next generation's frames, held while its clock is acquired. acquire: Option>, @@ -321,16 +323,30 @@ impl Buffer { /// The previous generation's clock, for a frame of a track the restart that opened the /// current one has not reached yet: still on the previous generation, crossing no restart, - /// and on that clock. + /// on that clock, and due no later than the current generation's first frame, so the + /// generations still go out in turn. fn lagging(&self, key: &K, arrival: &Arrival) -> Option { let previous = self.previous?; let track = self.tracks.get(key)?; + let deadline = self.deadline(&previous, arrival.decode); let lags = track.generation == previous.generation && track.restart == arrival.restart - && self.lands(self.deadline(&previous, arrival.decode), arrival.arrived); + && self.lands(deadline, arrival.arrived) + && deadline.is_some_and(|deadline| self.opened().is_none_or(|first| deadline <= first)); lags.then_some(previous) } + /// The earliest deadline queued on the current generation. + fn opened(&self) -> Option { + let generation = self.clock?.generation; + self.tracks + .values() + .flat_map(|track| &track.queue) + .filter(|(queued, ..)| *queued == generation) + .map(|(_, _, deadline, _)| *deadline) + .min() + } + /// Whether a frame moves its track off the clock: a restart, unless the track is crossing /// one another track already opened a generation for and the frame lands on its clock, or /// a frame held more than a delay past everything queued after a skip, or while a @@ -646,6 +662,10 @@ impl Buffer { { let (generation, _, _, item) = self.tracks.get_mut(key).and_then(|t| t.queue.pop_front()).unwrap(); self.released = true; + // A previous-generation frame queued now would go out after this one. + if self.clock.is_some_and(|clock| clock.generation == generation) { + self.previous = None; + } return Poll::Ready(Ready { track: key.clone(), generation, @@ -980,32 +1000,28 @@ mod tests { assert_eq!(buffer.dropped(), 0); } - /// A track sent further behind than the new generation's acquisition lasts crosses the - /// restart after the new clock is set. The new clock keeps the lead the previous - /// generation measured for it, so its frames land there, and the frames it sends before - /// crossing stay on the previous generation: nothing is dropped and no third generation - /// opens. - #[tokio::test(start_paused = true)] - async fn a_track_crossing_after_the_new_clock_keeps_its_lead() { - // The audio is sent this far behind the video, more than two delays. + /// Two tracks every 100 ms of decode time for `steps` frames, the audio sent more than two + /// delays behind the video and running `tail` frames past it, then a restart onto a + /// timeline 15 seconds on: each released frame's generation, and how many were dropped. + async fn cross(steps: u64, tail: u64) -> (Vec, u64) { const LAG: u64 = 300; let start = Instant::now(); let mut buffer = Buffer::new(DELAY); buffer.expect([1, 2]); - // (read, track, restarts, decode), every 100 ms of decode time: four seconds, so the - // clock is steered and the leads measured, then a restart onto a timeline 15 seconds on. + // (read, track, restarts, decode) let mut frames = Vec::new(); - for step in 0..40 { + for step in 0..steps { frames.push((step * 100, 1, 0, 1_000 + step * 100)); } - for step in 0..=(LAG / 100 + 39) { + for step in 0..LAG / 100 + steps + tail { frames.push((step * 100, 2, 0, 1_000 - LAG + step * 100)); } + let restart = steps * 100; for step in 0..20 { let decode = 20_000 + step * 100; - frames.push((4_000 + step * 100, 1, 1, decode)); - frames.push((4_000 + step * 100 + LAG, 2, 1, decode)); + frames.push((restart + step * 100, 1, 1, decode)); + frames.push((restart + step * 100 + LAG, 2, 1, decode)); } frames.sort(); @@ -1014,22 +1030,37 @@ mod tests { let at = start + Duration::from_millis(read); tokio::time::advance(at - Instant::now()).await; out.extend(released(&mut buffer).into_iter().map(|(_, generation)| generation)); - let frame = after(restart, arrival(at, decode, "frame")); - assert_eq!( - buffer.push(key, frame).unwrap(), - Push::Queued, - "track {key} at {decode}" - ); + buffer.push(key, after(restart, arrival(at, decode, "frame"))).unwrap(); } tokio::time::advance(Duration::from_secs(2)).await; out.extend(released(&mut buffer).into_iter().map(|(_, generation)| generation)); + (out, buffer.dropped()) + } - assert_eq!(buffer.dropped(), 0); + /// A track sent further behind than the new generation's acquisition lasts crosses the + /// restart after the new clock is set. Four seconds in, the leads are measured, so the new + /// clock keeps the lead the previous generation measured for it, its frames land there, + /// and the frames it sends before crossing stay on the previous generation: nothing is + /// dropped and no third generation opens. + #[tokio::test(start_paused = true)] + async fn a_track_crossing_after_the_new_clock_keeps_its_lead() { + let (out, dropped) = cross(40, 0).await; + assert_eq!(dropped, 0); + assert!(out.is_sorted(), "a generation went out after the next: {out:?}"); assert_eq!(out.iter().filter(|&&generation| generation == 0).count(), 83); assert_eq!(out.iter().filter(|&&generation| generation == 1).count(), 40); assert_eq!(out.len(), 123, "a third generation opened, or a frame never went out"); } + /// The same restart before any steering step, with the audio's old timeline running a frame + /// past the video's: no lead covers it, so that frame, read after the new generation's + /// first went out, is dropped rather than sent out of turn. + #[tokio::test(start_paused = true)] + async fn a_track_crossing_before_any_lead_keeps_the_generations_in_turn() { + let (out, _) = cross(10, 1).await; + assert!(out.is_sorted(), "a generation went out after the next: {out:?}"); + } + /// A skipped group whose timeline carried on keeps the clock: a frame it made late is /// dropped, and the next on-time frame goes out on the same generation. #[tokio::test(start_paused = true)] From 6dfd45fafa8849a6a54fde778df80d8ca1242f57 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 7 Oct 2026 16:57:03 -0700 Subject: [PATCH 39/40] fix(moq-mux): saturate the HRD rate and name the clocks --delay spans Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 6 +++--- rs/moq-mux/src/container/ts/export.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 725194f4839..66cc9fbe135 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -164,7 +164,7 @@ Each frame goes out as early as a receiver's buffers for its PID admit (the ISO 13818-1 T-STD: no more of a PID's packets per interval than its transport buffer passes on, and no more bytes than its decoder buffer holds, sized from the SPS's HRD or level for video and per codec for audio), up to `--delay` ahead of its -decode time, earliest decode time first. So a heavy passage rides the intervals +decode time on the output's PCR clock, earliest decode time first. So a heavy passage rides the intervals before it, and the output trails the source by twice the delay. A frame that cannot arrive by its decode time at the rate fails the export with an error naming both knobs; a broadcast-sized decoder buffer (a CPB of a second or more) needs a delay to @@ -496,8 +496,8 @@ groups fetchable, which the [HLS gateway](/bin/hls) depends on. `export --max-de behind the live edge before *this* consumer skips it. Raising the first never delays playback. `export ts` takes `--delay` (default 500 ms) instead, and works like an SRT -receiver's latency. Every frame is muxed that long after its decode time, on -a clock that keeps the source's pace and muxes all tracks in decode order +receiver's latency. Every frame is released to the multiplexer that long after +its decode time, on a clock that keeps the source's pace and muxes all tracks in decode order whatever their arrival skew: two exporters of one broadcast emit them in one order. The export joins at the newest group and holds its output until it has heard from every audio, video, DVB AC-3 and teletext track (two delays at most) and a diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 1913bdce473..de56a3080f1 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -395,7 +395,7 @@ impl Timing { fn buffer(&self) -> Option { let level = self.level?; let (rate, cpb) = match self.hrd { - Some(hrd) => (hrd.bit_rate * level.factor / 1_000, hrd.cpb_size), + Some(hrd) => (hrd.bit_rate.saturating_mul(level.factor) / 1_000, hrd.cpb_size), None => (level.leak, level.cpb), }; Some(Buffer { From 50935e6f37001b378a628709354c393aea724643 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 7 Oct 2026 21:52:46 -0700 Subject: [PATCH 40/40] fix(moq-mux): drop an old-timeline frame without steering on it A frame from a track still on an earlier generation, crossing no restart, that the previous clock refused fell through to the current clock: it was steered on (seeding a floor with the old decode time on a forward jump) and, after a backward restart, queued about one jump in the future. It now drops as late unless it lands on the current clock, which keeps unmarked crossings joining the new generation. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/jitter.rs | 52 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index aa24f6c5f5d..b00c7e5861a 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -315,6 +315,20 @@ impl Buffer { return Ok(self.queue(key, arrival, previous.generation, deadline, false)); } let deadline = self.deadline(&clock, arrival.decode); + // A track still on an earlier generation, crossing no restart, that the previous clock + // refused and the current one does not land is on the old timeline: it is late, and + // says nothing about the current clock. + let behind = self + .tracks + .get(&key) + .filter(|track| track.generation != clock.generation && track.restart == arrival.restart); + if let Some(track) = behind + && !self.delay.is_zero() + && !self.lands(deadline, arrival.arrived) + { + let generation = track.generation; + return Ok(self.queue(key, arrival, generation, None, false)); + } if self.released && !self.delay.is_zero() { self.steer(&key, deadline, &arrival)?; } @@ -1061,6 +1075,44 @@ mod tests { assert!(out.is_sorted(), "a generation went out after the next: {out:?}"); } + /// A restart from 10 s onto a timeline at `to`, then, once the new generation's first frame + /// went out, a frame of the old timeline from a track that crossed no restart: it lands on + /// neither clock, so it drops as late without steering, and the track's next frame on the + /// new timeline still joins it. + async fn stray(to: u64) { + let start = Instant::now(); + let mut buffer = Buffer::new(DELAY); + buffer.expect([1]); + buffer.push(1, arrival(start, 10_000, "v")).unwrap(); + buffer.push(2, arrival(start, 10_000, "d")).unwrap(); + tokio::time::advance(DELAY).await; + assert_eq!(due(&mut buffer), ["v", "d"]); + + let resume = start + Duration::from_secs(1); + tokio::time::advance(resume - Instant::now()).await; + buffer.push(1, after(1, arrival(resume, to, "v'"))).unwrap(); + tokio::time::advance(DELAY).await; + assert_eq!(released(&mut buffer), [("v'", 1)]); + + let now = Instant::now(); + assert_eq!(buffer.push(2, arrival(now, 10_040, "d")).unwrap(), Push::Late); + assert!(buffer.is_empty(), "the old frame was queued on the new clock"); + assert!(buffer.steer.floor.is_none(), "the old frame was steered on"); + assert_eq!(buffer.push(2, arrival(now, to + 40, "d'")).unwrap(), Push::Queued); + tokio::time::advance(DELAY).await; + assert_eq!(released(&mut buffer), [("d'", 1)]); + } + + #[tokio::test(start_paused = true)] + async fn an_old_frame_after_a_forward_restart_is_not_steered_on() { + stray(20_000).await; + } + + #[tokio::test(start_paused = true)] + async fn an_old_frame_after_a_backward_restart_is_dropped() { + stray(0).await; + } + /// A skipped group whose timeline carried on keeps the clock: a frame it made late is /// dropped, and the next on-time frame goes out on the same generation. #[tokio::test(start_paused = true)]