From 5374451785ccc7c27cd862ebd0864323a2b8f91c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 20:51:29 -0700 Subject: [PATCH 01/16] chore(quest): claim TS stream routing Co-Authored-By: Claude Opus 5.5 From 28f27e95f0bb6b41ba115df5ea030594e1d23227 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 21:14:30 -0700 Subject: [PATCH 02/16] fix(mux): give every TS PID a declared route Carry MPEG-1/2 video whole while it still paces the program, read the DSM-CC types as sections, count every stream the import does not carry in refused (0x86 without CUEI, base catalogs, TS-scrambled packets), and count packets on PIDs no table claims once every listed PMT is in. Programs owns that count for --program all. Export lists the NIT's network PID in the PAT whenever it carries the NIT. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/claims.rs | 231 ++++++++++++++ rs/moq-mux/src/container/ts/export.rs | 75 ++++- rs/moq-mux/src/container/ts/import.rs | 348 ++++++++++++++------- rs/moq-mux/src/container/ts/import_test.rs | 21 +- rs/moq-mux/src/container/ts/mod.rs | 1 + rs/moq-mux/src/container/ts/programs.rs | 127 +++++++- rs/moq-mux/src/container/ts/stats.rs | 49 ++- 7 files changed, 707 insertions(+), 145 deletions(-) create mode 100644 rs/moq-mux/src/container/ts/claims.rs diff --git a/rs/moq-mux/src/container/ts/claims.rs b/rs/moq-mux/src/container/ts/claims.rs new file mode 100644 index 00000000000..0e14520115e --- /dev/null +++ b/rs/moq-mux/src/container/ts/claims.rs @@ -0,0 +1,231 @@ +//! Which PIDs a multiplex's tables claim, and the packets on every other one. +//! +//! A PID is claimed by the PAT (PID 0 and each listed program's PMT PID), by a delivered PMT +//! (its elementary streams and its PCR PID), or by a fixed table the importer captures (the SI +//! PIDs). The null PID is stuffing and never counted. A packet on any other PID is dropped, but +//! only once every program the PAT lists has delivered its PMT: until then a PID may belong to +//! a program whose PMT is still on its way, as on a mid-stream join. A PMT that lapses past +//! TR 101 290's 0.5 s stops holding that back. + +use std::collections::{BTreeMap, HashMap, HashSet}; + +use mpeg2ts::ts::Pid; + +/// TS packets that carry nothing: stuffing, rebuilt on export. +const NULL_PID: u16 = 0x1fff; + +/// `PMT_error_2`: a PMT recurs at least every 0.5 s of program clock. +const PMT_LIMIT: u64 = super::passthrough::PCR_HZ / 2; + +/// The claims of every program a PAT lists, and the packets on the PIDs none of them claim. +pub(super) struct Claims { + /// PIDs claimed whatever the tables say beside the PAT's: the SI the importer captures. + fixed: &'static [u16], + /// Every PID a listed program claims, with how many claims name it. + claimed: HashMap, + /// Each program the PAT lists, by program number. Empty until a PAT lists one. + programs: HashMap, + /// Packets dropped on each PID, cumulative across its claims. + dropped: BTreeMap, + /// The PIDs dropped since a table last claimed them, whose row is the dropped one. + unclaimed: HashSet, +} + +/// One listed program's claims. +struct Program { + pmt_pid: u16, + /// The elementary stream and PCR PIDs of its last PMT, `None` until one arrives. + pids: Option>, + /// The program clock when the PAT listed it, `None` until the clock starts. + listed: Option, + /// Its PMT lapsed before arriving, so it no longer holds the count back. + lapsed: bool, +} + +impl Claims { + /// Claims for an importer that captures the SI on `si`, if any. + pub(super) fn new(si: &'static [u16]) -> Self { + Self { + fixed: si, + claimed: HashMap::new(), + programs: HashMap::new(), + dropped: BTreeMap::new(), + unclaimed: HashSet::new(), + } + } + + /// A whole PAT listing `programs` as `(program_number, pmt_pid)`, the network entry excluded, + /// at program clock `now`. A program it no longer lists releases its claims; a new one, or + /// one moved to another PMT PID, holds the count back until its PMT arrives or lapses. + pub(super) fn pat(&mut self, programs: &[(u16, Pid)], now: Option) { + let stale: Vec = self + .programs + .iter() + .filter(|(number, program)| { + !programs + .iter() + .any(|(listed, pid)| listed == *number && pid.as_u16() == program.pmt_pid) + }) + .map(|(&number, _)| number) + .collect(); + for number in stale { + let program = self.programs.remove(&number).expect("a listed program"); + self.release(program.pmt_pid); + for pid in program.pids.into_iter().flatten() { + self.release(pid); + } + } + for &(number, pmt_pid) in programs { + if self.programs.contains_key(&number) { + continue; + } + self.claim(pmt_pid.as_u16()); + self.programs.insert( + number, + Program { + pmt_pid: pmt_pid.as_u16(), + pids: None, + listed: now, + lapsed: false, + }, + ); + } + } + + /// The PMT of `program` lists `pids`, its elementary streams and PCR PID. Ignored for a + /// program the PAT does not list. + pub(super) fn pmt(&mut self, program: u16, pids: impl IntoIterator) { + let Some(entry) = self.programs.get_mut(&program) else { + return; + }; + let mut pids: Vec = pids.into_iter().filter(|&pid| pid != NULL_PID).collect(); + pids.sort_unstable(); + pids.dedup(); + let old = entry.pids.replace(pids.clone()).unwrap_or_default(); + for pid in pids { + self.claim(pid); + } + for pid in old { + self.release(pid); + } + } + + /// The program clock reached `now`: a PMT awaited for longer than 0.5 s lapses. + pub(super) fn tick(&mut self, now: u64) { + for program in self.programs.values_mut() { + if program.pids.is_some() || program.lapsed { + continue; + } + let listed = *program.listed.get_or_insert(now); + program.lapsed = now.saturating_sub(listed) > PMT_LIMIT; + } + } + + /// Count a packet on `pid` if no table claims it and every listed PMT is in. + pub(super) fn packet(&mut self, pid: u16) { + if pid == NULL_PID + || pid == Pid::PAT + || self.claimed.contains_key(&pid) + || self.fixed.contains(&pid) + || !self.acquired() + { + return; + } + *self.dropped.entry(pid).or_default() += 1; + if self.unclaimed.insert(pid) { + tracing::warn!( + pid, + "dropping a PID no PAT, PMT or captured SI claims; carry the multiplex whole to keep it" + ); + } + } + + /// Whether a PAT listed a program and every one it lists has delivered or lapsed its PMT. + fn acquired(&self) -> bool { + !self.programs.is_empty() + && self + .programs + .values() + .all(|program| program.pids.is_some() || program.lapsed) + } + + /// Each PID with dropped packets: how many, and whether it is dropped now rather than + /// claimed again since. + pub(super) fn rows(&self) -> impl Iterator + '_ { + self.dropped + .iter() + .map(|(&pid, &dropped)| (pid, dropped, self.unclaimed.contains(&pid))) + } + + fn claim(&mut self, pid: u16) { + *self.claimed.entry(pid).or_default() += 1; + self.unclaimed.remove(&pid); + } + + fn release(&mut self, pid: u16) { + if let std::collections::hash_map::Entry::Occupied(mut entry) = self.claimed.entry(pid) { + *entry.get_mut() -= 1; + if *entry.get() == 0 { + entry.remove(); + } + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + fn pid(pid: u16) -> Pid { + Pid::new(pid).unwrap() + } + + /// Nothing counts before every listed PMT is in, and a claimed, fixed or null PID never. + #[test] + fn counts_only_unclaimed_pids_once_acquired() { + let mut claims = Claims::new(&[0x10]); + claims.packet(0x300); + claims.pat(&[(1, pid(0x20)), (2, pid(0x30))], Some(0)); + claims.pmt(1, [0x100, 0x101]); + claims.packet(0x300); + assert_eq!(claims.rows().count(), 0, "program 2's PMT is still awaited"); + claims.pmt(2, [0x200]); + for pid in [0x0000, 0x0010, 0x0020, 0x0030, 0x0100, 0x0200, NULL_PID, 0x0300, 0x0300] { + claims.packet(pid); + } + assert_eq!(claims.rows().collect::>(), [(0x300, 2, true)]); + } + + /// A PMT that never arrives stops holding the count back 0.5 s after the PAT listed it. + #[test] + fn a_lapsed_pmt_releases_the_count() { + let mut claims = Claims::new(&[]); + claims.pat(&[(1, pid(0x20)), (2, pid(0x30))], Some(0)); + claims.pmt(1, [0x100]); + claims.tick(PMT_LIMIT); + claims.packet(0x300); + assert_eq!(claims.rows().count(), 0); + claims.tick(PMT_LIMIT + 1); + claims.packet(0x300); + assert_eq!(claims.rows().collect::>(), [(0x300, 1, true)]); + } + + /// A PID two programs claim stays claimed until both release it. + #[test] + fn a_shared_pid_is_released_by_its_last_claim() { + let mut claims = Claims::new(&[]); + claims.pat(&[(1, pid(0x20)), (2, pid(0x30))], Some(0)); + claims.pmt(1, [0x100, 0x1ff]); + claims.pmt(2, [0x200, 0x1ff]); + claims.pmt(1, [0x100]); + claims.packet(0x1ff); + assert_eq!(claims.rows().count(), 0); + claims.pat(&[(1, pid(0x20))], Some(0)); + claims.packet(0x1ff); + claims.packet(0x200); + assert_eq!(claims.rows().collect::>(), [(0x1ff, 1, true), (0x200, 1, true)]); + // A PMT claiming it again keeps its count but ends its dropped row. + claims.pmt(1, [0x100, 0x200]); + assert_eq!(claims.rows().collect::>(), [(0x1ff, 1, true), (0x200, 1, false)]); + } +} diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 5fdd86d6a1f..9528e3b47c8 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -774,7 +774,11 @@ impl Kind { /// alike at both edges. fn class(&self) -> super::stats::Class { match self { - Kind::Video(_) => super::stats::Class::Video, + // MPEG-1/2 video rides whole, but paces the program as decoded video does. + Kind::Video(_) + | Kind::PesPacket { + stream_type: 0x01 | 0x02, + } => super::stats::Class::Video, Kind::Aac { .. } | Kind::Opus { .. } | Kind::Mp2 { .. } | Kind::Ac3 | Kind::Eac3 => { super::stats::Class::Audio } @@ -791,6 +795,39 @@ struct Psi { /// PID the PMT rides on: the source's original (preserved from the service /// record) or the synthesized [`PMT_PID`] for a media-only source. pmt_pid: u16, + transport_stream_id: u16, + program_number: u16, + /// Whether the PAT lists the network entry, pointing at the NIT the export carries. + network: bool, +} + +/// The DVB NIT's PID (ETSI EN 300 468 5.1.3), the network PID the PAT's program 0 names. +const NIT_PID: u16 = 0x0010; +/// The `table_id` of the NIT describing the delivering network. +const NIT_ACTUAL: u8 = 0x40; + +/// The PAT for the exported program at `version`, listing the network PID as program 0 first when +/// the export carries the NIT (ISO/IEC 13818-1 2.4.4.3), which a receiver otherwise cannot find. +fn encode_pat( + transport_stream_id: u16, + program_number: u16, + pmt_pid: u16, + network: bool, + version: VersionNumber, +) -> anyhow::Result { + let network = network.then(|| ProgramAssociation { + program_num: 0, + program_map_pid: Pid::new(NIT_PID).expect("a valid PID"), + }); + let program = ProgramAssociation { + program_num: program_number, + program_map_pid: Pid::new(pmt_pid)?, + }; + super::psi::encode_pat(&Pat { + transport_stream_id, + version_number: version, + table: network.into_iter().chain([program]).collect(), + }) } /// Per-frame PES descriptor (everything but the payload bytes). @@ -1504,6 +1541,22 @@ where } } + // A NIT acquired or lost after the tables went out revises the PAT's network entry. + let network = self.carries_nit(); + if let Some(psi) = self.psi.as_mut() + && psi.network != network + { + self.version.increment(); + psi.pat = encode_pat( + psi.transport_stream_id, + psi.program_number, + psi.pmt_pid, + network, + self.version, + )?; + psi.network = network; + } + // The desired track set: media renditions plus the verbatim streams. let mut active: BTreeMap = BTreeMap::new(); for name in catalog.video.renditions.keys() { @@ -2059,14 +2112,8 @@ where .filter(|&id| id != 0) .unwrap_or(1); - let pat = Pat { - transport_stream_id, - version_number: self.version, - table: vec![ProgramAssociation { - program_num: program_number, - program_map_pid: Pid::new(pmt_pid)?, - }], - }; + let network = self.carries_nit(); + let pat = encode_pat(transport_stream_id, program_number, pmt_pid, network, self.version)?; let pmt = super::psi::Pmt { program_number, pcr_pid: Some(Pid::new(pcr_pid)?), @@ -2080,14 +2127,22 @@ where self.liveness.register(track.pid); } self.psi = Some(Psi { - pat: super::psi::encode_pat(&pat)?, + pat, pmt: super::psi::encode_pmt(&pmt, self.version)?, pcr_pid, pmt_pid, + transport_stream_id, + program_number, + network, }); Ok(()) } + /// Whether the catalog carries a NIT on its PID, which the PAT then points a receiver at. + fn carries_nit(&self) -> bool { + self.si.contains_key(&(NIT_PID, NIT_ACTUAL)) + } + /// Packetize one media frame and queue it on the [`Schedule`], re-emitting PAT/PMT /// before video keyframes (and periodically) so receivers can tune in mid-stream. fn mux(&mut self, name: &str, queued: Queued) -> anyhow::Result<()> { diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index f74b20ddc75..43f70c5c809 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -37,10 +37,11 @@ use moq_net::Timestamp; /// Demuxes an MPEG-TS byte stream into a MoQ broadcast. /// -/// Supports H.264 (stream type 0x1B), H.265 (0x24), ADTS AAC (0x0F), MP2 -/// (0x03/0x04), AC-3 (0x81), and E-AC-3 (0x87). LATM/LOAS AAC (0x11) is not -/// ADTS-framed and is dropped. Each codec stream is fed to its importer, which -/// manages the track, catalog config, and keyframe-based group boundaries. +/// Decodes H.264 (stream type 0x1B), H.265 (0x24), ADTS AAC (0x0F), MP2 +/// (0x03/0x04), AC-3 (0x81), and E-AC-3 (0x87). Each codec stream is fed to its importer, which +/// manages the track, catalog config, and keyframe-based group boundaries. MPEG-1/2 video +/// (0x01/0x02) is not decoded but still paces the program: its PTS anchors the timebase while +/// its PES ride whole as below. /// /// Partial header contents hold at most 264 bytes per PID. PES payloads and sections awaiting a /// timestamp share half a MoQ group's byte budget; held sections also share half its frame @@ -48,10 +49,19 @@ use moq_net::Timestamp; /// /// Elementary streams we don't decode are carried verbatim, one MoQ track per /// PID, when the catalog `E` carries the [`mpegts`](super::Mpegts) section: section-framed -/// streams (SCTE-35, marked by a program-level 'CUEI' registration descriptor, and other -/// private sections) are reassembled as sections, and any other stream travels as whole PES -/// packets ([`PesPacket`](super::PesPacket)). With a base `Catalog<()>` they're logged and -/// dropped instead. +/// streams (private sections 0x05, the ISO 13818-6 DSM-CC types 0x0A to 0x0D, and SCTE-35 0x86 +/// marked by a program-level 'CUEI' registration descriptor) are reassembled as sections, and +/// any other stream travels as whole PES packets ([`PesPacket`](super::PesPacket)), LATM/LOAS +/// AAC (0x11) included. A 0x86 stream without 'CUEI', which may be DTS audio or SCTE-35, is not +/// carried. With a base `Catalog<()>` no undecoded stream is carried. +/// +/// Every PID is accounted for. A listed stream that is not carried counts once in +/// [`stats::Stream::refused`] with a warning, and so does each TS-scrambled packet on a carried +/// one. Once every program the PAT lists has delivered its PMT, or let it lapse past TR 101 +/// 290's 0.5 s, a packet on a PID no table claims (the null PID aside) counts in its own row: +/// see [`stats::Snapshot::streams`]. That holds when the PAT lists only the imported program; a +/// [`with_program`](Self::with_program) import of a multiplex cannot tell another program's PIDs +/// from unclaimed ones, so it leaves them to [`Programs`](super::Programs). /// /// A whole PES or section is timestamped with its placement: the program clock at its first byte, /// interpolated between the PCRs either side of it (ISO 13818-1 2.4.2.2) and shifted onto the @@ -202,6 +212,11 @@ pub struct Import { /// The program number chosen by [`with_program`](Self::with_program). `None` imports the /// multiplex's only program and refuses a PAT that lists more than one. program: Option, + /// The PIDs no table claims, counted while the PAT lists only the imported program. `None` + /// under [`Programs`](super::Programs), which counts them once for the multiplex. + claims: Option, + /// Carried PIDs already warned about a TS-scrambled packet. + scrambled: HashSet, } impl Import { @@ -265,6 +280,12 @@ impl Import { last_pts: None, pts: PtsUnwrap::default(), program: None, + claims: Some(super::claims::Claims::new(if supports_mpegts { + catalog::SI_PIDS + } else { + &[] + })), + scrambled: HashSet::new(), } } @@ -289,6 +310,13 @@ impl Import { self } + /// Leave the PIDs no program claims to [`Programs`](super::Programs), which reads every + /// program's PMT and counts them once for the multiplex. + pub(super) fn in_multiplex(mut self) -> Self { + self.claims = None; + self + } + /// A reservation on this program's timebase, so every stream shifts by its offset. fn reserve(&self) -> crate::catalog::Reserved { self.timebase.reserve() @@ -321,6 +349,12 @@ impl Import { if self.supports_mpegts { self.mux_rate.packet(); } + // A packet flagged corrupt may carry another PID's number. + if pkt[1] & 0x80 == 0 + && let Some(claims) = &mut self.claims + { + claims.packet(pid); + } // Every packet is graded, but only the media PIDs and the clock route on the // verdict: PSI and section PIDs keep their reassemblers' own. let continuation = self.health.packet(&pkt, self.liveness.now()); @@ -387,6 +421,9 @@ impl Import { self.liveness.pcr(pcr); if let Some(now) = self.liveness.now() { self.health.tick(now); + if let Some(claims) = &mut self.claims { + claims.tick(now); + } } if self.supports_mpegts { let ready = self.mux_rate.ready(); @@ -402,6 +439,21 @@ impl Import { } } } + // TS-level scrambling hides a payload from all but a descrambler, so the semantic lane + // cannot carry it where the raw lane can. Its adaptation field stays clear: the clock + // above has read it. + if pkt[1] & 0x80 == 0 + && pkt[3] & 0xc0 != 0 + && let Ok(pid) = Pid::new(pid) + && (self.sections.contains_key(&pid.as_u16()) + || self + .streams + .get(&pid) + .is_some_and(|stream| !matches!(stream, Stream::Ignored))) + { + self.scrambled(pid)?; + continue; + } if let Some(section) = self.sections.get_mut(&pid) { let unplaced = section.packet(&pkt, offset, &self.placement)?; let (units, dropped) = section.release(&self.timebase, &self.pts, self.placement.generation)?; @@ -451,10 +503,10 @@ impl Import { } else if self.pmt_sections.contains_key(&pid) { self.pmt_packet(pid, &pkt)?; } else { - // PIDs we don't decode and don't carry (`Stream::Ignored`: a base catalog's - // undecoded streams, or an ambiguous 0x86 PID without CUEI) are dropped, as is - // every PID before the PSI that declares it: a live capture joins mid-stream, - // so PES arrive before their PMT. + // PIDs we don't carry (`Stream::Ignored`, counted when refused) are dropped, as is + // every PID no table claims (counted by `claims` above) and every PID before the + // PSI that declares it: a live capture joins mid-stream, so PES arrive before + // their PMT. match self.streams.get(&pid) { None | Some(Stream::Ignored) => {} Some(_) => self.pes_packet(pid, &pkt, offset)?, @@ -550,11 +602,14 @@ impl Import { if self.supports_mpegts { self.es_descriptors.insert(es.pid.as_u16(), es.descriptors.clone()); } - // Section-framed private data is reassembled as sections, never PES-parsed: - // private sections (0x05) and CUEI-marked SCTE-35 (0x86). Everything else - // routes through ensure_stream (a decoded codec, PES-framed verbatim, or - // dropped). - if es.stream_type == 0x05 || (cuei && es.stream_type == StreamType::Dts8ChannelLosslessAudio as u8) { + // Section-framed data is reassembled as sections, never PES-parsed: private sections + // (0x05), the ISO 13818-6 DSM-CC types, each of which rides DSM-CC sections (0x0A + // multiprotocol encapsulation, 0x0B U-N messages and carousels, 0x0C stream + // descriptors, 0x0D any DSM-CC section), and CUEI-marked SCTE-35 (0x86). Everything + // else routes through ensure_stream (a decoded codec, whole PES, or refused). + if matches!(es.stream_type, 0x05 | 0x0a..=0x0d) + || (cuei && es.stream_type == StreamType::Dts8ChannelLosslessAudio as u8) + { self.ensure_section(es.pid, es.stream_type, &es.descriptors)?; } else { self.ensure_stream(es.pid, es.stream_type, &es.descriptors)?; @@ -573,16 +628,9 @@ impl Import { } } self.media_clock = pmt.streams.iter().any(|es| { - matches!( - self.streams.get(&es.pid), - Some( - Stream::H264 { .. } - | Stream::H265 { .. } - | Stream::Aac(_) | Stream::Opus(_) - | Stream::Legacy(_) - | Stream::Clock - ) - ) + self.streams.get(&es.pid).is_some_and(|stream| { + stream.video() || matches!(stream, Stream::Aac(_) | Stream::Opus(_) | Stream::Legacy(_)) + }) }); if let Some(pid) = self.pcr_pid { let initial = self.pcr_candidates.remove(&pid); @@ -593,6 +641,16 @@ impl Import { } let pids: Vec = pmt.streams.iter().map(|es| es.pid.as_u16()).collect(); self.health.pmt_streams(&pids); + // The PIDs this import routes, carried or refused, which are the PMT's own once a PMT that + // drops a stream ends its route. + if let Some(claims) = &mut self.claims { + let routed = self + .streams + .keys() + .map(|pid| pid.as_u16()) + .chain(self.sections.keys().copied()); + claims.pmt(pmt.program_number, routed.chain(pmt.pcr_pid.map(|pid| pid.as_u16()))); + } Ok(()) } @@ -685,7 +743,7 @@ impl Import { self.unclocked.insert(pid, carried); return Ok(()); } - Some(Stream::Ignored) + Some(Stream::Ignored | Stream::Clock) if self .unclocked .get(&pid) @@ -718,7 +776,7 @@ impl Import { } } // Only ADTS-framed AAC (0x0F). 0x11 is LATM/LOAS, which uses a different - // framing and syncword, so it falls through to the ignored arm below. + // framing and syncword, so it rides whole like any other undecoded stream. Some(StreamType::AdtsAac) => Stream::Aac(Box::new(AacStream { import: None, asc: bytes::Bytes::new(), @@ -755,54 +813,35 @@ impl Import { import: opus::Import::new(track, self.reserve(), config)?, })) } - Err(err) => { - tracing::warn!( - pid = pid.as_u16(), - error = %err, - "unsupported Opus channel configuration, dropping" - ); - Stream::Ignored - } + Err(err) => self.uncarried( + pid, + stream_type, + &format!("unsupported Opus channel configuration: {err:#}"), + ), } } - Some(StreamType::Mpeg1Video | StreamType::Mpeg2Video) => Stream::Clock, - // A codec we don't decode, named or not. Carry it as whole PES packets when the - // catalog supports the `mpegts` section. 0x86 is excluded: it's ambiguous (DTS - // audio, or a non-conformant SCTE-35 mux without CUEI, which is sections a PES - // parse would abort on), so drop it rather than risk PES-parsing sections. - _ => { - if self.supports_mpegts && stream_type != StreamType::Dts8ChannelLosslessAudio as u8 { - if self.pcr_pid.is_none() { - // Nothing could place its PES, so no track is advertised. - tracing::warn!( - stream_type, - pid = pid.as_u16(), - "refused a private PES stream in a program with no PCR to place it by" - ); - *self.refused.entry(pid.as_u16()).or_default() += 1; - self.unclocked.insert(pid, stream_type); - Stream::Ignored - } else { - let descriptors = descriptors.to_vec(); - match PesPacketStream::new( - self.broadcast.clone(), - self.catalog.clone(), - pid.as_u16(), - stream_type, - descriptors, - ) { - Ok(stream) => Stream::PesPacket(Box::new(stream)), - Err(err) => { - tracing::warn!(?err, pid = pid.as_u16(), "failed to create a PES stream, dropping"); - Stream::Ignored - } - } - } - } else { - tracing::warn!(stream_type, pid = pid.as_u16(), "unsupported TS stream type, dropping"); - Stream::Ignored - } + // 0x86 is ambiguous: DTS audio, or a non-conformant SCTE-35 mux without CUEI, which is + // sections a PES parse would abort on. Refuse it rather than guess. + _ if stream_type == StreamType::Dts8ChannelLosslessAudio as u8 => self.uncarried( + pid, + stream_type, + "0x86 without a CUEI registration may be DTS or SCTE-35", + ), + // A codec we don't decode, named or not, MPEG-1/2 video included, rides as whole PES + // packets when the catalog supports the `mpegts` section. + _ if !self.supports_mpegts => self.uncarried(pid, stream_type, "the catalog has no mpegts section"), + _ if self.pcr_pid.is_none() => { + // Nothing could place its PES, so no track is advertised. + self.unclocked.insert(pid, stream_type); + self.uncarried(pid, stream_type, "the program has no PCR to place it by") } + _ => Stream::PesPacket(Box::new(PesPacketStream::new( + self.broadcast.clone(), + self.catalog.clone(), + pid.as_u16(), + stream_type, + descriptors.to_vec(), + )?)), }; // Clock is not a decodable track, so it doesn't initialize the importer. @@ -817,6 +856,17 @@ impl Import { Ok(()) } + /// The route of a listed stream this import does not carry, counted once as refused. MPEG-1/2 + /// video still paces the program by its PTS. + fn uncarried(&mut self, pid: Pid, stream_type: u8, why: &str) -> Stream { + *self.refused.entry(pid.as_u16()).or_default() += 1; + tracing::warn!(pid = pid.as_u16(), stream_type, why, "refused a listed stream"); + match mpeg_video(stream_type) { + true => Stream::Clock, + false => Stream::Ignored, + } + } + fn legacy_stream(&self, pid: Pid, descriptor: &'static legacy::Descriptor) -> Stream { Stream::Legacy(Box::new(LegacyStream { descriptor, @@ -852,15 +902,12 @@ impl Import { self.health.restart(pid.as_u16()); if !self.supports_mpegts { // Always route to Ignored, replacing any prior codec on this PID (a later PMT - // can reassign it), so a private section is never PES-parsed. Warn once. - let previous = self.retire_stream(pid); - self.streams.insert(pid, Stream::Ignored); - if !matches!(previous, Some(Stream::Ignored)) { - tracing::warn!( - pid = pid.as_u16(), - "private section stream detected without `mpegts` catalog support; dropping" - ); - } + // can reassign it), so a private section is never PES-parsed. Counted once. + let stream = match self.retire_stream(pid) { + Some(Stream::Ignored) => Stream::Ignored, + _ => self.uncarried(pid, stream_type, "the catalog has no mpegts section"), + }; + self.streams.insert(pid, stream); return Ok(()); } // A prior PMT may have routed this PID to Ignored; drop it so the PID has one route. @@ -924,11 +971,11 @@ impl Import { } let stream = &self.streams[&pid]; - let is_video = matches!(stream, Stream::H264 { .. } | Stream::H265 { .. } | Stream::Clock); + let is_video = stream.video(); let is_clock = matches!(stream, Stream::Clock); // Verbatim streams with no cadence (SCTE-35, DVB subtitles) are left ungraded // rather than watched against an invented one. - if pes.pts.is_some() && !matches!(stream, Stream::PesPacket(_) | Stream::Ignored) { + if pes.pts.is_some() && (is_video || !matches!(stream, Stream::PesPacket(_) | Stream::Ignored)) { self.health.pts(pid.as_u16(), self.liveness.now()); } // Resolve the wrap at PES start so every elementary stream and private section shares @@ -1054,10 +1101,7 @@ impl Import { return Ok(()); }; - let batched = self - .streams - .values() - .any(|stream| matches!(stream, Stream::H264 { .. } | Stream::H265 { .. } | Stream::Clock)); + let batched = self.streams.values().any(Stream::video); let Some(stream) = self.streams.get_mut(&pid) else { return Ok(()); }; @@ -1184,15 +1228,40 @@ impl Import { if let Some(section) = self.sections.remove(&pid.as_u16()) { section.abort(moq_net::Error::Unsupported); } - if let Some(stream) = self.retire_stream(pid) { - stream.abort(moq_net::Error::Unsupported); - } - self.streams.insert(pid, Stream::Ignored); + // MPEG-1/2 video still paces the program once its PES are refused. + let clock = match self.retire_stream(pid) { + Some(stream) => { + let video = stream.video(); + stream.abort(moq_net::Error::Unsupported); + video + } + None => false, + }; + self.streams + .insert(pid, if clock { Stream::Clock } else { Stream::Ignored }); *self.refused.entry(pid.as_u16()).or_default() += 1; tracing::warn!(pid = pid.as_u16(), why, "refused an undecoded stream"); Ok(()) } + /// A TS-scrambled packet on the carried `pid`: refused, and the unit it interrupts lost with + /// it. The route stays, so the stream resumes once its packets are clear again. + fn scrambled(&mut self, pid: Pid) -> anyhow::Result<()> { + self.pending.remove(&pid); + self.headers.remove(&pid); + if let Some(stream) = self.streams.get_mut(&pid) { + stream.lost()?; + } + *self.refused.entry(pid.as_u16()).or_default() += 1; + if self.scrambled.insert(pid.as_u16()) { + tracing::warn!( + pid = pid.as_u16(), + "refusing TS-scrambled packets; carry the multiplex whole to keep them" + ); + } + Ok(()) + } + /// The packet chain on `pid` was cut: salvage the truncated PES, and treat the bytes /// after it as [lost](Stream::lost). fn broken(&mut self, pid: Pid) -> anyhow::Result<()> { @@ -1370,6 +1439,15 @@ impl Import { }; self.pmt_sections.entry(entry.pmt_pid).or_default(); self.health.pmt_pids(&[entry.pmt_pid.as_u16()], self.liveness.now()); + if let Some(claims) = &mut self.claims { + // Alongside another program, a PID could be that program's: the count waits for a PAT + // listing this one alone. + let listed = match programs.as_slice() { + [_] => vec![(entry.program_number, entry.pmt_pid)], + _ => Vec::new(), + }; + claims.pat(&listed, self.liveness.now()); + } self.record_program_identity(pat.transport_stream_id, entry) } @@ -1493,6 +1571,9 @@ impl Import { .or_insert_with(|| stats::Stream::new(".ts", stats::Class::Data)) .refused = refused; } + if let Some(claims) = &self.claims { + stats::Stream::dropped(&mut streams, claims.rows()); + } for (pid, stats) in &mut streams { (stats.units, stats.quiet) = self.liveness.stream(*pid); } @@ -2583,6 +2664,12 @@ fn unit_error(err: crate::Error) -> anyhow::Error { } } +/// Whether `stream_type` is MPEG-1 or MPEG-2 video, which is carried whole but still paces the +/// program like decoded video. +fn mpeg_video(stream_type: u8) -> bool { + matches!(stream_type, 0x01 | 0x02) +} + /// One elementary stream's codec importer. enum Stream { H264 { @@ -2596,10 +2683,10 @@ enum Stream { Aac(Box>), Opus(Box), Legacy(Box>), - /// A codec we don't decode, carried as whole PES (DTS audio, private PES, ...). + /// A codec we don't decode, carried as whole PES (MPEG-1/2 video, LATM AAC, private PES, ...). PesPacket(Box>), - /// MPEG-1/2 video we don't decode, kept only to advance the media clock. - /// `is_video` counts it, so never reuse this variant for audio or data. + /// MPEG-1/2 video this import refused to carry, kept only to advance the media clock. + /// [`Self::video`] counts it, so never reuse this variant for audio or data. Clock, Ignored, } @@ -2673,6 +2760,15 @@ impl Stream { } } + /// Whether this is the program's video, decoded or not, whose pictures pace the program. + fn video(&self) -> bool { + match self { + Stream::H264 { .. } | Stream::H265 { .. } | Stream::Clock => true, + Stream::PesPacket(stream) => mpeg_video(stream.stream_type), + _ => false, + } + } + /// Whether nothing waits to be placed on the program clock. fn placed(&self) -> bool { match self { @@ -2804,7 +2900,10 @@ impl Stream { Stream::H264 { .. } => stats::Stream::new(".avc3", stats::Class::Video), Stream::H265 { .. } => stats::Stream::new(".hev1", stats::Class::Video), Stream::Opus(_) => stats::Stream::new(".opus", stats::Class::Audio), - Stream::PesPacket(_) => stats::Stream::new(".ts", stats::Class::Data), + Stream::PesPacket(stream) => match mpeg_video(stream.stream_type) { + true => stats::Stream::new(".ts", stats::Class::Video), + false => stats::Stream::new(".ts", stats::Class::Data), + }, Stream::Clock => stats::Stream::new("", stats::Class::Video), Stream::Ignored => return None, }) @@ -4209,7 +4308,10 @@ pub(super) mod test { .unwrap(); assert!(matches!(import.streams.get(&pid), Some(super::Stream::Ignored))); - assert_eq!(import.stats(), before, "a PMT remap must not reset lifetime counters"); + // A base catalog cannot carry the section route, which counts as refused. + let mut expected = before; + expected.streams.get_mut(&pid.as_u16()).unwrap().refused = 1; + assert_eq!(import.stats(), expected, "a PMT remap must not reset lifetime counters"); } /// Build a payload-only TS packet (PID 0x0021, afc 0b01). `body` is the bytes @@ -4486,7 +4588,7 @@ pub(super) mod test { import.finish().unwrap(); assert_eq!( - catalog.snapshot().ext.mpegts.tracks.len(), + cue_tracks(&catalog.snapshot().ext.mpegts), 1, "expected one scte35 rendition" ); @@ -4547,7 +4649,7 @@ pub(super) mod test { first.video.renditions.is_empty() && first.audio.renditions.is_empty(), "only the extension lists a track" ); - assert_eq!(first.ext.mpegts.tracks.len(), 1, "the cue track is listed"); + assert_eq!(cue_tracks(&first.ext.mpegts), 1, "the cue track is listed"); } // The base catalog (`Catalog<()>`) can't carry cues, so a detected CUEI PID routes to @@ -4643,14 +4745,14 @@ pub(super) mod test { "upgrade drops the stale Ignored route" ); assert_eq!( - catalog.snapshot().ext.mpegts.tracks.len(), + cue_tracks(&catalog.snapshot().ext.mpegts), 1, "upgrade advertises the cue track" ); // The importer clears its verbatim entries from the catalog when it drops, so read the // track name while it is still registered. - let name = catalog.snapshot().ext.mpegts.tracks.keys().next().unwrap().clone(); + let name = track_on(&catalog.snapshot().ext.mpegts, SECTION_PID); import.finish().unwrap(); let track = consumer .track(&name) @@ -4707,7 +4809,7 @@ pub(super) mod test { let stats = import.stats(); assert_eq!(stats.streams[&VIDEO].class, super::stats::Class::Video); - assert_eq!(stats.streams[&VIDEO].track, ""); + assert_eq!(stats.streams[&VIDEO].track, ".ts"); assert!( stats.streams[&VIDEO].units >= 1, "the video PID delivered before it stalled" @@ -4732,6 +4834,17 @@ pub(super) mod test { }); } + /// The section tracks the `mpegts` section lists, beside any whole-PES video clocking them. + fn cue_tracks(mpegts: &super::catalog::Mpegts) -> usize { + mpegts.tracks.values().filter(|track| track.verbatim.is_some()).count() + } + + /// The track the `mpegts` section carries `pid` on. + fn track_on(mpegts: &super::catalog::Mpegts, pid: u16) -> String { + let found = mpegts.tracks.iter().find(|(_, track)| track.pid == pid); + found.expect("a track on the PID").0.clone() + } + /// A PUSI TS packet on `pid` carrying a minimal PES with `pts` (90 kHz) and a /// 1-byte dummy payload, for streams we observe only for their PTS. fn pes_packet(pid: u16, pts: u64) -> Vec { @@ -6594,7 +6707,7 @@ pub(super) mod test { let expected = 1_000_000 + (at - (2 * 188 + 10)) * 100_000 / (3 * 188); import.finish().unwrap(); - let name = catalog.snapshot().ext.mpegts.tracks.keys().next().unwrap().clone(); + let name = track_on(&catalog.snapshot().ext.mpegts, 0x21); let track = consumer .track(&name) .unwrap() @@ -6649,7 +6762,7 @@ pub(super) mod test { let last = data.len() as u64 + 10; data.extend(pcr_packet(PCR_PID, 31_050_000)); let (consumer, catalog, _import) = import_all(&data).unwrap(); - let name = catalog.snapshot().ext.mpegts.tracks.keys().next().unwrap().clone(); + let name = track_on(&catalog.snapshot().ext.mpegts, 0x21); let frames = read_track(&consumer, &name, crate::container::Kind::Data).await; let times: Vec<_> = frames.iter().map(|frame| frame.timestamp.as_micros() as u64).collect(); let clock = |at: u64, before: u64, ticks: u64, after: u64| { @@ -6684,7 +6797,7 @@ pub(super) mod test { data.extend(packet(true, 1, 0, &CUE)); let (consumer, catalog, import) = import_all(&data).unwrap(); assert_eq!(import.stats().streams[&0x21].unplaced, 2); - let name = catalog.snapshot().ext.mpegts.tracks.keys().next().unwrap().clone(); + let name = track_on(&catalog.snapshot().ext.mpegts, 0x21); assert!( read_track(&consumer, &name, crate::container::Kind::Data) .await @@ -6731,7 +6844,13 @@ pub(super) mod test { "video kept importing past the dropped section PID" ); assert!( - catalog.snapshot().ext.mpegts.tracks.is_empty(), + !catalog + .snapshot() + .ext + .mpegts + .tracks + .values() + .any(|track| track.pid == SECTION_PID), "a 0x86 PID without CUEI must not be cataloged" ); } @@ -7299,7 +7418,7 @@ pub(super) mod test { import.decode(pes_packet(PCR_PID, 90_000).as_slice()).unwrap(); import.decode(pcr_packet(PCR_PID, 27_000_000).as_slice()).unwrap(); import.finish().unwrap(); - let name = catalog.snapshot().ext.mpegts.tracks.keys().next().unwrap().clone(); + let name = track_on(&catalog.snapshot().ext.mpegts, 0x21); let (frames, breaks) = read_breaks(&consumer, &name, crate::container::Kind::Data).await; assert_eq!(frames.len(), 3); assert_eq!( @@ -7880,10 +7999,7 @@ pub(super) mod test { let mut import = super::Import::new(broadcast, catalog.reserve()); import .decode(&synth_pmt( - &[ - (StreamType::Mpeg2Video, 0x50), - (StreamType::Dts8ChannelLosslessAudio, 0x21), - ], + &[(StreamType::H264, 0x50), (StreamType::Dts8ChannelLosslessAudio, 0x21)], true, )) .unwrap(); @@ -8972,7 +9088,7 @@ pub(super) mod test { let mut updates = catalog.consume().unwrap(); let mut import = super::Import::new(broadcast, catalog.reserve()); import - .decode(&synth_pmt(&[(StreamType::Mpeg2Video, PCR_PID)], false)) + .decode(&synth_pmt(&[(StreamType::H264, PCR_PID)], false)) .unwrap(); let waiter = kio::Waiter::noop(); // Each epoch carries 0.96 s of media, then a flagged PCR restarts the clock. @@ -9598,7 +9714,7 @@ pub(super) mod test { (257, 0x06, "verbatim", "pesPacket"), (258, 0x05, "verbatim", "section"), (259, 0x86, "dropped", "none"), - (260, 0x02, "dropped", "clock only"), + (260, 0x02, "verbatim", "pesPacket"), (261, 0x0f, "track", "decoded"), ] { if !streams.iter().any(|line| { @@ -9716,7 +9832,7 @@ pub(super) mod test { assert!(catalog.snapshot().ext.mpegts.tracks.is_empty()); assert!(matches!(import.streams[&pid], super::Stream::Ignored)); import.handle_pmt(pmt(Some(Pid::new(PCR_PID).unwrap()))).unwrap(); - let name = catalog.snapshot().ext.mpegts.tracks.keys().next().unwrap().clone(); + let name = track_on(&catalog.snapshot().ext.mpegts, 0x21); let data = [ pcr_packet(PCR_PID, 27_000_000), pes_packet(PCR_PID, 99_000), @@ -9745,7 +9861,7 @@ pub(super) mod test { ); assert!(catalog.snapshot().ext.mpegts.tracks.is_empty()); import.handle_pmt(pmt(Some(Pid::new(PCR_PID).unwrap()))).unwrap(); - let next = catalog.snapshot().ext.mpegts.tracks.keys().next().unwrap().clone(); + let next = track_on(&catalog.snapshot().ext.mpegts, 0x21); assert_ne!(next, name, "restoring a stream publishes a new track"); import.finish().unwrap(); } diff --git a/rs/moq-mux/src/container/ts/import_test.rs b/rs/moq-mux/src/container/ts/import_test.rs index 7c67921646b..7636f0e21e2 100644 --- a/rs/moq-mux/src/container/ts/import_test.rs +++ b/rs/moq-mux/src/container/ts/import_test.rs @@ -959,14 +959,24 @@ fn import_reports_a_silent_scte35_pid() { assert_stalls(data, 0x14d, ".ts", 0x100, 2.5, 3.5); } -/// Every elementary stream the importer carries reports a row, decoded, clock-only, and -/// verbatim alike, and every row has delivered. +/// Every PID reports a row: each elementary stream the importer carries, decoded, whole PES +/// (MPEG-2 video among them) and sections alike, has delivered, and each PID no table claims +/// (an ATSC capture's CAT, PSIP base PID and its private PIDs) counts what it dropped. #[test] fn import_reports_every_elementary_stream() { for (data, rows) in [ ( &include_bytes!("test_data/kyrion_mpeg2av_ac3.ts")[..], - &[(0x100, ""), (0x101, ".ac3"), (0x102, ".mp2"), (0x14d, ".ts")][..], + &[ + (0x0001, ""), + (0x100, ".ts"), + (0x101, ".ac3"), + (0x102, ".mp2"), + (0x14d, ".ts"), + (0x1fd0, ""), + (0x1fd1, ""), + (0x1ffb, ""), + ][..], ), ( &include_bytes!("test_data/scte35/bbb5s.ts")[..], @@ -977,6 +987,11 @@ fn import_reports_every_elementary_stream() { let tracks: Vec<(u16, &str)> = stats.streams.iter().map(|(&pid, s)| (pid, s.track.as_str())).collect(); assert_eq!(tracks, rows); for (pid, stream) in &stats.streams { + if stream.track.is_empty() { + assert_eq!(stream.class, super::stats::Class::Data, "{pid:#x}: {stream:?}"); + assert!(stream.refused > 0 && stream.units == 0, "{pid:#x}: {stream:?}"); + continue; + } assert!(stream.units > 0, "{pid:#x} delivered nothing: {stream:?}"); assert!(stream.quiet.is_some(), "{pid:#x} has no program clock: {stream:?}"); } diff --git a/rs/moq-mux/src/container/ts/mod.rs b/rs/moq-mux/src/container/ts/mod.rs index 78e1df6ae0c..560dea92c02 100644 --- a/rs/moq-mux/src/container/ts/mod.rs +++ b/rs/moq-mux/src/container/ts/mod.rs @@ -24,6 +24,7 @@ //! [`Release`] writes that track back out on a fixed delay. mod adts; +mod claims; mod damage; mod export; mod follower; diff --git a/rs/moq-mux/src/container/ts/programs.rs b/rs/moq-mux/src/container/ts/programs.rs index e48dd5780ce..9abf0f47a6f 100644 --- a/rs/moq-mux/src/container/ts/programs.rs +++ b/rs/moq-mux/src/container/ts/programs.rs @@ -1,11 +1,12 @@ //! Every program of a multiplex, each imported as its own broadcast. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use anyhow::Context; use mpeg2ts::ts::{Pid, TsPacket}; -use super::import::{Framer, PatReader}; +use super::claims::Claims; +use super::import::{Framer, Liveness, PatReader, SectionReassembler}; use super::{Ext, Import, psi, stats}; use crate::catalog; @@ -26,6 +27,8 @@ pub struct Programs { scan: PatScan, /// Empty until the PAT arrives. programs: Vec, + /// Every program's PAT and PMT, read once for the PIDs none of them claims. + tables: Tables, } /// One program's broadcast. @@ -71,6 +74,7 @@ impl Programs { pending: Vec::new(), scan: PatScan::default(), programs: Vec::new(), + tables: Tables::new(), } } @@ -96,7 +100,9 @@ impl Programs { .clone() .with_catalog(catalog::hang::Catalog::::default()); let catalog = catalog::Producer::new(&mut broadcast, config)?; - let mut import = Import::new(broadcast.clone(), catalog.reserve()).with_program(program); + let mut import = Import::new(broadcast.clone(), catalog.reserve()) + .with_program(program) + .in_multiplex(); // Seeded with the scanner's PAT rather than replaying its packets, so every section // reaches the importer and nothing the scanner counted is counted again. import.handle_pat(&pat)?; @@ -112,6 +118,7 @@ impl Programs { pending: Some((route, listed)), }); } + self.tables.pat(&pat); let pending = std::mem::take(&mut self.pending); self.feed(&pending) } @@ -121,6 +128,7 @@ impl Programs { program.import.decode(data)?; program.announce_if_listed()?; } + self.tables.decode(data); Ok(()) } @@ -128,7 +136,9 @@ impl Programs { /// /// Every importer reads the same PAT, and programs sharing a PMT PID read the same PMT /// sections, so a dropped section counts once however many read it. Every importer also - /// grades every packet of the multiplex, so each TR 101 290 error counts once too. + /// grades every packet of the multiplex, so each TR 101 290 error counts once too. A PID no + /// program's PMT and no captured SI claims is counted here, once for the multiplex, in the + /// row of the program that carries it whenever one does. pub fn stats(&self) -> stats::Snapshot { let mut stats = stats::Snapshot::default(); let mut crc_errors = BTreeMap::::new(); @@ -141,6 +151,7 @@ impl Programs { } errors.max(program.import.errors()); } + stats::Stream::dropped(&mut stats.streams, self.tables.claims.rows()); // The importers only see the PATs after the one that started them. *crc_errors.entry(Pid::PAT).or_default() += self.scan.crc_error; stats.crc_error = crc_errors.values().sum(); @@ -187,6 +198,114 @@ fn program_broadcast(name: &str, program: u16) -> String { } } +/// The multiplex's own reading of the PAT and every PMT it lists, so a PID is counted dropped only +/// when no program claims it. Each importer reads only its own program's PMT. +struct Tables { + framer: Framer, + /// A trailing partial packet. + scratch: Vec, + pat: PatReader, + /// A reassembler for each PMT PID the PAT lists. + pmts: HashMap, + /// The program and PCR PID whose clock times a PMT's lapse: the first program to deliver one. + clock: Option<(u16, Pid)>, + liveness: Liveness, + claims: Claims, +} + +impl Tables { + fn new() -> Self { + Self { + framer: Framer::default(), + scratch: Vec::new(), + pat: PatReader::default(), + pmts: HashMap::new(), + clock: None, + liveness: Liveness::default(), + // Every program's catalog carries the `mpegts` section, so the SI is captured. + claims: Claims::new(super::catalog::SI_PIDS), + } + } + + /// A whole PAT: the programs it lists, and their PMT PIDs to read. + fn pat(&mut self, pat: &psi::Pat) { + let listed: Vec<_> = pat + .programs + .iter() + .filter(|entry| entry.program_number != 0) + .map(|entry| (entry.program_number, entry.pmt_pid)) + .collect(); + self.pmts.retain(|pid, _| listed.iter().any(|(_, pmt)| pmt == pid)); + for (_, pid) in &listed { + self.pmts.entry(*pid).or_default(); + } + self.claims.pat(&listed, self.liveness.now()); + } + + /// Read the tables and the clock out of `data`, counting each packet no table claims. + fn decode(&mut self, data: &[u8]) { + self.scratch.extend_from_slice(data); + let mut off = 0; + while let Some(at) = self.framer.next(&self.scratch, &mut off) { + let pkt: [u8; TsPacket::SIZE] = self.scratch[at..at + TsPacket::SIZE].try_into().unwrap(); + // A packet flagged corrupt may carry another PID's number, and its tables and clock + // are as untrustworthy. + if pkt[1] & 0x80 != 0 { + continue; + } + let pid = (u16::from(pkt[1] & 0x1f) << 8) | u16::from(pkt[2]); + self.claims.packet(pid); + let Ok(pid) = Pid::new(pid) else { + continue; + }; + if pid.as_u16() == Pid::PAT { + // The importers count a bad CRC. + if let Some(pat) = self.pat.push(&pkt, &mut 0) { + self.pat(&pat); + } + } else if let Some(reassembler) = self.pmts.get_mut(&pid) { + let mut sections = Vec::new(); + reassembler.push(&pkt, &mut sections); + for section in sections { + if section.first() != Some(&psi::PMT_TABLE_ID) || !psi::crc_ok(§ion) { + continue; + } + if let Some(pmt) = psi::Pmt::parse(§ion) { + self.pmt(&pmt); + } + } + } + if self.clock.is_some_and(|(_, clock)| clock == pid) && super::import::adaptation_valid(&pkt) { + if super::import::discontinuity_indicator(&pkt) { + self.liveness.discontinuity(); + } + if let Some(pcr) = super::import::pcr(&pkt) { + self.liveness.pcr(pcr); + if let Some(now) = self.liveness.now() { + self.claims.tick(now); + } + } + } + } + self.scratch.drain(..off); + } + + /// A PMT's claims, and the clock it names if it is the first to name one. + fn pmt(&mut self, pmt: &psi::Pmt) { + let pids = pmt.streams.iter().map(|es| es.pid.as_u16()); + self.claims + .pmt(pmt.program_number, pids.chain(pmt.pcr_pid.map(|pid| pid.as_u16()))); + match (self.clock, pmt.pcr_pid) { + (None, Some(pcr)) => self.clock = Some((pmt.program_number, pcr)), + (Some((program, old)), pcr) if program == pmt.program_number && pcr != Some(old) => { + self.clock = pcr.map(|pcr| (program, pcr)); + self.liveness.discontinuity(); + } + _ => {} + } + } +} + /// Finds the first whole PAT listing a program, before any importer exists to read one. #[derive(Default)] struct PatScan { diff --git a/rs/moq-mux/src/container/ts/stats.rs b/rs/moq-mux/src/container/ts/stats.rs index a760599e5e4..b39d39d1ba6 100644 --- a/rs/moq-mux/src/container/ts/stats.rs +++ b/rs/moq-mux/src/container/ts/stats.rs @@ -50,8 +50,14 @@ pub struct Report { #[serde(default, rename_all = "camelCase")] #[non_exhaustive] pub struct Snapshot { - /// One row per elementary stream PID the importer carries, ordered by PID. A PID the - /// importer drops (an undecoded stream without the `mpegts` catalog section) has none. + /// One row per PID, ordered by PID: each elementary stream the PMT lists, carried or + /// refused, and each PID no table claims that the importer dropped packets on. A dropped + /// PID's row names no track, is [`Class::Data`], and counts its packets in + /// [`Stream::refused`]. A PID keeps one row for its life: a PMT that claims it, releases it, + /// or claims it again moves the row with its counters, and while dropped it names no track. + /// Packets before every listed PMT has arrived, or lapsed past 0.5 s, are acquisition and + /// not counted; so are null packets. An import of one program from a multiplex that lists + /// others counts no PID no table claims, since another program's PMT may. // Keys read from strings: a flattened section loses JSON's integer-key parsing. #[serde_as(as = "BTreeMap")] #[serde(skip_serializing_if = "BTreeMap::is_empty")] @@ -123,8 +129,8 @@ impl Snapshot { #[non_exhaustive] pub struct Stream { /// The current or most recent MoQ track suffix for this PID (`.avc3`, `.mp2`, `.ts`, ...), - /// or empty for MPEG-1/2 video, which is read for its clock and not published. At export, - /// the suffix an import of the output would give the PID. + /// or empty for a PID that publishes no track: one dropped, or MPEG-1/2 video refused and + /// read only for its clock. At export, the suffix an import of the output would give the PID. #[serde(skip_serializing_if = "String::is_empty")] pub track: String, /// How this PID was classified. See [`Class`]. @@ -166,11 +172,15 @@ pub struct Stream { /// still behind a wrap the import's first PTS already crossed. Export stays zero. #[serde(skip_serializing_if = "is_zero")] pub unplaced: u64, - /// Times this PID's stream was refused rather than carried altered: an undecoded stream - /// in a program with no PCR to place it by, or a private PES stream with a clock field only - /// the PTS and DTS of are rebased (ESCR, a `pack_header_field`, a TREF), PES-level scrambling, - /// an unbounded non-video PES, or a header export refuses. Its track ends and leaves the - /// catalog. Export stays zero. + /// What this PID lost to a route the import does not carry. Once for each time its stream + /// was refused: a listed stream the import cannot carry (an ambiguous 0x86 without a CUEI + /// registration, an Opus configuration it cannot map, any undecoded stream without the + /// `mpegts` catalog section or in a program with no PCR to place it by), or a whole-PES + /// stream with a clock field only the PTS and DTS of are rebased (ESCR, a + /// `pack_header_field`, a TREF), PES-level scrambling, an unbounded non-video PES, or a + /// header export refuses, whose track ends and leaves the catalog. Once for each + /// TS-scrambled packet on a carried stream, which the raw lane could carry. And once for + /// each packet on a PID no table claims ([`Snapshot::streams`]). Export stays zero. #[serde(skip_serializing_if = "is_zero")] pub refused: u64, /// This PID's share of [`Snapshot::continuity_count_error`]. @@ -193,6 +203,21 @@ impl Stream { } } + /// Fold the packets dropped on each PID no table claims into `streams`, as + /// [`Claims::rows`](super::claims::Claims::rows) lists them: counted in `refused`, and while + /// the PID is dropped its row names no track. A PID a table claims again keeps its count + /// under the row of the route that carries it now. + pub(super) fn dropped(streams: &mut BTreeMap, rows: impl Iterator) { + for (pid, dropped, now) in rows { + let row = streams.entry(pid).or_insert_with(|| Self::new("", Class::Data)); + row.refused += dropped; + if now { + row.track.clear(); + row.class = Class::Data; + } + } + } + /// Add a newer route's frame-sync counters while naming the route that is active now. /// Delivery and damage are kept per PID rather than per route, so they need no merging. pub(super) fn merge(&mut self, current: &Self) { @@ -214,10 +239,10 @@ impl Stream { pub enum Class { /// Continuous audio. Audio = 1, - /// Continuous video, including MPEG-1/2 video read only for its clock. + /// Continuous video, including MPEG-1/2 video carried whole or read only for its clock. Video = 2, - /// Undecoded or sparse data (SCTE-35, ID3, private sections, private PES), or a PCR PID - /// that carries no elementary stream. A quiet second is not a stall. A class a later + /// Undecoded or sparse data (SCTE-35, ID3, private sections, private PES), a PCR PID that + /// carries no elementary stream, or a dropped PID. A quiet second is not a stall. A class a later /// version adds reads as data. It must stay last for `#[serde(other)]`; the explicit /// discriminants keep every variant's `as` value. #[default] From a4384f0c80fdf6a3876f4c7666404f90930e9ec7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 21:23:51 -0700 Subject: [PATCH 03/16] test(mux): cover every TS PID route A refused stream with no track now reports an empty track, and a retired section keeps its row. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export_test.rs | 18 +- rs/moq-mux/src/container/ts/import.rs | 17 +- rs/moq-mux/src/container/ts/mod.rs | 2 + rs/moq-mux/src/container/ts/routes_test.rs | 362 +++++++++++++++++++++ 4 files changed, 390 insertions(+), 9 deletions(-) create mode 100644 rs/moq-mux/src/container/ts/routes_test.rs diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 7f0c2bc6c68..908b55936e9 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -102,7 +102,7 @@ async fn drain(consumer: moq_net::broadcast::Consumer) -> BytesMut { } /// `drain` for an exporter built with an explicit catalog extension. -async fn drain_with(exporter: Export) -> BytesMut +pub(super) async fn drain_with(exporter: Export) -> BytesMut where S::Ext: tscat::Catalog, { @@ -3130,15 +3130,19 @@ async fn service_layer_survives_roundtrip() { .await; assert_packet_aligned(&ts); - // The rebuilt PAT preserves the transport/service identity and PMT PID. + // The rebuilt PAT preserves the transport/service identity and PMT PID, and points at the + // carried NIT through the network entry. let mut reader = TsPacketReader::new(Cursor::new(ts.as_ref())); let mut checked_pat = false; while let Some(packet) = reader.read_ts_packet().unwrap() { if let Some(TsPayload::Pat(pat)) = packet.payload { assert_eq!(pat.transport_stream_id, 1, "TSID preserved in the rebuilt PAT"); - assert_eq!(pat.table.len(), 1); - assert_eq!(pat.table[0].program_num, 1, "service number preserved"); - assert_eq!(pat.table[0].program_map_pid.as_u16(), 0x1000, "PMT PID preserved"); + let table: Vec<_> = pat + .table + .iter() + .map(|entry| (entry.program_num, entry.program_map_pid.as_u16())) + .collect(); + assert_eq!(table, [(0, 0x0010), (1, 0x1000)], "network entry, then the service"); checked_pat = true; break; } @@ -3156,6 +3160,10 @@ async fn service_layer_survives_roundtrip() { let mut import2 = crate::container::ts::Import::new(broadcast2, catalog2.reserve()); import2.decode(&BytesMut::from(ts.as_ref())).unwrap(); import2.finish().unwrap(); + assert!( + !import2.stats().streams.contains_key(&0x0010), + "the network entry names captured SI, not a PID to count" + ); let snapshot2 = catalog2.snapshot(); let program2 = snapshot2 diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 43f70c5c809..5fe73f30aed 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -729,7 +729,7 @@ impl Import { // or it would keep intercepting the PID and the new stream would never get data. // This only fires on a genuine remap: section PIDs otherwise route to // `ensure_section`, never here. - if let Some(mut section) = self.sections.remove(&pid.as_u16()) { + if let Some(mut section) = self.retire_section(pid) { section.finish()?; self.pending.remove(&pid); self.headers.remove(&pid); @@ -945,6 +945,15 @@ impl Import { Some(stream) } + /// Remove the section route on `pid`, keeping the row of the track it published. + fn retire_section(&mut self, pid: Pid) -> Option> { + let section = self.sections.remove(&pid.as_u16())?; + self.retired_stats + .entry(pid.as_u16()) + .or_insert_with(|| stats::Stream::new(".ts", stats::Class::Data)); + Some(section) + } + /// Open the PES whose header `pes` begins where `span` says, if any PCR came before it. fn handle_pes_start(&mut self, pid: Pid, pes: PesStart, span: Option) -> anyhow::Result<()> { // Finish the preceding PES before accepting another header. @@ -1225,7 +1234,7 @@ impl Import { fn refuse(&mut self, pid: Pid, why: &str) -> anyhow::Result<()> { self.pending.remove(&pid); self.headers.remove(&pid); - if let Some(section) = self.sections.remove(&pid.as_u16()) { + if let Some(section) = self.retire_section(pid) { section.abort(moq_net::Error::Unsupported); } // MPEG-1/2 video still paces the program once its PES are refused. @@ -1562,13 +1571,13 @@ impl Import { for (&pid, &unplaced) in &self.unplaced { streams .entry(pid) - .or_insert_with(|| stats::Stream::new(".ts", stats::Class::Data)) + .or_insert_with(|| stats::Stream::new("", stats::Class::Data)) .unplaced = unplaced; } for (&pid, &refused) in &self.refused { streams .entry(pid) - .or_insert_with(|| stats::Stream::new(".ts", stats::Class::Data)) + .or_insert_with(|| stats::Stream::new("", stats::Class::Data)) .refused = refused; } if let Some(claims) = &self.claims { diff --git a/rs/moq-mux/src/container/ts/mod.rs b/rs/moq-mux/src/container/ts/mod.rs index 560dea92c02..0597d6bd3d9 100644 --- a/rs/moq-mux/src/container/ts/mod.rs +++ b/rs/moq-mux/src/container/ts/mod.rs @@ -71,3 +71,5 @@ mod pes_packet_test; mod recovery_test; #[cfg(test)] mod release_test; +#[cfg(test)] +mod routes_test; diff --git a/rs/moq-mux/src/container/ts/routes_test.rs b/rs/moq-mux/src/container/ts/routes_test.rs new file mode 100644 index 00000000000..b37d64c85da --- /dev/null +++ b/rs/moq-mux/src/container/ts/routes_test.rs @@ -0,0 +1,362 @@ +//! Every PID of an imported program takes one declared route: decoded, whole PES, section, +//! captured PSI/SI, or not carried and counted. + +use std::time::Duration; + +use super::catalog::Ext; +use super::fixture::{Flags, MS, Mux}; +use super::stats::{Class, Stream}; +use super::{Import, Programs, psi}; + +const PMT: u16 = 0x20; +const CLOCK: u16 = 0x1ff; +const VIDEO: u16 = 0x100; +const DATA: u16 = 0x101; +const UNLISTED: u16 = 0x300; +/// The PCR leads each picture's PTS by this many 90 kHz ticks. +const LEAD: u64 = 9_000; +/// `registration_descriptor` naming SMPTE RP 217 KLV ('KLVA'). +const KLVA: &[u8] = &[0x05, 0x04, b'K', b'L', b'V', b'A']; + +/// A 33-bit PTS field behind the '0010' prefix. +fn pts_field(ticks: u64) -> [u8; 5] { + [ + 0x21 | ((ticks >> 29) & 0x0e) as u8, + (ticks >> 22) as u8, + ((ticks >> 14) & 0xfe) as u8 | 1, + (ticks >> 7) as u8, + ((ticks << 1) & 0xfe) as u8 | 1, + ] +} + +/// An MPEG-2 video PES presenting at `pts`, its length left unbounded as video may: a picture +/// start code, then `len` bytes of `fill`. +fn picture(pts: u64, fill: u8, len: usize) -> Vec { + let mut pes = vec![0, 0, 1, 0xe0, 0, 0, 0x80, 0x80, 5]; + pes.extend_from_slice(&pts_field(pts)); + pes.extend_from_slice(&[0, 0, 1, 0x00]); + pes.resize(pes.len() + len, fill); + pes +} + +/// An asynchronous KLV PES (private_stream_1, no PTS) of `len` bytes of `fill`. +fn klv(fill: u8, len: usize) -> Vec { + let mut pes = vec![0, 0, 1, 0xbd]; + pes.extend_from_slice(&((3 + len) as u16).to_be_bytes()); + pes.extend_from_slice(&[0x84, 0x00, 0x00]); + pes.resize(pes.len() + len, fill); + pes +} + +/// A bounded audio PES presenting at `pts`, carrying `payload`. +fn audio(pts: u64, payload: &[u8]) -> Vec { + let mut pes = vec![0, 0, 1, 0xc0]; + pes.extend_from_slice(&((8 + payload.len()) as u16).to_be_bytes()); + pes.extend_from_slice(&[0x84, 0x80, 5]); + pes.extend_from_slice(&pts_field(pts)); + pes.extend_from_slice(payload); + pes +} + +/// A `pid` packet whose payload is TS-scrambled. +fn scrambled(mux: &mut Mux, pid: u16) { + let at = mux.out.len(); + mux.fill(pid, 1); + mux.out[at + 3] |= 0x80; +} + +/// An importer of one program, onto a catalog with the `mpegts` section. +struct Imported { + consumer: moq_net::broadcast::Consumer, + catalog: crate::catalog::Producer, + import: Import, +} + +impl Imported { + fn new() -> Self { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let import = Import::new(broadcast, catalog.reserve()); + Self { + consumer, + catalog, + import, + } + } + + /// Import all of `data`. + fn all(data: &[u8]) -> Self { + let mut imported = Self::new(); + imported.import.decode(data).unwrap(); + imported.import.finish().unwrap(); + imported + } + + /// The `mpegts` catalog entry for `pid`: its track name and record. + fn entry(&self, pid: u16) -> Option<(String, super::catalog::Track)> { + let mpegts = self.catalog.snapshot().ext.mpegts; + mpegts.tracks.into_iter().find(|(_, track)| track.pid == pid) + } + + /// The stats row for `pid`. + fn row(&self, pid: u16) -> Stream { + self.import.stats().streams.get(&pid).cloned().unwrap_or_default() + } + + /// Every frame on the track carrying `pid`. + async fn frames(&self, pid: u16) -> Vec { + let (name, _) = self.entry(pid).expect("a track on the PID"); + let subscription = moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(3600)); + let track = self + .consumer + .track(&name) + .unwrap() + .subscribe(subscription) + .await + .unwrap(); + let mut reader = crate::container::Consumer::new( + track, + crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), + ); + let mut frames = Vec::new(); + while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await { + frames.push(frame); + } + frames + } +} + +/// MPEG-2 video rides whole while its PTS still paces the program, and a TS round trip keeps +/// every PES byte of it. +#[tokio::test(start_paused = true)] +async fn mpeg2_video_rides_whole_and_survives_a_round_trip() { + let mut mux = Mux::default(); + mux.pat(&[(1, PMT)]).pmt(PMT, 1, CLOCK, &[(0x02, VIDEO)], 0); + let start = 2_000 * MS; + let mut pictures = Vec::new(); + for i in 0..8u64 { + let ticks = start + i * 40 * MS; + mux.packet(CLOCK, Flags::pcr(ticks)); + let pes = picture(ticks / 300 + LEAD, i as u8, 300 + i as usize * 50); + mux.whole(VIDEO, &pes); + pictures.push(pes); + } + mux.packet(CLOCK, Flags::pcr(start + 8 * 40 * MS)); + + let imported = Imported::all(&mux.bytes()); + let (_, entry) = imported.entry(VIDEO).expect("the video is in the catalog"); + assert_eq!(entry.pes_packet.as_ref().map(|pes| pes.stream_type), Some(0x02)); + let snapshot = imported.catalog.snapshot(); + assert!( + snapshot.video.renditions.is_empty(), + "no rendition a player cannot decode" + ); + let frames = imported.frames(VIDEO).await; + let payloads: Vec<_> = frames.iter().map(|frame| frame.payload.to_vec()).collect(); + assert_eq!(payloads, pictures); + assert!(frames.windows(2).all(|pair| pair[0].timestamp < pair[1].timestamp)); + let row = imported.row(VIDEO); + assert_eq!((row.track.as_str(), row.class), (".ts", Class::Video)); + assert_eq!((row.units, row.refused, row.damaged), (8, 0, 0)); + + let ts = super::export_test::drain_with( + super::export_test::open_ts( + crate::source::announced(&imported.consumer).await, + crate::catalog::CatalogFormat::Hang, + ) + .await + .unwrap(), + ) + .await; + let again = Imported::all(&ts); + let (_, entry) = again.entry(VIDEO).expect("the export lists the video"); + assert_eq!(entry.pes_packet.as_ref().map(|pes| pes.stream_type), Some(0x02)); + let payloads: Vec<_> = again + .frames(VIDEO) + .await + .iter() + .map(|frame| frame.payload.to_vec()) + .collect(); + assert_eq!(payloads, pictures, "the round trip keeps every PES byte"); +} + +/// Each ISO 13818-6 DSM-CC type rides DSM-CC sections, so each is reassembled as sections +/// rather than failing a PES start code. +#[tokio::test(start_paused = true)] +async fn dsmcc_types_ride_as_sections() { + let types = [ + (0x0a, 0x110, 0x3a), + (0x0b, 0x111, 0x3b), + (0x0c, 0x112, 0x3d), + (0x0d, 0x113, 0x3e), + ]; + let mut mux = Mux::default(); + let streams: Vec<_> = types.iter().map(|&(stream_type, pid, _)| (stream_type, pid)).collect(); + mux.pat(&[(1, PMT)]).pmt(PMT, 1, CLOCK, &streams, 0); + mux.packet(CLOCK, Flags::pcr(0)); + let mut sections = Vec::new(); + for &(_, pid, table_id) in &types { + let section = psi::section(table_id, pid, 0, 0, 0, &[pid as u8; 40]); + mux.section(pid, §ion); + sections.push(section); + } + mux.packet(CLOCK, Flags::pcr(40 * MS)); + + let imported = Imported::all(&mux.bytes()); + for (&(stream_type, pid, _), section) in types.iter().zip(sections) { + let (_, entry) = imported.entry(pid).expect("a section track"); + assert_eq!(entry.verbatim.as_ref().map(|v| v.stream_type), Some(stream_type)); + let frames = imported.frames(pid).await; + assert_eq!(frames.len(), 1, "{stream_type:#04x}"); + assert_eq!(frames[0].payload.as_ref(), section); + let row = imported.row(pid); + assert_eq!((row.units, row.damaged, row.refused), (1, 0, 0), "{stream_type:#04x}"); + } +} + +/// A 0x86 stream without a CUEI registration is refused once and never PES-parsed, while LATM +/// AAC, which nothing here decodes, rides whole. +#[tokio::test(start_paused = true)] +async fn ambiguous_streams_are_refused_and_latm_rides_whole() { + const DTS: u16 = 0x120; + const LATM: u16 = 0x121; + let mut mux = Mux::default(); + mux.pat(&[(1, PMT)]).pmt(PMT, 1, CLOCK, &[(0x86, DTS), (0x11, LATM)], 0); + mux.packet(CLOCK, Flags::pcr(0)); + let latm = audio(LEAD, &[0x56, 0xe0, 0x10, 0xaa, 0xbb]); + mux.whole(LATM, &latm); + mux.whole(DTS, &audio(LEAD, &[0x7f, 0xfe, 0x80, 0x01])); + mux.packet(CLOCK, Flags::pcr(40 * MS)); + + let imported = Imported::all(&mux.bytes()); + assert!(imported.entry(DTS).is_none()); + let row = imported.row(DTS); + assert_eq!((row.track.as_str(), row.class), ("", Class::Data)); + assert_eq!((row.refused, row.damaged, row.units), (1, 0, 0)); + + let (_, entry) = imported.entry(LATM).expect("LATM is carried"); + assert_eq!(entry.pes_packet.as_ref().map(|pes| pes.stream_type), Some(0x11)); + let frames = imported.frames(LATM).await; + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].payload.as_ref(), latm); +} + +/// Once the program's PMT is in, a packet on a PID nothing claims counts in its own row: the CAT, +/// an ATSC PSIP base PID, and a PID no table lists. Packets before the PAT or the PMT are +/// acquisition, and the null PID, the captured SI and the tables themselves are never counted. +/// A TS-scrambled packet on a carried stream is refused rather than damaged. +#[tokio::test(start_paused = true)] +async fn unclaimed_pids_count_once_the_pmt_is_in() { + const CAT: u16 = 0x0001; + const PSIP: u16 = 0x1ffb; + let mut mux = Mux::default(); + // A mid-stream join: media and strays before the PAT, then before the PMT. + mux.fill(DATA, 2).fill(UNLISTED, 2); + mux.pat(&[(1, PMT)]); + mux.fill(DATA, 1).fill(UNLISTED, 1); + mux.pmt_with(PMT, 1, CLOCK, &[(0x06, DATA, KLVA)], 0); + mux.packet(CLOCK, Flags::pcr(0)); + mux.section(CAT, &psi::section(0x01, 0xffff, 0, 0, 0, &[])); + mux.fill(PSIP, 2).fill(UNLISTED, 3).fill(0x1fff, 2).fill(0x11, 1); + mux.pat(&[(1, PMT)]).pmt_with(PMT, 1, CLOCK, &[(0x06, DATA, KLVA)], 0); + scrambled(&mut mux, DATA); + let pes = klv(0x5a, 64); + mux.whole(DATA, &pes); + mux.packet(CLOCK, Flags::pcr(40 * MS)); + + let imported = Imported::all(&mux.bytes()); + let stats = imported.import.stats(); + let rows: Vec = stats.streams.keys().copied().collect(); + assert_eq!(rows, [CAT, DATA, UNLISTED, PSIP]); + assert_eq!( + stats.streams[&UNLISTED], + Stream { + class: Class::Data, + refused: 3, + ..Default::default() + } + ); + for (pid, refused) in [(CAT, 1), (PSIP, 2)] { + let row = &stats.streams[&pid]; + assert_eq!((row.track.as_str(), row.class, row.refused), ("", Class::Data, refused)); + } + let data = &stats.streams[&DATA]; + assert_eq!( + (data.track.as_str(), data.units, data.refused, data.damaged), + (".ts", 1, 1, 0) + ); + assert_eq!(imported.frames(DATA).await[0].payload.as_ref(), pes); +} + +/// Two programs, the second's media and a stray PID ahead of its PMT. Under `--program all` +/// nothing counts until both PMTs are in, and a stray PID counts once for the multiplex. A +/// `--program 1` import cannot tell program 2's PIDs from strays, so it counts neither. +#[tokio::test(start_paused = true)] +async fn an_mpts_counts_strays_once_both_pmts_are_in() { + const PMT2: u16 = 0x30; + const DATA2: u16 = 0x201; + let mut mux = Mux::default(); + mux.pat(&[(1, PMT), (2, PMT2)]); + mux.pmt_with(PMT, 1, 0x1f1, &[(0x06, DATA, KLVA)], 0); + mux.packet(0x1f1, Flags::pcr(0)).packet(0x1f2, Flags::pcr(0)); + mux.whole(DATA2, &klv(2, 32)); + mux.fill(UNLISTED, 2); + mux.pmt_with(PMT2, 2, 0x1f2, &[(0x06, DATA2, KLVA)], 0); + mux.whole(DATA, &klv(1, 32)); + mux.whole(DATA2, &klv(2, 32)); + mux.fill(UNLISTED, 4); + mux.packet(0x1f1, Flags::pcr(40 * MS)) + .packet(0x1f2, Flags::pcr(40 * MS)); + let data = mux.bytes(); + + let origin = crate::source::produce_origin(); + let mut programs = Programs::new(origin, "event.hang", crate::catalog::Config::default()); + programs.decode(&data).unwrap(); + let stats = programs.stats(); + assert_eq!( + stats.streams[&UNLISTED], + Stream { + class: Class::Data, + refused: 4, + ..Default::default() + } + ); + for pid in [DATA, DATA2] { + assert_eq!(stats.streams[&pid].refused, 0, "{pid:#x}"); + assert_eq!(stats.streams[&pid].track, ".ts", "{pid:#x}"); + } + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut one = Import::new(broadcast, catalog.reserve()).with_program(1); + one.decode(&data).unwrap(); + let rows: Vec = one.stats().streams.keys().copied().collect(); + assert_eq!(rows, [DATA], "only program 1's own PIDs"); +} + +/// A PMT the PAT lists but that never arrives holds the count back for TR 101 290's 0.5 s of +/// program clock, timed on a program whose clock runs, and no longer. +#[tokio::test(start_paused = true)] +async fn a_pmt_that_never_arrives_stops_holding_the_count() { + let mut mux = Mux::default(); + mux.pat(&[(1, PMT), (2, 0x30)]); + mux.pmt_with(PMT, 1, CLOCK, &[(0x06, DATA, KLVA)], 0); + for ms in [0, 200, 400, 500] { + mux.packet(CLOCK, Flags::pcr(ms * MS)).fill(UNLISTED, 1); + } + mux.packet(CLOCK, Flags::pcr(501 * MS)).fill(UNLISTED, 2); + + let origin = crate::source::produce_origin(); + let mut programs = Programs::new(origin, "event.hang", crate::catalog::Config::default()); + programs.decode(&mux.bytes()).unwrap(); + assert_eq!(programs.stats().streams[&UNLISTED].refused, 2); +} From d04c93b4e0b9d9294ffa5b5f1ee43c68913212ee Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 21:41:22 -0700 Subject: [PATCH 04/16] fix(mux): carry whole MPEG-2 pictures without starving audio A placed PES with a decode time is due by it rather than by its placement, so a whole picture shares the multiplex. test/ts gains an --mpeg2 arm decoding the round trip, and the T-STD model H.262 video. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ts.yml | 9 +++ rs/moq-mux/src/container/ts/export.rs | 8 ++- rs/moq-mux/src/container/ts/schedule.rs | 34 ++++++++- test/justfile | 3 +- test/ts/README.md | 20 ++++-- test/ts/compliance.py | 57 +++++++++++++-- test/ts/mpeg2.py | 92 +++++++++++++++++++++++++ test/ts/run.sh | 38 +++++++++- 8 files changed, 245 insertions(+), 16 deletions(-) create mode 100644 test/ts/mpeg2.py diff --git a/.github/workflows/ts.yml b/.github/workflows/ts.yml index 5ae868e04f0..aef043581d0 100644 --- a/.github/workflows/ts.yml +++ b/.github/workflows/ts.yml @@ -72,6 +72,15 @@ jobs: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs shell: bash -leo pipefail {0} + # MPEG-2 video the importer carries as whole PES: FFmpeg decodes the capture to the + # source's pictures, and the video passes the T-STD model. + - name: TS MPEG-2 video + if: ${{ !cancelled() }} + run: nix develop --command just test ts --mpeg2 + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + shell: bash -leo pipefail {0} + - name: TS audio-only SCTE-35 if: ${{ !cancelled() }} run: nix develop --command just test ts --clockless audio-scte35 diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 9528e3b47c8..a233ee7e579 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -2315,10 +2315,11 @@ where }; let pid = track.pid; let place = frame.timestamp.as_nanos(); + // Its first access unit decodes at the DTS, which a reordered picture carries ahead of its PTS. let decode = pes .as_ref() - .and_then(|pes| pes.pts) - .map(|pts| unwrap_pts(pts, frame.timestamp)); + .and_then(|pes| pes.dts.or(pes.pts)) + .map(|ticks| unwrap_pts(ticks, frame.timestamp)); let dolby = Dolby::described(&track.descriptors) .filter(|_| matches!(track.kind, Kind::PesPacket { stream_type: 0x06 })); let frames = match (&pes, decode, dolby) { @@ -2869,6 +2870,8 @@ const PES_DTS_LEN: usize = 5; struct PesPacket { /// Its PTS in 90 kHz ticks, as the 33-bit field carries it. pts: Option, + /// Its DTS, likewise, where it differs from the PTS. + dts: Option, /// Where its payload begins, past the header. payload: usize, } @@ -2892,6 +2895,7 @@ impl PesPacket { } Ok(Self { pts: pes.pts, + dts: pes.dts, payload: pes.header.len(), }) } diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index 73d7fe8856a..8fe53f38f13 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -688,6 +688,8 @@ impl Schedule { /// bytes (the section pointer field) just before it. It /// decodes at `decode`, or each of `frames` at its own time, as [`Self::push_frames`] /// lists them, which only the PID's decoder buffer reads: the unit misses no deadline. + /// It yields to units due before it decodes, so a large one, such as a whole picture, + /// shares the multiplex rather than holding it from placement until its last byte. pub fn push_placed( &mut self, pid: u16, @@ -697,7 +699,10 @@ impl Schedule { keyframe: bool, frames: Vec<(u128, usize)>, ) -> anyhow::Result<()> { - let timing = Timing { by: place, decode }; + let timing = Timing { + by: decode.max(place), + decode, + }; self.queue(pid, timing, Some((place, prefix)), packets, keyframe, frames) } @@ -1448,6 +1453,33 @@ mod tests { out } + /// A placed picture five slots long, decoding a second out, shares the multiplex with a + /// unit due while it is still going out, rather than holding the rate until its last byte. + #[test] + fn a_placed_pes_yields_to_units_due_before_it_decodes() { + let mut schedule = Schedule::new(Duration::from_millis(25)); + schedule.set_rate(Some(RATE)); + schedule.set_buffer(1, VIDEO); + let mut picture = unit(7, 200); + picture[1] |= 0x40; + schedule + .push_placed(7, (ms(50), ms(1_000)), 0, picture, true, Vec::new()) + .unwrap(); + schedule.push(1, timing(100), unit(1, 30), true).unwrap(); + let slots = drain(&mut schedule); + let last = slots + .iter() + .filter(|(_, pids, _)| pids.contains_key(&1)) + .map(|&(index, ..)| index) + .max(); + assert!( + last.is_some_and(|last| last <= slot(ms(100))), + "the unit due at 100 ms goes out by slot {last:?}" + ); + let sent: usize = slots.iter().filter_map(|(_, pids, _)| pids.get(&7)).sum(); + assert_eq!(sent, 200); + } + /// The pointer byte goes just before placement, so it cannot cost a section another packet. #[test] fn placed_first_byte_accounts_for_the_section_pointer() { diff --git a/test/justfile b/test/justfile index 241df719711..154f3f9766c 100644 --- a/test/justfile +++ b/test/justfile @@ -149,7 +149,8 @@ drill-sensitivity *args: # (use a real capture), `--live` (grade PCR release timing and byte position off # the pipe instead, which a capture cannot carry; nightly runs this arm), # `--open-gop` (an open-GOP clip whose orphan leading pictures are trimmed), `--hrd` -# (1080p video filling a broadcast-sized CPB), `--delay 2s` (the exporter's). See +# (1080p video filling a broadcast-sized CPB), `--mpeg2` (MPEG-2 video carried whole, +# decoded from the capture), `--delay 2s` (the exporter's). See # ts/README.md. # Check the subscriber's MPEG-TS output for IRD compliance. diff --git a/test/ts/README.md b/test/ts/README.md index 8e7f71e464e..3f33d592269 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -44,6 +44,7 @@ just test ts --leak # 1080p video with no HRD, filling its level's 2 just test ts --full # the same with two audio PIDs, filling its multiplex with no nulls just test ts --fragment-pes # legal headers split across packets, with FFmpeg/TSDuck grading just test ts --klv # KLV and DVB AC-3 carried as whole PES, graded against the source +just test ts --mpeg2 # MPEG-2 video carried as whole PES, decoded from the capture just test ts --delay 2s # pass the exporter's --delay just test ts --hrd --rate 7000000 # congest the subscriber's leg below the media rate just test ts --loss 1 # drop 1% of the relay's datagrams to the subscriber @@ -68,6 +69,12 @@ source placement on the program clock, and none after its PTS; and FFmpeg readin each as KLV with the same PES and PTS. `holes.py` and the T-STD model grade the AC-3, whose buffer drains each sync frame at its own decode time. +`--mpeg2` publishes Main Profile at Main Level MPEG-2 video beside MPEG audio. The +importer does not decode MPEG-2 video, so it rides its track whole while its PTS +still paces the program. `mpeg2.py` decodes source and capture with FFmpeg: past +the GOP the subscriber joins in, every picture is the source's next one, through +the source's last. `holes.py` and the T-STD model grade the rest. + `--live` swaps the analyzer, not the rig: the same round-trip runs, but the subscriber's stdout goes straight into `pcr-timing.py` instead of a capture file. That is the only arm that can see release timing at all, and it is what nightly @@ -144,8 +151,8 @@ decoder (2.4.2; Rec. ITU-T H.222.0, whose 10/2014 edition is a free download), fed on the stream's own PCR clock: ```text -video (AVC 2.14.3.1, HEVC 2.17.2) TB --Rx--> MB --Rbx (leak)--> EB --DTS--> decoder -audio (2.4.2.3) TB --Rx--> B ----------------PTS--> decoder +video (H.262 2.4.2.3, AVC 2.14.3.1, HEVC 2.17.2) TB --Rx--> MB --Rbx (leak)--> EB --DTS--> decoder +audio (2.4.2.3) TB --Rx--> B ----------------PTS--> decoder ``` It fails a stream where TB, MB, EB or B overflows, where TB stays occupied for a @@ -156,7 +163,7 @@ underflowed access unit finished arriving, and the longest any access unit waite TSDuck supplies the transport analysis, not this decoder-buffer model. Commercial analyzers advertise T-STD checks; no suitable maintained open-source replacement -was established for this harness. Its parameters are transcribed from the specs: H.264 Table A-1 and H.265 Table A.8 for the level, the +was established for this harness. Its parameters are transcribed from the specs: H.262 Tables 8-13 and 8-14, H.264 Table A-1 and H.265 Table A.8 for the level, the ADTS and "other audio" rates and sizes in H.222.0 2.4.2.3, and ATSC A/52 and A/53 Part 5 for AC-3 and E-AC-3. TSDuck does the parsing: `tstables` decodes the PMT and `tsp -P pes --avc-access-unit` the SPS, AVC and HEVC alike. @@ -171,6 +178,11 @@ level defaults too. An `AVC_timing_and_HRD_descriptor` or transfer to the HRD's own schedule, which is not modelled, so such a stream is refused. +MPEG-2 video takes Rx = 1.2 Rmax and Rbx = Rmax from its level, EB from the +sequence header's `vbv_buffer_size`, and MB from what the level's VBV leaves over, +as H.222.0 2.4.2.3 specifies for the leak method a variable-rate encoder signals +with `vbv_delay` 0xFFFF. Its picture, GOP or sequence header opens each access unit. + Opus is graded against ADTS's buffers for the same channel count: the Opus-in-TS draft gives Rx (2 Mb/s for 1-2 channels, matching ADTS) but leaves the buffer size unset, so that size is borrowed rather than specified. A stream with no parameters @@ -894,7 +906,7 @@ browser interop. It runs nightly on `main`, and release PRs touching a workspace crate `run.sh` builds (kept in sync by `sh/gh/ts-paths.py`), the harness, or the toolchain run it too: `just test ts-controls`, `just test ts-pair-controls`, the default and 2 Mb/s round trips, HRD, leak, full, headroom, -open-GOP, KLV (strict), passthrough (generated, Kyrion and MPTS), EIT, and T-STD controls. Each step runs even after another test fails. +open-GOP, KLV (strict), MPEG-2 video, passthrough (generated, Kyrion and MPTS), EIT, and T-STD controls. Each step runs even after another test fails. All recipes use the same pinned Nix FFmpeg and TSDuck locally and in CI. Nightly and manual runs also invoke `just test ts --pair`, diff --git a/test/ts/compliance.py b/test/ts/compliance.py index 829100951ef..4fe25562878 100755 --- a/test/ts/compliance.py +++ b/test/ts/compliance.py @@ -436,6 +436,11 @@ def check_service_descriptors(analysis: dict) -> Check: # H.265 Table A.9 CpbNalFactor for Main, Main 10 and Main Still Picture (profile_idc 1-3). HEVC_NAL_FACTOR = 1100 +# H.262 Tables 8-13 and 8-14: level (the low nibble of profile_and_level_indication) -> +# (Rmax bit/s, VBV_max bits), the same for every profile that has the level. +MPEG2_LEVELS = {10: (4_000_000, 475_136), 8: (15_000_000, 1_835_008), 6: (60_000_000, 7_340_032), 4: (80_000_000, 9_781_248)} +MPEG2_LEVEL_NAMES = {10: "Low", 8: "Main", 6: "High-1440", 4: "High"} + # H.222.0 2.4.2.3: ADTS (Rx, BSn) by channels, the LFE not counted. ADTS_BUFFERS = ((2, 2_000_000, 3584), (8, 5_529_600, 8976), (12, 8_294_400, 12804), (48, 33_177_600, 51216)) # channel_configuration -> full-bandwidth channels (5.1 and 7.1 drop the LFE). @@ -462,6 +467,7 @@ def check_service_descriptors(analysis: dict) -> Check: } # stream_type -> model, for the types identified by stream_type alone. STREAM_KINDS = { + 0x02: "mpeg2-video", 0x1B: "avc", 0x24: "hevc", 0x0F: "adts", @@ -662,6 +668,38 @@ def _hrd(sps: dict[str, str], prefix: str, value: str) -> tuple[float, float] | return rate, size +def mpeg2_params(es: bytes) -> Params: + """H.222.0 2.4.2.3 buffers for H.262 video from its first sequence header and extension. + + Graded by the leak method (Rbx = Rmax), which H.222.0 applies when vbv_delay is 0xFFFF, as + a variable-rate encoder codes it. + """ + head = es.find(b"\x00\x00\x01\xb3") + ext = es.find(b"\x00\x00\x01\xb5", head) + if head < 0 or ext < 0 or len(es) < ext + 9 or es[ext + 4] >> 4 != 1: + raise Refused("no sequence header and sequence extension, which an H.262 stream starts with") + # vbv_buffer_size_value: 10 bits after bit_rate_value and its marker; the extension's 8 bits + # follow its bit_rate_extension and marker. + vbv_low = (es[head + 10] & 0x1F) << 5 | es[head + 11] >> 3 + vbv_high = es[ext + 8] + # profile_and_level_indication follows the 4-bit extension_start_code_identifier. + indication = (es[ext + 4] & 0x0F) << 4 | es[ext + 5] >> 4 + level = indication & 0x0F + if indication & 0x80 or level not in MPEG2_LEVELS: + raise Refused(f"H.262 profile_and_level_indication {indication:#04x} has no Table 8-13 entry") + rmax, vbv_max = MPEG2_LEVELS[level] + vbv = (vbv_high << 10 | vbv_low) * 16 * 1024 + overhead = (0.004 + 1 / 750) * max(rmax, 2_000_000) + return Params( + label=f"H.262 {MPEG2_LEVEL_NAMES[level]} level, vbv {vbv / 1e3:.0f} kbit", + # Rx = 1.2 * Rmax; MBS = BSmux + BSoh + VBV_max - vbv_buffer_size; EBS = vbv_buffer_size. + rx=1.2 * rmax, + mb=(overhead + vbv_max - vbv) / 8, + eb=vbv / 8, + rbx=rmax, + ) + + def avc_params(sps: dict[str, str]) -> Params: """H.222.0 2.14.3.1 buffers from the SPS: level limits, overridden by a declared NAL HRD. @@ -835,7 +873,7 @@ def stream_kind(stream: Stream) -> str | None: return "opus" if kind == 0x06 and tags & {"DVB_enhanced_AC3_descriptor", "enhanced_AC3_descriptor"}: return "eac3-dvb" - if kind in (0x01, 0x02, 0x10, 0x11, 0x1C, 0x20, 0x21, 0x42, 0xD1, 0xEA): + if kind in (0x01, 0x10, 0x11, 0x1C, 0x20, 0x21, 0x42, 0xD1, 0xEA): raise Refused(f"stream_type 0x{kind:02X} is audio/video this model has no parameters for") return None @@ -845,6 +883,8 @@ def stream_params(kind: str, stream: Stream) -> tuple[Params, object]: es = stream.es if kind == "avc": return avc_params(stream.sps), None + if kind == "mpeg2-video": + return mpeg2_params(bytes(es)), None if kind == "hevc": return hevc_params(stream.sps), None if kind == "teletext": @@ -879,9 +919,16 @@ def stream_params(kind: str, stream: Stream) -> tuple[Params, object]: # NAL unit types that open a new access unit when they follow the last VCL NAL unit # of the previous one (H.264 7.4.1.2.3, H.265 7.4.2.4.4), and the VCL types. -AU_OPENERS = {"avc": {6, 7, 8, 9, 14, 15, 16, 17, 18}, "hevc": {32, 33, 34, 35, 39, 41, 42, 43, 44, *range(48, 56)}} -VCL = {"avc": set(range(1, 6)), "hevc": set(range(0, 32))} -DELIMITER = {"avc": 9, "hevc": 35} +# H.262 6.2.1 has no NAL units: a picture opens at its sequence, GOP or picture header after +# the last slice of the one before, keyed by the start code value. +AU_OPENERS = { + "avc": {6, 7, 8, 9, 14, 15, 16, 17, 18}, + "hevc": {32, 33, 34, 35, 39, 41, 42, 43, 44, *range(48, 56)}, + "mpeg2-video": {0xB3, 0xB8, 0x00}, +} +VCL = {"avc": set(range(1, 6)), "hevc": set(range(0, 32)), "mpeg2-video": set(range(0x01, 0xB0))} +# H.262 has no access unit delimiter, and needs none: its picture header is one. +DELIMITER = {"avc": 9, "hevc": 35, "mpeg2-video": 0x00} def video_units(stream: Stream, kind: str) -> list[AccessUnit]: @@ -900,7 +947,7 @@ def video_units(stream: Stream, kind: str) -> list[AccessUnit]: after_vcl = True # the first opener starts the first whole access unit pos = es.find(b"\x00\x00\x01") while 0 <= pos < len(es) - 3: - nal = (es[pos + 3] >> 1) & 0x3F if kind == "hevc" else es[pos + 3] & 0x1F + nal = {"hevc": (es[pos + 3] >> 1) & 0x3F, "mpeg2-video": es[pos + 3]}.get(kind, es[pos + 3] & 0x1F) delimited |= nal == DELIMITER[kind] if nal in AU_OPENERS[kind] and after_vcl: # The zero_byte ahead of the start code belongs to this NAL unit (H.264 B.1.1, diff --git a/test/ts/mpeg2.py b/test/ts/mpeg2.py new file mode 100644 index 00000000000..c9db77bca1e --- /dev/null +++ b/test/ts/mpeg2.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Grade a round trip of MPEG-2 video, which the importer carries as whole PES, by decoding it. + + ./mpeg2.py source.ts capture.ts + +The importer does not decode MPEG-2 video: it carries each PES whole and the exporter writes it +back unchanged but for its PTS and DTS. So FFmpeg must read the capture's video as MPEG-2, decode +it without an error, and produce the source's pictures: once its first whole GOP starts, every +decoded picture is the source's next one, through the source's last. The subscriber joins at the +relay's latest group, so the run may start up to holes.JOIN into the source, and the pictures of +the GOP it joins in, whose references never arrived, may decode to garbage or not at all. + +Exit status is 0 when every check passes, 1 otherwise. +""" + +import argparse +import json +import subprocess +import sys +from fractions import Fraction +from pathlib import Path + +import holes + + +def codecs(path): + """The codec of each audio and video stream FFmpeg finds.""" + report = json.loads(subprocess.run( + ["ffprobe", "-v", "error", "-show_entries", "stream=codec_type,codec_name", "-of", "json", str(path)], + check=True, capture_output=True, text=True, + ).stdout) + return sorted((s["codec_type"], s["codec_name"]) for s in report["streams"] if s["codec_type"] in ("audio", "video")) + + +def pictures(path): + """The MD5 of each decoded picture, and FFmpeg's decode errors.""" + result = subprocess.run( + ["ffmpeg", "-v", "error", "-i", str(path), "-map", "0:v:0", "-f", "framemd5", "-"], + capture_output=True, text=True, check=False, + ) + if result.returncode: + raise SystemExit(f"error: ffmpeg could not decode {path}: {result.stderr.strip()}") + digests = [line.rsplit(",", 1)[1].strip() for line in result.stdout.splitlines() if line and not line.startswith("#")] + return digests, [line for line in result.stderr.splitlines() if line.strip()] + + +def grade(source, capture, gop, rate): + """Raise SystemExit naming the first miss; return a summary otherwise.""" + if codecs(capture) != codecs(source): + raise SystemExit(f"error: the capture's streams are {codecs(capture)}, the source's {codecs(source)}") + ours, errors = pictures(source) + if errors: + raise SystemExit(f"error: the source does not decode cleanly: {errors[:3]}") + theirs, errors = pictures(capture) + index = {digest: k for k, digest in enumerate(ours)} + # The pictures of the joined GOP before its next I picture may not be the source's. + lead = next((j for j, digest in enumerate(theirs) if digest in index), None) + if lead is None: + raise SystemExit("error: no decoded picture of the capture is the source's") + if lead > gop: + raise SystemExit(f"error: the first {lead} pictures are not the source's, past one {gop}-picture GOP") + first = index[theirs[lead]] + run = theirs[lead:] + if run != ours[first : first + len(run)]: + miss = next(j for j, digest in enumerate(run) if first + j >= len(ours) or ours[first + j] != digest) + raise SystemExit(f"error: decoded picture {lead + miss} is not the source's picture {first + miss}") + if first + len(run) != len(ours): + raise SystemExit(f"error: the capture ends at picture {first + len(run)} of the source's {len(ours)}") + if Fraction(first, rate) > holes.JOIN: + raise SystemExit(f"error: the capture starts at picture {first}, past the {float(holes.JOIN)}s join") + # Damage past the join already shows as a picture that is not the source's, so an error is + # only allowed for missing references at the join. + if errors and not lead: + raise SystemExit(f"error: FFmpeg reports decode errors on a capture that joined on a whole GOP: {errors[:3]}") + return {"pictures": len(run), "source": len(ours), "first": first, "joined": lead} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("source", type=Path) + parser.add_argument("capture", type=Path) + parser.add_argument("--gop", type=int, default=25, help="pictures per GOP in the source") + parser.add_argument("--rate", type=int, default=25, help="pictures per second in the source") + args = parser.parse_args() + report = grade(args.source, args.capture, args.gop, args.rate) + print(f"MPEG-2 video: {report['pictures']} of {report['source']} pictures decoded as the source's, " + f"from picture {report['first']} ({report['joined']} joined pictures skipped)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/ts/run.sh b/test/ts/run.sh index cbca53d1473..f4bf9958574 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -26,6 +26,7 @@ # ./run.sh --fragment-pes # split generated clear PES headers across packets # ./run.sh --clockless sections # sections, pes, or audio-scte35 without video # ./run.sh --klv # add KLV and DVB AC-3, carried as whole PES, and grade them +# ./run.sh --mpeg2 # MPEG-2 video, carried as whole PES, graded by decoding it # ./run.sh --delay 2s # pass the exporter's --delay # ./run.sh --rate 7000000 # bufferbloat: 7 Mb/s to the subscriber behind a deep queue (moq-shaper) # ./run.sh --loss 10 # drop 10% of the relay's datagrams to the subscriber (moq-shaper) @@ -87,6 +88,7 @@ FULL="" # the same with audio, in a multiplex the source fills with no n FRAGMENT_PES="" # use following null slots for fragmented PES headers CLOCKLESS="" # generated programs clocked by audio or a dedicated PCR-only PID KLV="" # inject asynchronous and synchronous KLV beside DVB AC-3, and grade every PES +MPEG2="" # publish MPEG-2 video, which the importer carries whole, and decode the capture LOSS="" # percent of relay -> subscriber datagrams dropped at random by moq-shaper RATE="" # bits/s moq-shaper carries from the relay to the subscriber, behind QUEUE QUEUE="" # how long a datagram may wait for RATE before the path tail-drops it @@ -184,6 +186,10 @@ while [[ $# -gt 0 ]]; do KLV=1 shift ;; + --mpeg2) + MPEG2=1 + shift + ;; --delay) EXPORT=(--delay "$2") shift 2 @@ -287,13 +293,18 @@ if [[ -n "$KLV" && -n "$SOURCE$ANALYZE_ONLY$HRD$HEADROOM$LEAK$FULL$OPEN_GOP$FRAG exit 1 fi +if [[ -n "$MPEG2" && -n "$SOURCE$ANALYZE_ONLY$HRD$HEADROOM$LEAK$FULL$OPEN_GOP$FRAGMENT_PES$PAIR$LIVE$PASSTHROUGH$MPTS$KLV" ]]; then + echo "error: --mpeg2 generates its own clip and decodes the demultiplexed export" >&2 + exit 1 +fi + if [[ -n "$CLOCKLESS" ]]; then case "$CLOCKLESS" in audio-scte35 | sections | pes) ;; *) echo "error: unsupported --clockless profile: $CLOCKLESS" >&2 exit 1 ;; esac - if [[ -n "$SOURCE$ANALYZE_ONLY$HRD$HEADROOM$LEAK$FULL$OPEN_GOP$FRAGMENT_PES$PAIR$LIVE$PASSTHROUGH$MPTS$KLV$RATE$LOSS" ]]; then + if [[ -n "$SOURCE$ANALYZE_ONLY$HRD$HEADROOM$LEAK$FULL$OPEN_GOP$FRAGMENT_PES$PAIR$LIVE$PASSTHROUGH$MPTS$KLV$MPEG2$RATE$LOSS" ]]; then echo "error: --clockless generates and grades an unimpaired program without video" >&2 exit 1 fi @@ -547,6 +558,17 @@ elif [[ -n "$KLV" ]]; then -c:a:0 aac -b:a:0 128k -c:a:1 ac3 -b:a:1 192k \ -f mpegts -mpegts_flags system_b -muxrate "$BITRATE" -pcr_period 20 "$HARNESS_RUN/media.ts" python3 "$DIR/klv.py" inject "$HARNESS_RUN/media.ts" "$SRC_TS" --report "$HARNESS_RUN/klv-injected.json" +elif [[ -n "$MPEG2" ]]; then + echo "### generating ~${DURATION}s MPEG-2 video and MPEG audio clip" + # Main Profile at Main Level with its whole VBV buffer, the shape of an SD contribution + # feed. Variable rate, so the encoder codes vbv_delay 0xFFFF and the T-STD leak method applies. + ffmpeg -y -hide_banner -loglevel error \ + -f lavfi -i "testsrc=size=720x576:rate=25" \ + -f lavfi -i "sine=frequency=1000:sample_rate=48000" \ + -t "$DURATION" \ + -c:v mpeg2video -g 25 -bf 2 -b:v 6M -maxrate 8M -bufsize 1835008 \ + -c:a mp2 -b:a 192k \ + -f mpegts -muxrate "$BITRATE" -pcr_period 20 "$SRC_TS" else echo "### generating ~${DURATION}s broadcast-like ${OPEN_GOP:+open-GOP }clip with ffmpeg" # CBR with a 20 ms PCR, like a contribution feed. Not cosmetic: `regulate` @@ -894,7 +916,7 @@ fi # pair. Other arms retain their specific structural/model oracles until their # acquisition and decoder-priming contracts are covered by fixtures. A path # narrower than the media sheds groups, which only holes.py grades. -if [[ -z "$SOURCE$OPEN_GOP$HRD$HEADROOM$LEAK$FULL$KLV$CLOCKLESS$RATE" ]]; then +if [[ -z "$SOURCE$OPEN_GOP$HRD$HEADROOM$LEAK$FULL$KLV$MPEG2$CLOCKLESS$RATE" ]]; then if ! python3 "$DIR/decode.py" "$SRC_TS" "$SUB_TS" --output "$HARNESS_RUN/decode"; then # The exporter counts what it dropped (late), trimmed (before its output began), # and lost upstream (gaps), and logs each loss, so the failure says which it was. @@ -913,7 +935,7 @@ fi # grade only the access units that arrived. Under --rate the path is narrower than the media, # so the export has to shed: only whole groups, and never the last, which a stall or a # latency that grew with the queue would cost. -if [[ -n "$HRD$HEADROOM$LEAK$FULL$KLV" || (-n "$RATE" && -z "$SOURCE") ]]; then +if [[ -n "$HRD$HEADROOM$LEAK$FULL$KLV$MPEG2" || (-n "$RATE" && -z "$SOURCE") ]]; then echo if ! python3 "$DIR/holes.py" "$SUB_TS" "$SRC_TS" ${RATE:+--shed}; then echo >&2 @@ -970,6 +992,16 @@ if [[ -n "$KLV" ]]; then fi fi +if [[ -n "$MPEG2" ]]; then + echo + if ! python3 "$DIR/mpeg2.py" "$SRC_TS" "$SUB_TS"; then + echo >&2 + echo "error: the MPEG-2 video did not survive the round trip (see round-trip logs below)" >&2 + dump_logs + exit 1 + fi +fi + # The delay's lead over the source's hides packets the export spends beyond the source's # until it runs out, so they are counted outright. if [[ -n "$FULL" ]]; then From b907921af15ff25900954ad8bf7fb9bc0070c4bf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 21:41:22 -0700 Subject: [PATCH 05/16] docs(mux): document TS routes and bench the PID accounting Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 10 +- doc/concept/stats.md | 12 ++- drafts/draft-lcurley-moq-mpegts.md | 7 +- rs/justfile | 5 + rs/moq-mux/Cargo.toml | 4 + rs/moq-mux/benches/ts_programs.rs | 150 +++++++++++++++++++++++++++++ sh/gh/nightly.sh | 2 +- 7 files changed, 182 insertions(+), 8 deletions(-) create mode 100644 rs/moq-mux/benches/ts_programs.rs diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 6c6ce75edec..27a93712b06 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -99,12 +99,16 @@ For a source already in MPEG-TS, `ffmpeg -c copy -f mpegts` re-muxes and drops P or use [`import srt`](/bin/srt). MPEG-TS import carries H.264/H.265 and AAC/MP2/AC-3/E-AC-3/Opus, and passes -SCTE-35 and every other stream it does not decode (KLV, subtitles, DVB AC-3) -through as tracks. Each such PES goes through whole, at its place on the program clock +SCTE-35, DSM-CC sections, and every other stream it does not decode (MPEG-2 +video, LATM AAC, KLV, subtitles, DVB AC-3) through as tracks. MPEG-2 video still +paces the program by its timestamps. Each such PES goes through whole, at its place on the program clock (PCR); one before the first PCR is dropped. A stream it cannot carry unaltered is refused with a warning: any in a program with no PCR, and one whose PES carry an ESCR, pack header or TREF, are scrambled, leave a non-video PES unbounded, -or have a malformed header. +or have a malformed header, as is `0x86` without a SCTE-35 registration, which +may be DTS audio. A PID no table claims, such as conditional access or ATSC +PSIP, is dropped with one warning and counted in the +[stats](/concept/stats#mpeg-ts-sections); `--passthrough` keeps them. SCTE-35 cues and private PES share the media clock, or anchor on the first PCR when the PMT declares only data. Audio-only and data-only programs round-trip without video; a data-only program still requires PCR. A program declaring diff --git a/doc/concept/stats.md b/doc/concept/stats.md index fcc5c5d19bf..ddf6e4c2553 100644 --- a/doc/concept/stats.md +++ b/doc/concept/stats.md @@ -53,7 +53,17 @@ program clock since its last access unit. Its losses are `resyncs`, bytes `discarded` scanning for audio sync, `unconfirmed` frames, `damaged` units, `unplaced` units the program clock could not place, the times the stream was `refused`, and its shares of `continuityCountError`, `transportError`, and -`ptsError`. +`ptsError`. A TS-scrambled packet on a carried stream counts as `refused`. + +Every PID but the null PID either takes a route or is counted. A listed stream +the import cannot carry, such as `0x86` without a SCTE-35 registration, gets a +row with an empty `track` and one `refused`. Once every program's PMT is in, or +has been missing for 0.5 s of program clock, a PID that no PAT, PMT, or +captured SI table claims, such as the CAT, conditional access, or ATSC PSIP, +gets a `data` row with an empty `track` and `refused` counting its packets, +with one warning when it starts. An import of one `--program` of a +multi-program stream cannot tell another program's PIDs from these, so it +counts none of them. The stream-wide counters count ETSI TR 101 290 checks at that standard's fixed limits: diff --git a/drafts/draft-lcurley-moq-mpegts.md b/drafts/draft-lcurley-moq-mpegts.md index 38dfd6ffeb9..d86286b3fae 100644 --- a/drafts/draft-lcurley-moq-mpegts.md +++ b/drafts/draft-lcurley-moq-mpegts.md @@ -232,7 +232,7 @@ type Verbatim = { } ~~~ -`streamType` is the PMT `stream_type` to re-announce: 0x86 for SCTE-35 {{scte35}}, 0x05 for private sections. +`streamType` is the PMT `stream_type` to re-announce: 0x86 for SCTE-35 {{scte35}}, 0x05 for private sections, 0x0A to 0x0D for ISO/IEC 13818-6 DSM-CC sections. `framing` is required, and `"section"` is its only value: each frame is one complete section, header and CRC included, timestamped with its placement: the program clock at its first section byte, on the same timeline as a whole PES below. A consumer MUST refuse a catalog whose `verbatim` record names any other `framing` or none, rather than guess; in particular it MUST NOT read the `"pes"` framing of draft -00, or a record without `framing`, as sections. @@ -247,7 +247,7 @@ type PesPacket = { } ~~~ -`streamType` is the PMT `stream_type` to re-announce, as for `Verbatim`: 0x06 for private data such as asynchronous KLV, teletext, DVB subtitles and DVB AC-3, 0x15 for synchronous metadata. +`streamType` is the PMT `stream_type` to re-announce, as for `Verbatim`: 0x01 and 0x02 for MPEG-1 and MPEG-2 video, 0x11 for LATM AAC, 0x06 for private data such as asynchronous KLV, teletext, DVB subtitles and DVB AC-3, 0x15 for synchronous metadata. Each frame MUST be exactly one PES packet, from its start code through its payload, with a `PES_packet_length` that matches the frame, or zero only where {{mpeg2}} lets a video stream leave it unbounded. The `stream_id`, the PTS and DTS when present, and every other header field ride in the frame, with the PTS and DTS on the catalog clock; a PES without a PTS carries none. The frame's timestamp is the packet's placement: the program clock at its first byte, on the same timeline. @@ -366,7 +366,7 @@ This document defines no packaging value; {{msfts}} defines `mpeg2ts` for the pa A consumer rebuilding a transport stream: - MUST place each track on its recorded `pid`, and MUST assign an unused PID to a track with none. -- MUST put the PMT on `program.pmtPid` and build the PAT and PMT from `program`. +- MUST put the PMT on `program.pmtPid` and build the PAT and PMT from `program`. When it carries a NIT (the `si` entry for `table_id` 0x40 on PID 0x0010), the PAT MUST also list program 0 with that `network_PID`, so a receiver can find it. - MUST choose a PCR PID and emit PCRs on it at the program clock's repetition interval. With no media stream it MUST use an unused PID with adaptation-only PCR packets and no PMT elementary-stream entry. It MUST NOT refuse a program for lacking video. - MUST re-emit each track's `descriptors` as its ES-level descriptors, and `programDescriptors` as the PMT's `program_info`. - MUST re-emit each `si` entry's sections byte-for-byte on that entry's PID, reading them from its track ({{si-track}}), at least as often as its `interval` when declared. @@ -450,6 +450,7 @@ A broadcast demultiplexed from a DVB transport stream: video and audio described - Section-framed `verbatim` timestamps use PCR-derived placement at the first section byte, as whole PES do, rather than a video presentation time. - Added the `pesPacket` record: each frame one whole PES packet, timestamped with its placement. A consumer that predates it drops the track rather than refusing the catalog. - `verbatim` carries sections only, breaking -00: `framing` is required and `"section"` its only value, and `streamId` is gone. A consumer refuses a -00 catalog with `"pes"` framing or a `verbatim` record without `framing`. A publisher carries an undecoded PES stream as `pesPacket`, and refuses one it cannot carry unaltered. +- MPEG-1/2 video and LATM AAC are named among `pesPacket` streams, and the DSM-CC types 0x0A to 0x0D among section-framed `verbatim` ones. A rebuilt PAT lists the network entry when the NIT is carried. - A flagged PCR jump on an `m2ts` track follows hang's discontinuity marker rather than a skipped group sequence. - An `m2ts` track unwraps PCRs from its history, clamps extrapolated object times at a forward discontinuity, and cuts a fallback group after one second. `randomAccess` covers every published group; `muxRate` follows the measured source rate. diff --git a/rs/justfile b/rs/justfile index 890c2934ceb..384753ad1bb 100644 --- a/rs/justfile +++ b/rs/justfile @@ -481,6 +481,11 @@ bench-latency: bench-ts-schedule *args: cargo bench --locked -p moq-mux --bench ts_schedule -- "$@" +# Split an MPTS across program count and stray PIDs, against null packets that skip the accounting. +[positional-arguments] +bench-ts-programs *args: + cargo bench --locked -p moq-mux --bench ts_programs -- "$@" + # Sweep container export tracks and queued frames independently. [positional-arguments] bench-export *args: diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index 16f90fc6ab0..e3638e3a93a 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -57,6 +57,10 @@ harness = false name = "ts_schedule" harness = false +[[bench]] +name = "ts_programs" +harness = false + [[bench]] name = "export" harness = false diff --git a/rs/moq-mux/benches/ts_programs.rs b/rs/moq-mux/benches/ts_programs.rs new file mode 100644 index 00000000000..e6aaec344b9 --- /dev/null +++ b/rs/moq-mux/benches/ts_programs.rs @@ -0,0 +1,150 @@ +//! Splitting an MPTS across program count and stray PID count. Each stray PID's packets go out +//! once as unclaimed PIDs, which every PID's accounting reads, and once as null packets, the +//! baseline that no accounting reads, so the gap between the two is the accounting's cost. + +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use std::time::Duration; + +const MPEG_2: crc::Crc = crc::Crc::::new(&crc::CRC_32_MPEG_2); +/// 27 MHz ticks per 40 ms interval. +const INTERVAL: u64 = 40 * 27_000; +const INTERVALS: u64 = 50; +const NULL: u16 = 0x1fff; +const STRAY: u16 = 0x1000; + +/// A transport stream under construction, continuity counted per PID. +#[derive(Default)] +struct Mux { + out: Vec, + cc: std::collections::HashMap, +} + +impl Mux { + /// A payload-only packet; `payload` fills it, or the tail is 0xFF. + fn packet(&mut self, pid: u16, start: bool, payload: &[u8]) { + let cc = self.cc.entry(pid).or_default(); + let mut packet = vec![0x47, (start as u8) << 6 | (pid >> 8) as u8, pid as u8, 0x10 | *cc]; + *cc = (*cc + 1) & 0x0f; + packet.extend_from_slice(payload); + packet.resize(188, 0xff); + self.out.extend_from_slice(&packet); + } + + /// A one-packet PSI section of `table_id` with `body`. + fn section(&mut self, pid: u16, table_id: u8, extension: u16, body: &[u8]) { + let mut section = vec![table_id, 0xb0, 0, (extension >> 8) as u8, extension as u8, 0xc1, 0, 0]; + section.extend_from_slice(body); + let len = section.len() - 3 + 4; + section[1] |= (len >> 8) as u8; + section[2] = len as u8; + section.extend_from_slice(&MPEG_2.checksum(§ion).to_be_bytes()); + let mut payload = vec![0]; + payload.extend_from_slice(§ion); + self.packet(pid, true, &payload); + } + + fn pcr(&mut self, pid: u16, ticks: u64) { + let base = ticks / 300; + let ext = ticks % 300; + let pcr = [ + (base >> 25) as u8, + (base >> 17) as u8, + (base >> 9) as u8, + (base >> 1) as u8, + ((base & 1) << 7) as u8 | 0x7e | (ext >> 8) as u8, + ext as u8, + ]; + // Adaptation only, so the continuity counter holds. + let cc = self.cc.entry(pid).or_default(); + let mut packet = vec![0x47, (pid >> 8) as u8, pid as u8, 0x20 | *cc, 183, 0x10]; + packet.extend_from_slice(&pcr); + packet.resize(188, 0xff); + self.out.extend_from_slice(&packet); + } + + /// A whole private-data PES of one packet, with no PTS. + fn pes(&mut self, pid: u16, fill: u8) { + let mut pes = vec![0, 0, 1, 0xbd, 0, 0, 0x84, 0, 0]; + pes.resize(184, fill); + let len = pes.len() - 6; + pes[4] = (len >> 8) as u8; + pes[5] = len as u8; + self.packet(pid, true, &pes); + } +} + +/// `programs` programs, each a clock and one private-data stream, alongside `strays` PIDs no +/// table lists, sent as null packets instead when `null`. +fn mpts(programs: u16, strays: u16, null: bool) -> Vec { + let mut mux = Mux::default(); + let pat: Vec = (1..=programs) + .flat_map(|program| { + let pmt = 0x20 + program; + [(program >> 8) as u8, program as u8, 0xe0 | (pmt >> 8) as u8, pmt as u8] + }) + .collect(); + for interval in 0..INTERVALS { + mux.section(0, 0x00, 1, &pat); + for program in 1..=programs { + let (pmt, clock, data) = (0x20 + program, 0x100 + program, 0x400 + program); + let mut body = vec![0xe0 | (clock >> 8) as u8, clock as u8, 0xf0, 0]; + body.extend_from_slice(&[0x06, 0xe0 | (data >> 8) as u8, data as u8, 0xf0, 0]); + mux.section(pmt, 0x02, program, &body); + mux.pcr(clock, interval * INTERVAL); + mux.pes(data, interval as u8); + } + for stray in 0..strays { + mux.packet(if null { NULL } else { STRAY + stray }, false, &[0xff; 184]); + } + } + mux.out +} + +fn split(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let _guard = runtime.enter(); + let mut group = c.benchmark_group("ts_programs"); + group.measurement_time(Duration::from_secs(1)); + group.warm_up_time(Duration::from_millis(200)); + group.sample_size(10); + for programs in [1u16, 4, 16] { + for strays in [1u16, 16, 256] { + for (lane, null) in [("null", true), ("stray", false)] { + let data = mpts(programs, strays, null); + let (origin, driver) = moq_net::origin::Producer::new(Default::default()); + std::mem::forget(driver); + let mut check = moq_mux::container::ts::Programs::new(origin, "bench", Default::default()); + check.decode(&data).unwrap(); + let counted = check.stats().streams.values().filter(|row| row.refused > 0).count(); + assert_eq!( + counted, + if null { 0 } else { strays as usize }, + "{lane} {programs}x{strays}" + ); + group.throughput(Throughput::Bytes(data.len() as u64)); + let id = BenchmarkId::new(lane, format!("{programs}x{strays}")); + group.bench_function(id, |b| { + b.iter_batched( + || { + let (origin, driver) = moq_net::origin::Producer::new(Default::default()); + std::mem::forget(driver); + moq_mux::container::ts::Programs::new(origin, "bench", Default::default()) + }, + |mut split| { + split.decode(&data).unwrap(); + split + }, + criterion::BatchSize::PerIteration, + ); + }); + } + } + } + group.finish(); +} + +criterion_group!(benches, split); +criterion_main!(benches); diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 909b9b3268c..7f780a263e2 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -92,7 +92,7 @@ case "$1" in # only Criterion estimates enter the trend report. criterion extra --no-fail-fast -p moq-e2ee -p moq-json -p moq-archive -p moq-cli -p moq-mux \ --bench protect --bench codec --bench table --bench allocations --bench window \ - --bench writer --bench play_buffer --bench jitter + --bench writer --bench play_buffer --bench jitter --bench ts_programs ;; rust-uring) # One run per target makes an unsupported/skipped ring fail report From 153a2d9b8d29da4124227371fc3bd92640591ce3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 21:46:12 -0700 Subject: [PATCH 06/16] docs(mux): migrate note for TS stats rows Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/MIGRATE.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rs/moq-mux/MIGRATE.md b/rs/moq-mux/MIGRATE.md index 0dcc889ce36..b5e2947cd38 100644 --- a/rs/moq-mux/MIGRATE.md +++ b/rs/moq-mux/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `ts::stats::Snapshot::streams` rows cover PIDs that carry no track (#6252) + +A TS import now reports a row for every PID it does not carry: a listed stream it refuses, and, once the PMTs are in, a PID no table claims. Such a row has an empty `track` and class `Data`, and its `refused` counts the refusal or the PID's packets; `refused` also counts TS-scrambled packets on a carried stream. +MPEG-1/2 video is a `pesPacket` track with class `Video` rather than a clock-only row. +Filter on a non-empty `track` where you read the rows as published tracks. + ### `Source` resolution takes the reference's epoch (#5699) `Source::bind(rel)`, `resolve(rel)`, and `subscribe_track(rel, name)` take the catalog entry's optional epoch after `rel`. From 90d55c7f51f601d2e3702500b8aafe4139197d74 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 01:30:41 -0700 Subject: [PATCH 07/16] test(mux): a TS PID keeps one row through its PMT lifecycle Unlisted, added by a PMT revision, removed while its packets continue, and added again, checked under one Import and under Programs. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/routes_test.rs | 57 ++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/rs/moq-mux/src/container/ts/routes_test.rs b/rs/moq-mux/src/container/ts/routes_test.rs index b37d64c85da..c9549fbba5d 100644 --- a/rs/moq-mux/src/container/ts/routes_test.rs +++ b/rs/moq-mux/src/container/ts/routes_test.rs @@ -360,3 +360,60 @@ async fn a_pmt_that_never_arrives_stops_holding_the_count() { programs.decode(&mux.bytes()).unwrap(); assert_eq!(programs.stats().streams[&UNLISTED].refused, 2); } + +/// One PID through its whole life, packets flowing throughout: unlisted, added by a PMT +/// revision, removed by the next, then added again. It keeps one row, whose counters never go +/// back, whichever owner holds it: the import while carried, the dropped-PID count otherwise, +/// which `Programs` holds for `--program all`. +#[tokio::test(start_paused = true)] +async fn a_pid_keeps_one_row_through_its_lifecycle() { + let mut steps = Vec::new(); + let mut mux = Mux::default(); + mux.pat(&[(1, PMT)]).pmt_at(0, PMT, 1, CLOCK, &[(0x06, DATA, KLVA)], 0); + mux.packet(CLOCK, Flags::pcr(0)); + mux.fill(UNLISTED, 3); + steps.push(mux.out.len()); + mux.pmt_at(1, PMT, 1, CLOCK, &[(0x06, DATA, KLVA), (0x06, UNLISTED, KLVA)], 0); + mux.packet(CLOCK, Flags::pcr(40 * MS)); + mux.whole(UNLISTED, &klv(1, 32)); + mux.whole(UNLISTED, &klv(2, 32)); + // A later PCR places them. + mux.packet(CLOCK, Flags::pcr(60 * MS)); + steps.push(mux.out.len()); + mux.pmt_at(2, PMT, 1, CLOCK, &[(0x06, DATA, KLVA)], 0); + mux.fill(UNLISTED, 4); + steps.push(mux.out.len()); + mux.pmt_at(3, PMT, 1, CLOCK, &[(0x06, DATA, KLVA), (0x06, UNLISTED, KLVA)], 0); + mux.packet(CLOCK, Flags::pcr(80 * MS)); + mux.whole(UNLISTED, &klv(3, 32)); + mux.packet(CLOCK, Flags::pcr(100 * MS)); + steps.push(mux.out.len()); + let data = mux.bytes(); + // (track, units, refused) after each step. + let expected = [("", 0, 3), (".ts", 2, 3), ("", 2, 7), (".ts", 3, 7)]; + + let mut one = Imported::new(); + let mut programs = Programs::new( + crate::source::produce_origin(), + "event.hang", + crate::catalog::Config::default(), + ); + let mut at = 0; + for (end, (track, units, refused)) in steps.into_iter().zip(expected) { + one.import.decode(&data[at..end]).unwrap(); + programs.decode(&data[at..end]).unwrap(); + at = end; + for (owner, row) in [ + ("import", one.row(UNLISTED)), + ("programs", programs.stats().streams[&UNLISTED].clone()), + ] { + let class = row.class; + assert_eq!( + (row.track.as_str(), row.units, row.refused, row.damaged), + (track, units, refused, 0), + "{owner} at {end}" + ); + assert_eq!(class, Class::Data, "{owner} at {end}"); + } + } +} From db75614f7739aaa672dd62f7cbaf071d929d71f8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 01:30:47 -0700 Subject: [PATCH 08/16] chore(quest): finish TS stream routes The epic's profile rows for MPEG-1/2 video, DSM-CC, uncarried PIDs and the PAT network entry are landed. A program a later PAT adds is left to pat-lifecycle, which now notes its PIDs count nowhere. Co-Authored-By: Claude Opus 5.5 --- quest/m1/ts-reliability/README.md | 9 +- quest/m1/ts-reliability/pat-lifecycle.md | 3 +- quest/m1/ts-reliability/stream-routes.md | 108 ----------------------- 3 files changed, 6 insertions(+), 114 deletions(-) delete mode 100644 quest/m1/ts-reliability/stream-routes.md diff --git a/quest/m1/ts-reliability/README.md b/quest/m1/ts-reliability/README.md index a4510a732b6..1105139c852 100644 --- a/quest/m1/ts-reliability/README.md +++ b/quest/m1/ts-reliability/README.md @@ -68,9 +68,9 @@ refused means an error or a counted drop. | Private and unknown PES types, LATM (0x11), metadata (0x15) | Preserved: `pesPacket`, PTS and DTS rebased | landed | | PES clock fields (ESCR, pack header, TREF), PES scrambling | Refused, counted in `refused` | landed | | Private sections (0x05), SCTE-35 (0x86 + `CUEI`) | Preserved: `verbatim` section, `pts_adjustment` rebased | landed | -| MPEG-1/2 video (0x01/0x02) | Preserved: `pesPacket`, keeping its clock role; today clock only, payload discarded | [routes](/quest/m1/ts-reliability/stream-routes.md) | -| DSM-CC types (0x0A to 0x0D), 0x86 without `CUEI` | Today PES-parsed into damage, or dropped uncounted | [routes](/quest/m1/ts-reliability/stream-routes.md) | -| CAT, EMM/ECM, ATSC PSIP, other SI PIDs, unlisted PIDs, TS scrambling | Outside the profile, raw lane; counted when the import reads every program. Today dropped uncounted | [routes](/quest/m1/ts-reliability/stream-routes.md) | +| MPEG-1/2 video (0x01/0x02) | Preserved: `pesPacket`, keeping its clock role | landed | +| DSM-CC types (0x0A to 0x0D), 0x86 without `CUEI` | DSM-CC preserved as a `verbatim` section; 0x86 without `CUEI` refused, counted in `refused` | landed | +| CAT, EMM/ECM, ATSC PSIP, other SI PIDs, unlisted PIDs, TS scrambling | Outside the profile, raw lane; counted in `refused`, unclaimed PIDs when the import reads every program | landed | | DVB SI on 0x0010, 0x0011, 0x0012, 0x0014 | Preserved: snapshot track per PID and `table_id` | landed | | Program identity, program and ES descriptors | Preserved: descriptors follow each PMT revision; an identity change refuses | landed | | PMT add, remove, retype; PID reuse | Normalized without a new epoch: a changed or reused PID ends its track and continues under a fresh name, and the PMT is rebuilt at the next `version_number`. Import landed; today refusal or stale PMT on export | [PMT export](/quest/m1/ts-reliability/pmt-export.md) | @@ -80,7 +80,7 @@ refused means an error or a counted drop. | Mux rate, null stuffing | Normalized: re-padded to `muxRate` while the source is CBR | landed | | PSI and SI repetition | Normalized: PAT/PMT every 500 ms and at keyframes, SI at DVB maxima | landed | | Media PES headers, TS packet layer | Normalized: rebuilt by export; the raw lane keeps them | landed | -| PAT network PID entry | Preserved: listed on 0x0010 when a NIT is carried; today dropped | [routes](/quest/m1/ts-reliability/stream-routes.md) | +| PAT network PID entry | Preserved: listed on 0x0010 when a NIT is carried | landed | PMT fragmentation (#5220), initial mux-rate measurement (#5287), and decoder-time admission (#5306) have landed; PSI runs and catalog linger @@ -90,7 +90,6 @@ remain with their current quests. - [PMT export](/quest/m1/ts-reliability/pmt-export.md) - a TS export announces each PMT revision at the next `version_number` - [PAT changes](/quest/m1/ts-reliability/pat-lifecycle.md) - a revised PAT never leaves a broadcast meaning another program -- [Stream routes](/quest/m1/ts-reliability/stream-routes.md) - every PID gets a declared route, and what is not carried is counted - [Arrival stamps](/quest/m1/ts-reliability/arrival-stamps.md) - frames carry their local arrival time, so a stalled path never lets late frames out - [Release clock false alarms](/quest/m1/ts-reliability/release-clock-false-alarms.md) - no false jitter leap at a 500 ms delay and no false drift alarm diff --git a/quest/m1/ts-reliability/pat-lifecycle.md b/quest/m1/ts-reliability/pat-lifecycle.md index 3998e01bae2..fbfda735c2e 100644 --- a/quest/m1/ts-reliability/pat-lifecycle.md +++ b/quest/m1/ts-reliability/pat-lifecycle.md @@ -13,7 +13,8 @@ What the code does today: - A PAT naming another PMT PID for the program adds a reassembler but never drops the old one, and `program.pmtPid` keeps the first. -- `Programs` ignores a program a later PAT adds, without a warning. +- `Programs` ignores a program a later PAT adds, without a warning. Its + PMT still claims its PIDs, so they show in no `ts::stats` row either. - Under `--program all`, a later PAT that removes one program reaches every per-program `Import`; the removed program's `handle_pat` errors, `Programs::feed` propagates it, and the whole multiplex ingest stops, diff --git a/quest/m1/ts-reliability/stream-routes.md b/quest/m1/ts-reliability/stream-routes.md deleted file mode 100644 index 81235e9582a..00000000000 --- a/quest/m1/ts-reliability/stream-routes.md +++ /dev/null @@ -1,108 +0,0 @@ -# [M] Every TS PID gets a declared route - -## Goal - -TS import gives every PID of the programs it imports, other than the null -PID 0x1FFF, one declared route: decoded, whole-PES `pesPacket`, `verbatim` -section, captured PSI/SI, or not carried. Null packets are stuffing, rebuilt -by export, and stay uncounted. A PID the semantic lane does not carry shows -in `ts::stats` with a count and one warning, so an operator sees what to -move to the raw lane. Multiplex-wide PIDs no program claims are counted when -the import reads every program (an SPTS, or `--program all`); a -`--program N` import of an MPTS reports only its own program. A known -`stream_type` never implies carriage it does not get. - -## Plan - -Found 2026-10-20 in the fidelity audit (`Import::ensure_stream`, -`Import::decode`): - -- **MPEG-1/2 video (0x01/0x02)** is `Stream::Clock`: its PTS anchors the - timebase and its payload is discarded, so TS to Hang to TS loses the - program's video without an error, while an unknown `stream_type` survives - as `pesPacket`. Decided 2026-10-20 by the maintainer: carry it as an - opaque `pesPacket` and keep its clock role, so the round trip keeps the - video and a Hang player still sees no rendition it cannot decode. The - draft already lets a video PES leave `PES_packet_length` unbounded. - Rejected: refusing it loudly and leaving the video to the raw lane, which - drops a stream the semantic lane can carry unchanged. -- **Section-framed types outside 0x05 and CUEI-marked 0x86** are PES-parsed - today. The ISO 13818-6 DSM-CC types (0x0A to 0x0D) ride sections, so every - unit fails the start code and counts as `damaged`. Target: route each - DSM-CC type 13818-6 confirms as section-framed through `ensure_section`. -- **0x86 without CUEI** drops with a warning and no count. Count it in - `refused`, since it is ambiguous by design. -- **LATM/LOAS AAC (0x11)** already rides `pesPacket` with the `mpegts` - section, though the `Import` docs and the `ensure_stream` comment say it is - dropped. Fix the docs to match. -- **PIDs outside the PMT and the captured SI** (CAT on 0x0001 and the - EMM/ECM PIDs it or a CA descriptor names, TSDT, ATSC PSIP on 0x1FFB and the - PIDs its MGT signals, the DVB RST/DIT/SIT PIDs, and any PID no table lists) - drop silently. Count each in a `ts::stats` row. ATSC PSIP and conditional access are outside the semantic - profile; the guide points them at the raw lane. -- **The PAT's network PID entry** (`program_number` 0) is filtered out on - import, and `Export::build_psi` writes a PAT with only the program, while - the NIT itself passes through on 0x0010 as a snapshot track. Export lists - program 0 on 0x0010 whenever the catalog carries a NIT snapshot, so the - rebuilt PAT points a receiver at the NIT it carries. -- **TS-scrambled packets** on an elementary PID become PES parse damage. - Count them in `refused` instead, since the semantic lane cannot carry them - and the raw lane can. - -A PID counts as dropped only once every program the current PAT lists has -delivered its PMT: packets on a PID before then are acquisition, as -`Import::decode` already treats them, so media that precedes its PMT on a -mid-stream join never shows as refused, including one program's media -arriving before its PMT while another program's PMT is already known. A PAT -that adds a program suspends the count for PIDs no PMT claims until that -program's PMT arrives. A listed PMT that does not arrive within the 0.5 s of -program clock TR 101 290 allows (`PMT_error_2`, already counted) stops -holding the gate, so a stale or corrupt program never disables the count. - -Under `--program all`, every per-program `Import` reads the whole multiplex -and `Programs::stats` merges their rows by PID, so a per-program drop row -would shadow the program that carries the PID. `Programs` owns the -unclaimed-PID rows instead: a PID is dropped only when no program's PMT and -no captured SI claims it, counted once at the multiplex rather than per -program. A `--program N` import of an MPTS reassembles only its own PMT, so -it cannot tell another program's PIDs from unclaimed ones; it counts only its -own program's PIDs and leaves multiplex-wide accounting to `--program all`. -When the PAT lists only the selected program, it counts as above. Chosen -over passively acquiring every PMT, which adds a reassembler per program to -an import that serves one. -Classifying a PID looks it up in one map of every claimed PID, kept as -PMTs arrive, never by searching each program's PMT. A benchmark in -`rs/moq-mux/benches` feeds `Programs` a generated MPTS and sweeps program -count and PID count independently, against a baseline without the -accounting, so any slope it adds beyond the existing per-program feed shows. - -A dropped PID reuses `stats::Stream`, with no new field: a row keyed by its -PID in `Snapshot::streams`, an empty `track`, `Class::Data`, and `refused` -counting the PID's dropped packets, with one warning on the first. Widen the -`Snapshot::streams` and `refused` docs to say so, and report that as a -public API change in meaning. A PID keeps one row for its whole life, -whoever owns it: a PMT that claims a dropped PID, releases a carried one, or -claims it again moves that row between the carrying `Import` and the -dropped-PID owner (`Programs` under `--program all`) with every cumulative -counter intact, since counters never go backward. `track` and `class` follow -the current owner, empty and `Data` while dropped, and two rows never -coexist under one key. Coordinate the serialized shape with -the [MPEG-TS sections](/doc/concept/stats.md#mpeg-ts-sections) of a media report. - -Rust: `rs/moq-mux` TS import (and export for MPEG-2 video `pesPacket` -packetization). JS: none. Draft -([mpegts](/drafts/draft-lcurley-moq-mpegts.md)): list MPEG-2 video among the -`pesPacket` examples and the DSM-CC types among the section-framed ones. - -Tests: generated `fixture::Mux` inputs with MPEG-2 video, each DSM-CC type, -0x86 without CUEI, LATM, a PAT network entry with a NIT, a CAT plus -scrambled ES, a PSIP base PID, an unlisted PID, media before its first PMT, -a two-program MPTS under `--program all` whose second program's media -precedes its PMT, a PAT listing a program whose PMT never arrives, one -PID's full lifecycle (unlisted, added by a PMT revision, removed while its -packets continue, then added again) with its row's counters checked at each -step, and the same MPTS under `--program 1` with no rows for -program 2's PIDs, asserting the catalog, the -stats rows (the unlisted PID's row asserted field by field), and an export -round trip of the carried streams. The MPEG-2 video arm also runs the TSDuck -round trip and an FFmpeg decode of the export in `test/ts`. From fc80562b8eb4f300f823be796e66600885dcc66d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 01:40:24 -0700 Subject: [PATCH 09/16] fix(mux): ignore a TS PMT on another program's PMT PID in Programs A PMT section only claims PIDs, or names the multiplex clock, when the PAT lists its program on the PID that carried it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/programs.rs | 10 ++++++++-- rs/moq-mux/src/container/ts/routes_test.rs | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/rs/moq-mux/src/container/ts/programs.rs b/rs/moq-mux/src/container/ts/programs.rs index 9abf0f47a6f..5203cba21f8 100644 --- a/rs/moq-mux/src/container/ts/programs.rs +++ b/rs/moq-mux/src/container/ts/programs.rs @@ -1,6 +1,6 @@ //! Every program of a multiplex, each imported as its own broadcast. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use anyhow::Context; use mpeg2ts::ts::{Pid, TsPacket}; @@ -207,6 +207,8 @@ struct Tables { pat: PatReader, /// A reassembler for each PMT PID the PAT lists. pmts: HashMap, + /// Each `(program_number, PMT PID)` the PAT lists: a PMT on another program's PID is ignored. + listed: HashSet<(u16, Pid)>, /// The program and PCR PID whose clock times a PMT's lapse: the first program to deliver one. clock: Option<(u16, Pid)>, liveness: Liveness, @@ -220,6 +222,7 @@ impl Tables { scratch: Vec::new(), pat: PatReader::default(), pmts: HashMap::new(), + listed: HashSet::new(), clock: None, liveness: Liveness::default(), // Every program's catalog carries the `mpegts` section, so the SI is captured. @@ -240,6 +243,7 @@ impl Tables { self.pmts.entry(*pid).or_default(); } self.claims.pat(&listed, self.liveness.now()); + self.listed = listed.into_iter().collect(); } /// Read the tables and the clock out of `data`, counting each packet no table claims. @@ -270,7 +274,9 @@ impl Tables { if section.first() != Some(&psi::PMT_TABLE_ID) || !psi::crc_ok(§ion) { continue; } - if let Some(pmt) = psi::Pmt::parse(§ion) { + if let Some(pmt) = psi::Pmt::parse(§ion) + && self.listed.contains(&(pmt.program_number, pid)) + { self.pmt(&pmt); } } diff --git a/rs/moq-mux/src/container/ts/routes_test.rs b/rs/moq-mux/src/container/ts/routes_test.rs index c9549fbba5d..6ce88b38a34 100644 --- a/rs/moq-mux/src/container/ts/routes_test.rs +++ b/rs/moq-mux/src/container/ts/routes_test.rs @@ -343,6 +343,27 @@ async fn an_mpts_counts_strays_once_both_pmts_are_in() { assert_eq!(rows, [DATA], "only program 1's own PIDs"); } +/// A PMT section on another program's PMT PID claims nothing: program 2's PMT arriving on +/// program 1's PID never takes program 2's claim on a stray. +#[tokio::test(start_paused = true)] +async fn a_pmt_on_another_programs_pid_claims_nothing() { + const PMT2: u16 = 0x30; + const DATA2: u16 = 0x201; + let mut mux = Mux::default(); + mux.pat(&[(1, PMT), (2, PMT2)]); + mux.pmt_with(PMT, 1, CLOCK, &[(0x06, DATA, KLVA)], 0); + mux.pmt_with(PMT2, 2, CLOCK, &[(0x06, DATA2, KLVA)], 0); + mux.packet(CLOCK, Flags::pcr(0)); + mux.fill(UNLISTED, 2); + mux.pmt_at(1, PMT, 2, CLOCK, &[(0x06, UNLISTED, KLVA)], 0); + mux.fill(UNLISTED, 3); + + let origin = crate::source::produce_origin(); + let mut programs = Programs::new(origin, "event.hang", crate::catalog::Config::default()); + programs.decode(&mux.bytes()).unwrap(); + assert_eq!(programs.stats().streams[&UNLISTED].refused, 5); +} + /// A PMT the PAT lists but that never arrives holds the count back for TR 101 290's 0.5 s of /// program clock, timed on a program whose clock runs, and no longer. #[tokio::test(start_paused = true)] From efac015608a3441c9f4da6a140e74f964a67d88b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 22:01:00 -0700 Subject: [PATCH 10/16] fix(mux): group MPEG-1/2 video by sequence header A PES carrying a sequence_header_code opens a group and the pictures after it join it, so a joiner or a dropped group never lands on a dependent picture. Pictures before the first sequence header, or after a loss until the next, are dropped. The mpegts draft's pesPacket section says so, and ts_programs sweeps stray PIDs against null stuffing. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 3 +- drafts/draft-lcurley-moq-mpegts.md | 4 +- quest/m1/ts-reliability/README.md | 2 +- rs/justfile | 2 +- rs/moq-mux/benches/ts_programs.rs | 100 ++++++++++++++++----- rs/moq-mux/src/container/ts/catalog.rs | 3 +- rs/moq-mux/src/container/ts/import.rs | 89 ++++++++++++++---- rs/moq-mux/src/container/ts/routes_test.rs | 48 ++++++---- 8 files changed, 192 insertions(+), 59 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index d7b6f2d0d5b..0338b2c24f1 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -94,7 +94,8 @@ or use [`import srt`](/bin/srt). MPEG-TS import carries H.264/H.265 and AAC/MP2/AC-3/E-AC-3/Opus, and passes SCTE-35, DSM-CC sections, and every other stream it does not decode (MPEG-2 video, LATM AAC, KLV, subtitles, DVB AC-3) through as tracks. MPEG-2 video still -paces the program by its timestamps. Each such PES goes through whole, at its place on the program clock +paces the program by its timestamps, and a subscriber joins it at a sequence +header. Each such PES goes through whole, at its place on the program clock (PCR); one before the first PCR is dropped. A stream it cannot carry unaltered is refused with a warning: any in a program with no PCR, and one whose PES carry an ESCR, pack header or TREF, are scrambled, leave a non-video PES unbounded, diff --git a/drafts/draft-lcurley-moq-mpegts.md b/drafts/draft-lcurley-moq-mpegts.md index 550ff1bf31e..0d66e9cdad0 100644 --- a/drafts/draft-lcurley-moq-mpegts.md +++ b/drafts/draft-lcurley-moq-mpegts.md @@ -255,7 +255,8 @@ type PesPacket = { Each frame MUST be exactly one PES packet, from its start code through its payload, with a `PES_packet_length` that matches the frame, or zero only where {{mpeg2}} lets a video stream leave it unbounded. The `stream_id`, the PTS and DTS when present, and every other header field ride in the frame, with the PTS and DTS on the catalog clock; a PES without a PTS carries none. The frame's timestamp is the packet's placement: the program clock at its first byte, on the same timeline. -Each frame MUST be a keyframe in its own group, as for `verbatim`. +Each frame MUST be a keyframe in its own group, as for `verbatim`, except on MPEG-1 and MPEG-2 video (0x01 and 0x02), whose pictures depend on earlier ones: there a PES carrying a `sequence_header_code` (0x000001B3) MUST be a keyframe opening a group, and the PES up to the next one MUST join it. +A publisher drops the video PES before the first sequence header, and after a lost one until the next, rather than publish a picture without what it references. A consumer MUST refuse a frame that is not one well-formed PES packet. The record is a member beside `verbatim` rather than a `framing` value, so a consumer that predates it ignores the track instead of refusing the catalog. @@ -456,6 +457,7 @@ A broadcast demultiplexed from a DVB transport stream: video and audio described - Added the `pesPacket` record: each frame one whole PES packet, timestamped with its placement. A consumer that predates it drops the track rather than refusing the catalog. - `verbatim` carries sections only, breaking -00: `framing` is required and `"section"` its only value, and `streamId` is gone. A consumer refuses a -00 catalog with `"pes"` framing or a `verbatim` record without `framing`. A publisher carries an undecoded PES stream as `pesPacket`, and refuses one it cannot carry unaltered. - MPEG-1/2 video and LATM AAC are named among `pesPacket` streams, and the DSM-CC types 0x0A to 0x0D among section-framed `verbatim` ones. A rebuilt PAT lists the network entry when the NIT is carried. +- MPEG-1/2 video `pesPacket` frames group from one sequence header to the next, rather than one PES per group. - A flagged PCR jump on an `m2ts` track follows hang's discontinuity marker rather than a skipped group sequence. - An `m2ts` track unwraps PCRs from its history, clamps extrapolated object times at a forward discontinuity, and cuts a fallback group after one second. `randomAccess` covers every published group; `muxRate` follows the measured source rate. - A `tracks` entry is never changed in place: a stream the PMT revises continues under a new name, and one it drops leaves the section. `programDescriptors` follows the PMT; `program`'s `transportStreamId` and `programNumber` do not change. diff --git a/quest/m1/ts-reliability/README.md b/quest/m1/ts-reliability/README.md index 52264ee2125..bb3c6e4537b 100644 --- a/quest/m1/ts-reliability/README.md +++ b/quest/m1/ts-reliability/README.md @@ -67,7 +67,7 @@ refused means an error or a counted drop. | Private and unknown PES types, LATM (0x11), metadata (0x15) | Preserved: `pesPacket`, PTS and DTS rebased | landed | | PES clock fields (ESCR, pack header, TREF), PES scrambling | Refused, counted in `refused` | landed | | Private sections (0x05), SCTE-35 (0x86 + `CUEI`) | Preserved: `verbatim` section, `pts_adjustment` rebased | landed | -| MPEG-1/2 video (0x01/0x02) | Preserved: `pesPacket`, keeping its clock role | landed | +| MPEG-1/2 video (0x01/0x02) | Preserved: `pesPacket` grouped from one sequence header to the next, keeping its clock role | landed | | DSM-CC types (0x0A to 0x0D), 0x86 without `CUEI` | DSM-CC preserved as a `verbatim` section; 0x86 without `CUEI` refused, counted in `refused` | landed | | CAT, EMM/ECM, ATSC PSIP, other SI PIDs, unlisted PIDs, TS scrambling | Outside the profile, raw lane; counted in `refused`, unclaimed PIDs when the import reads every program | landed | | DVB SI on 0x0010, 0x0011, 0x0012, 0x0014 | Preserved: snapshot track per PID and `table_id` | landed | diff --git a/rs/justfile b/rs/justfile index 77b6f1c088c..7b191ffe28f 100644 --- a/rs/justfile +++ b/rs/justfile @@ -488,7 +488,7 @@ bench-latency: bench-ts-schedule *args: cargo bench --locked -p moq-mux --bench ts_schedule -- "$@" -# Split an MPTS across program count and stray PIDs, against null packets that skip the accounting. +# Ingest an MPTS across program count, bitrate and stray PIDs, the strays against null packets that skip the accounting. [positional-arguments] bench-ts-programs *args: cargo bench --locked -p moq-mux --bench ts_programs -- "$@" diff --git a/rs/moq-mux/benches/ts_programs.rs b/rs/moq-mux/benches/ts_programs.rs index 138d738ce9a..5c9e3a7763f 100644 --- a/rs/moq-mux/benches/ts_programs.rs +++ b/rs/moq-mux/benches/ts_programs.rs @@ -1,7 +1,8 @@ //! `--program all` TS ingest across program count and per-program bitrate, over one second of //! a generated multiplex. Throughput is the whole input, so a parse that repeats per program //! shows as a falling rate with program count. A PAT that also names programs no PMT -//! describes shows what a program costs before it starts. +//! describes shows what a program costs before it starts. Stray PIDs no table lists sweep the +//! unclaimed-PID accounting, against the same packets sent as null stuffing, which skips it. use std::time::Duration; @@ -58,6 +59,13 @@ fn undescribed_pid(program: u16) -> u16 { 0x1000 + program } +/// A PID no table lists, clear of every other. +fn stray_pid(stray: u16) -> u16 { + 0x1800 + stray +} + +const NULL_PID: u16 = 0x1fff; + #[derive(Default)] struct Mux { out: Vec, @@ -131,10 +139,25 @@ impl Mux { } } +/// What the multiplex carries besides its described programs. +#[derive(Clone, Copy)] +struct Extra { + /// Programs the PAT names whose PMT never arrives. + undescribed: u16, + /// PIDs no table lists, one packet each per frame. + strays: u16, + /// Send the strays' packets as null stuffing instead. + null: bool, +} + /// One second of `programs` H.264 programs at `bitrate` bits per second each, interleaved -/// frame by frame, each program carrying its PCR on its video PID. The PAT also names -/// `undescribed` programs whose PMT never arrives. -fn multiplex(programs: u16, undescribed: u16, bitrate: u64) -> Vec { +/// frame by frame, each program carrying its PCR on its video PID, beside `extra`. +fn multiplex(programs: u16, extra: Extra, bitrate: u64) -> Vec { + let Extra { + undescribed, + strays, + null, + } = extra; let packets = (bitrate / FPS / 8).div_ceil(188) as usize; let mut mux = Mux::default(); let entries: Vec = (1..=programs) @@ -176,6 +199,12 @@ fn multiplex(programs: u16, undescribed: u16, bitrate: u64) -> Vec { for program in 1..=programs { mux.frame(video_pid(program), packets, pts * 300, pts, frame % GOP == 0); } + for stray in 0..strays { + let pid = if null { NULL_PID } else { stray_pid(stray) }; + mux.header(pid, false, false); + let end = mux.out.len() + 184; + mux.out.resize(end, 0xff); + } } mux.out } @@ -201,34 +230,59 @@ fn programs(c: &mut Criterion) { group.measurement_time(Duration::from_secs(2)); group.warm_up_time(Duration::from_millis(300)); group.sample_size(10); + let mut cases = Vec::new(); for programs in [1u16, 4, 16] { for undescribed in [0u16, 1000] { for mbps in [1u64, 4] { - let input = multiplex(programs, undescribed, mbps * 1_000_000); - let (stats, _driver) = ingest(programs, &input); - assert!( - stats.streams.len() == usize::from(programs) && stats.streams.values().all(|s| s.units == FPS), - "every program delivers every frame" - ); - group.throughput(Throughput::Bytes(input.len() as u64)); + let extra = Extra { + undescribed, + strays: 0, + null: false, + }; let name = match undescribed { 0 => format!("{programs}"), _ => format!("{programs}+{undescribed}"), }; - group.bench_function(BenchmarkId::new(name, format!("{mbps}mbps")), |b| { - b.iter_batched( - setup, - |(mut programs, driver)| { - for chunk in input.chunks(188 * 7) { - programs.decode(chunk).unwrap(); - } - (programs, driver) - }, - BatchSize::PerIteration, - ) - }); + cases.push((programs, extra, mbps, name)); } } + for strays in [16u16, 256] { + for (lane, null) in [("null", true), ("stray", false)] { + let extra = Extra { + undescribed: 0, + strays, + null, + }; + cases.push((programs, extra, 1, format!("{programs}-{lane}{strays}"))); + } + } + } + for (programs, extra, mbps, name) in cases { + let input = multiplex(programs, extra, mbps * 1_000_000); + let (stats, _driver) = ingest(programs, &input); + let (media, strays): (Vec<_>, Vec<_>) = stats.streams.values().partition(|s| !s.track.is_empty()); + assert!( + media.len() == usize::from(programs) && media.iter().all(|s| s.units == FPS), + "every program delivers every frame" + ); + let counted = if extra.null { 0 } else { usize::from(extra.strays) }; + assert!( + strays.len() == counted && strays.iter().all(|s| s.refused == FPS), + "every stray packet is counted" + ); + group.throughput(Throughput::Bytes(input.len() as u64)); + group.bench_function(BenchmarkId::new(name, format!("{mbps}mbps")), |b| { + b.iter_batched( + setup, + |(mut programs, driver)| { + for chunk in input.chunks(188 * 7) { + programs.decode(chunk).unwrap(); + } + (programs, driver) + }, + BatchSize::PerIteration, + ) + }); } group.finish(); } diff --git a/rs/moq-mux/src/container/ts/catalog.rs b/rs/moq-mux/src/container/ts/catalog.rs index 63cfff2eab5..b72691befe7 100644 --- a/rs/moq-mux/src/container/ts/catalog.rs +++ b/rs/moq-mux/src/container/ts/catalog.rs @@ -367,7 +367,8 @@ impl Verbatim { /// Each frame is one complete PES packet, start code through payload, with its PTS, DTS /// and every other header field already on the catalog clock. Its timestamp is the /// packet's placement: the program clock at its first byte, which export sends it at. -/// A member beside [`Verbatim`] rather than a [`Framing`], so a consumer that predates it +/// Each frame is a group of its own, except MPEG-1/2 video's, grouped from one sequence header +/// to the next. A member beside [`Verbatim`] rather than a [`Framing`], so a consumer that predates it /// ignores the track instead of refusing the catalog. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default)] #[serde(rename_all = "camelCase")] diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index adab27c03bc..bbf24675ff7 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -2177,14 +2177,35 @@ fn adjust_splice(section: &mut [u8], offset: Offset) { /// catalog clock and every other byte as received, timestamped with its placement on the program /// clock. A PES without a PTS keeps none. See [`Import`] for when one is held, dropped, or refuses /// its stream. +/// +/// Each PES is a group of its own, except MPEG-1/2 video's: a PES carrying a sequence header opens +/// a group, where a decoder can start, and the pictures after it join that group. Those before the +/// first sequence header, or after a loss until the next, are dropped, since they depend on +/// pictures a subscriber never gets. struct PesPacketStream { track: crate::container::Producer, /// Held for its `Drop`, which clears this track's catalog entry. _entry: VerbatimEntry, /// The PMT `stream_type`, which says whether an unbounded PES is video. stream_type: u8, - /// Whole PES waiting for the PCR after them or the timebase's offset, in input order. - held: VecDeque, + /// Whole PES waiting for the PCR after them or the timebase's offset, in input order, each + /// with where it falls in the groups. + held: VecDeque<(Held, Join)>, + /// The stream lost a unit since the last PES held, so the next joins no group. + lost: bool, +} + +/// Where a whole PES falls in its track's groups. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Join { + /// It opens a group: every PES of a stream other than MPEG-1/2 video, and one of that video + /// carrying a sequence header. + Opens, + /// It joins the group open before it. + Continues, + /// It follows a loss, so the group before it ends and it is dropped with the PES after it + /// until one opens a group. + Resyncs, } /// A whole PES or section waiting to be placed. @@ -2230,38 +2251,53 @@ impl PesPacketStream { _entry: entry, stream_type, held: VecDeque::new(), + lost: false, }) } /// Hold one complete PES until it can be placed: [`Self::release`] publishes it. fn hold(&mut self, pending: Pending) { let span = pending.span.expect("a whole PES is spanned at its start"); - self.held.push_back(Held { + let join = match mpeg_video(self.stream_type) { + false => Join::Opens, + true if sequence_header(&pending.data) => Join::Opens, + true if self.lost => Join::Resyncs, + true => Join::Continues, + }; + self.lost = false; + let held = Held { payload: pending.data, span, _retained: pending.retained, - }); + }; + self.held.push_back((held, join)); + } + + /// The unit in progress was lost: an MPEG-1/2 picture after it may reference it. + fn lost(&mut self) { + self.lost = true; } /// The PCR `after` arrived on timebase `generation`: every PES held since the last is /// bracketed. fn bracket(&mut self, generation: u64, after: Sample) { - for held in &mut self.held { + for (held, _) in &mut self.held { held.span.bracket(generation, after); } } /// The timebase of `placement` ends: extrapolate the PES held past its last PCR. fn settle(&mut self, placement: &Placement) { - for held in &mut self.held { + for (held, _) in &mut self.held { held.span.settle(placement); } } - /// Publish the placed PES at the front, each in its own group, once `timebase` has its + /// Publish the placed PES at the front, grouped as [`Join`] says, once `timebase` has its /// offset, and drop those no PCR on their own timebase can place: one ended before them, now /// that `generation` is the timebase running. `pts` is the program's PTS history, which - /// picks each PCR's wrap. Returns how many went each way. + /// picks each PCR's wrap. Returns how many went each way; a picture no group takes is + /// neither. fn release( &mut self, timebase: &crate::catalog::Timebase, @@ -2275,23 +2311,32 @@ impl PesPacketStream { while let Some(timestamp) = self .held .front() - .and_then(|held| held.timestamp(pts, offset, generation)) + .and_then(|(held, _)| held.timestamp(pts, offset, generation)) { - let mut held = self.held.pop_front().expect("a front"); + let (mut held, join) = self.held.pop_front().expect("a front"); let Some(timestamp) = timestamp else { unplaced += 1; continue; }; rebase(&mut held.payload, offset); + if join == Join::Resyncs { + self.track.cut(None)?; + } let frame = crate::container::Frame { timestamp, duration: None, payload: bytes::Bytes::from(held.payload), - keyframe: true, + keyframe: join == Join::Opens, }; - self.track.write(frame)?; - self.track.cut(None)?; - published += 1; + match self.track.write(frame) { + Ok(()) => published += 1, + // A picture with no group open: before the first sequence header, or after a loss. + Err(crate::Error::MissingKeyframe(_)) => continue, + Err(err) => return Err(err.into()), + } + if !mpeg_video(self.stream_type) { + self.track.cut(None)?; + } } Ok((published, unplaced)) } @@ -2905,6 +2950,18 @@ fn mpeg_video(stream_type: u8) -> bool { matches!(stream_type, 0x01 | 0x02) } +/// Whether the whole PES `pes` carries an MPEG-1/2 `sequence_header_code` (ISO 13818-2 6.2.2.1), +/// where a decoder can start. Start codes never occur inside coded data, so any match is one. +fn sequence_header(pes: &[u8]) -> bool { + const SEQUENCE_HEADER: [u8; 4] = [0x00, 0x00, 0x01, 0xb3]; + let header = match pes.get(3) { + Some(&stream_id) if psi::has_optional_header(stream_id) => 9 + usize::from(pes.get(8).copied().unwrap_or(0)), + _ => 6, + }; + pes.get(header..) + .is_some_and(|payload| memchr::memmem::find(payload, &SEQUENCE_HEADER).is_some()) +} + /// One elementary stream's codec importer. enum Stream { H264 { @@ -3053,6 +3110,7 @@ impl Stream { match self { Stream::H264 { import, .. } => import.cut(None)?, Stream::H265 { import, .. } => import.cut(None)?, + Stream::PesPacket(stream) => stream.lost(), _ => {} } Ok(()) @@ -5044,7 +5102,8 @@ pub(super) mod test { true, )); bytes.extend_from_slice(&pcr_packet(VIDEO, 24_300_000)); - bytes.extend_from_slice(&pes_packet(VIDEO, 90_000)); + // A sequence header, so the picture opens a group rather than waiting for one. + bytes.extend_from_slice(&video_pes(VIDEO, 0, 90_000, None, &[0, 0, 1, 0xb3, 0, 0, 1, 0])); bytes.extend_from_slice(&packet(true, 0, 0, &CUE)); bytes.extend_from_slice(&pcr_packet(VIDEO, 27_000_000)); import.decode(&bytes).unwrap(); diff --git a/rs/moq-mux/src/container/ts/routes_test.rs b/rs/moq-mux/src/container/ts/routes_test.rs index 6ce88b38a34..dd9e97bb70e 100644 --- a/rs/moq-mux/src/container/ts/routes_test.rs +++ b/rs/moq-mux/src/container/ts/routes_test.rs @@ -29,11 +29,14 @@ fn pts_field(ticks: u64) -> [u8; 5] { ] } -/// An MPEG-2 video PES presenting at `pts`, its length left unbounded as video may: a picture -/// start code, then `len` bytes of `fill`. -fn picture(pts: u64, fill: u8, len: usize) -> Vec { +/// An MPEG-2 video PES presenting at `pts`, its length left unbounded as video may: a sequence +/// header start code when `sequence`, a picture start code, then `len` bytes of `fill`. +fn picture(pts: u64, sequence: bool, fill: u8, len: usize) -> Vec { let mut pes = vec![0, 0, 1, 0xe0, 0, 0, 0x80, 0x80, 5]; pes.extend_from_slice(&pts_field(pts)); + if sequence { + pes.extend_from_slice(&[0, 0, 1, 0xb3]); + } pes.extend_from_slice(&[0, 0, 1, 0x00]); pes.resize(pes.len() + len, fill); pes @@ -132,21 +135,34 @@ impl Imported { } /// MPEG-2 video rides whole while its PTS still paces the program, and a TS round trip keeps -/// every PES byte of it. +/// every PES byte of it. A sequence header opens each group, so the picture joined mid-GOP is +/// dropped, and so is the picture after one lost to scrambling, until the next sequence header. #[tokio::test(start_paused = true)] async fn mpeg2_video_rides_whole_and_survives_a_round_trip() { let mut mux = Mux::default(); mux.pat(&[(1, PMT)]).pmt(PMT, 1, CLOCK, &[(0x02, VIDEO)], 0); let start = 2_000 * MS; let mut pictures = Vec::new(); - for i in 0..8u64 { + for i in 0..9u64 { let ticks = start + i * 40 * MS; mux.packet(CLOCK, Flags::pcr(ticks)); - let pes = picture(ticks / 300 + LEAD, i as u8, 300 + i as usize * 50); + // Picture 6 is still open, its length unbounded, when the scrambled packet loses it. + if i == 7 { + scrambled(&mut mux, VIDEO); + } + let pes = picture( + ticks / 300 + LEAD, + matches!(i, 1 | 5 | 8), + i as u8, + 300 + i as usize * 50, + ); mux.whole(VIDEO, &pes); pictures.push(pes); } - mux.packet(CLOCK, Flags::pcr(start + 8 * 40 * MS)); + mux.packet(CLOCK, Flags::pcr(start + 9 * 40 * MS)); + // What a subscriber gets, and which pictures open a group. + let carried: Vec<_> = [1, 2, 3, 4, 5, 8].map(|i| pictures[i].clone()).into(); + let opens = [true, false, false, false, true, true]; let imported = Imported::all(&mux.bytes()); let (_, entry) = imported.entry(VIDEO).expect("the video is in the catalog"); @@ -158,11 +174,13 @@ async fn mpeg2_video_rides_whole_and_survives_a_round_trip() { ); let frames = imported.frames(VIDEO).await; let payloads: Vec<_> = frames.iter().map(|frame| frame.payload.to_vec()).collect(); - assert_eq!(payloads, pictures); + assert_eq!(payloads, carried); + let keyframes: Vec<_> = frames.iter().map(|frame| frame.keyframe).collect(); + assert_eq!(keyframes, opens); assert!(frames.windows(2).all(|pair| pair[0].timestamp < pair[1].timestamp)); let row = imported.row(VIDEO); assert_eq!((row.track.as_str(), row.class), (".ts", Class::Video)); - assert_eq!((row.units, row.refused, row.damaged), (8, 0, 0)); + assert_eq!((row.units, row.refused, row.damaged), (6, 1, 0)); let ts = super::export_test::drain_with( super::export_test::open_ts( @@ -176,13 +194,11 @@ async fn mpeg2_video_rides_whole_and_survives_a_round_trip() { let again = Imported::all(&ts); let (_, entry) = again.entry(VIDEO).expect("the export lists the video"); assert_eq!(entry.pes_packet.as_ref().map(|pes| pes.stream_type), Some(0x02)); - let payloads: Vec<_> = again - .frames(VIDEO) - .await - .iter() - .map(|frame| frame.payload.to_vec()) - .collect(); - assert_eq!(payloads, pictures, "the round trip keeps every PES byte"); + let frames = again.frames(VIDEO).await; + let payloads: Vec<_> = frames.iter().map(|frame| frame.payload.to_vec()).collect(); + assert_eq!(payloads, carried, "the round trip keeps every PES byte"); + let keyframes: Vec<_> = frames.iter().map(|frame| frame.keyframe).collect(); + assert_eq!(keyframes, opens, "and its groups"); } /// Each ISO 13818-6 DSM-CC type rides DSM-CC sections, so each is reassembled as sections From 9d66bd9a37ad1646037f258c61cd8c490d78c58b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 22:20:57 -0700 Subject: [PATCH 11/16] fix(mux): send the tables leading a placed PES by its placement Program tables ride one lane in push order, and took the deadline of the unit they lead. A placed PES is due by its decode time, so tables leading an MPEG-2 picture that decodes well after its placement held the lane past the deadline of later tables leading audio, failing the export. They now go out by the PES's placement, ahead of its first byte. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/schedule.rs | 46 ++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index 9dc229e2790..f54ea6efb8f 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -785,9 +785,15 @@ impl Schedule { holds, leads: pid, }; + // They go out ahead of the first byte of a placed unit, which a decode time well past + // its placement would otherwise let them hold the lane until, past later tables' due. + let lead = Timing { + by: place.map_or(timing.by, |(place, _)| place.min(timing.by)), + ..timing + }; // The unit they lead keeps its own start. let last = self.last; - self.enqueue(pmt, timing, tables.packets, Some(program)); + self.enqueue(pmt, lead, tables.packets, Some(program)); self.last = last; } if packets.is_empty() { @@ -1634,6 +1640,44 @@ mod tests { assert_eq!(sent, 200); } + /// Tables leading a placed picture that decodes a second out go out by its placement, so + /// the tables leading a unit due sooner, which ride the same lane after them, are not held + /// past that unit's deadline. + #[test] + fn tables_leading_a_placed_pes_go_out_by_its_placement() { + let mut schedule = Schedule::new(Duration::from_millis(25)); + schedule.set_rate(Some(RATE)); + schedule.set_tables(100); + schedule.set_clock(1); + schedule.set_buffer(1, VIDEO); + let tables = || Tables { + packets: [unit(0, 1), unit(100, 1)].concat(), + clock: 1, + rate: Some(RATE), + added: Vec::new(), + }; + let mut picture = unit(7, 200); + picture[1] |= 0x40; + schedule.lead(tables()); + schedule + .push_placed(7, (ms(50), ms(1_000)), 0, picture, true, Vec::new()) + .unwrap(); + schedule.lead(tables()); + schedule.push(1, timing(100), unit(1, 30), true).unwrap(); + let slots = drain(&mut schedule); + let last = slots + .iter() + .filter(|(_, pids, _)| pids.contains_key(&1)) + .map(|&(index, ..)| index) + .max(); + assert!( + last.is_some_and(|last| last <= slot(ms(100))), + "the unit due at 100 ms goes out by slot {last:?}" + ); + let tables: usize = slots.iter().filter_map(|(_, pids, _)| pids.get(&100)).sum(); + assert_eq!(tables, 2); + } + /// The pointer byte goes just before placement, so it cannot cost a section another packet. #[test] fn placed_first_byte_accounts_for_the_section_pointer() { From 6f7e9b888e3fc9cfa50b2019a06ca47b2203b4d6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 22:36:23 -0700 Subject: [PATCH 12/16] fix(mux): drop a partial section on a scrambled packet A section route's continuity counter never saw the scrambled packets the import refuses, so after 16 of them the next clear continuation looked contiguous and completed the partial with another section's bytes. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/import.rs | 5 ++++ rs/moq-mux/src/container/ts/routes_test.rs | 29 ++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index bbf24675ff7..daf33da11d1 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -1442,6 +1442,11 @@ impl Import { if let Some(stream) = self.streams.get_mut(&pid) { stream.lost()?; } + // A section's continuity counter never sees the packet, so after 16 the next clear one + // would look contiguous and complete the partial with bytes from another section. + if let Some(section) = self.sections.get_mut(&pid.as_u16()) { + section.reassembler = SectionReassembler::default(); + } *self.refused.entry(pid.as_u16()).or_default() += 1; if self.scrambled.insert(pid.as_u16()) { tracing::warn!( diff --git a/rs/moq-mux/src/container/ts/routes_test.rs b/rs/moq-mux/src/container/ts/routes_test.rs index dd9e97bb70e..e7ef722b5ed 100644 --- a/rs/moq-mux/src/container/ts/routes_test.rs +++ b/rs/moq-mux/src/container/ts/routes_test.rs @@ -235,6 +235,35 @@ async fn dsmcc_types_ride_as_sections() { } } +/// Scrambled packets lose the section they interrupt, even when 16 of them wrap the continuity +/// counter so the clear packet after them looks contiguous with the partial. +#[tokio::test(start_paused = true)] +async fn scrambled_packets_lose_a_partial_section() { + const DSMCC: u16 = 0x110; + let mut mux = Mux::default(); + mux.pat(&[(1, PMT)]).pmt(PMT, 1, CLOCK, &[(0x0b, DSMCC)], 0); + mux.packet(CLOCK, Flags::pcr(0)); + let at = mux.out.len(); + mux.section(DSMCC, &psi::section(0x3b, 1, 0, 0, 0, &[0x11; 300])); + let tail = mux.out.split_off(at + 188); + for cc in 1..=16u8 { + let mut pkt = vec![0x47, (DSMCC >> 8) as u8, DSMCC as u8, 0x90 | (cc & 0x0f)]; + pkt.resize(188, 0xee); + mux.out.extend_from_slice(&pkt); + } + mux.out.extend_from_slice(&tail); + let clear = psi::section(0x3b, 2, 0, 0, 0, &[0x22; 40]); + mux.section(DSMCC, &clear); + mux.packet(CLOCK, Flags::pcr(40 * MS)); + + let imported = Imported::all(&mux.bytes()); + let frames = imported.frames(DSMCC).await; + let payloads: Vec<_> = frames.iter().map(|frame| frame.payload.to_vec()).collect(); + assert_eq!(payloads, [clear]); + let row = imported.row(DSMCC); + assert_eq!((row.units, row.refused, row.damaged), (1, 16, 0)); +} + /// A 0x86 stream without a CUEI registration is refused once and never PES-parsed, while LATM /// AAC, which nothing here decodes, rides whole. #[tokio::test(start_paused = true)] From 28a947e81b3ae8989bf136705eff37ae75af7697 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 00:04:06 -0700 Subject: [PATCH 13/16] test(mux): clock the shared-budget test with audio Main now holds H.264 AUs against the shared staging budget, as this branch does MPEG-2 video, so a video clock PES cannot enter once held sections fill it. AAC stages without a unit, so it still supplies the clock. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/import.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index c92f8ec0487..3bcb477b2c6 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -8408,7 +8408,10 @@ pub(super) mod test { let mut import = super::Import::new(broadcast, catalog.reserve()); import .decode(&synth_pmt( - &[(StreamType::H264, 0x50), (StreamType::Dts8ChannelLosslessAudio, 0x21)], + &[ + (StreamType::AdtsAac, 0x50), + (StreamType::Dts8ChannelLosslessAudio, 0x21), + ], true, )) .unwrap(); From 8c60315cf9fe94c0aba3f4ee2b29017ba33dbd5c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 08:32:00 -0700 Subject: [PATCH 14/16] fix(mux): find an MPEG-2 sequence header split across two PES A video PES whose payload ends in part of a start code waits for the next PES on its stream, or the end of the stream, before its group is decided, so a sequence header straddling the boundary opens a group at the PES it begins in. The T-STD harness also refuses the vbv_delay method and a zero VBV buffer, takes Rbx from the stream rate at High-1440 and High level, and aligns repeated decoded pictures by suffix. Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-mpegts.md | 2 +- rs/moq-mux/src/container/ts/import.rs | 76 ++++++++++++++++++---- rs/moq-mux/src/container/ts/routes_test.rs | 43 ++++++++++++ test/ts/README.md | 11 ++-- test/ts/compliance.py | 27 ++++++-- test/ts/compliance.test.py | 32 +++++++++ test/ts/mpeg2.py | 10 ++- 7 files changed, 174 insertions(+), 27 deletions(-) diff --git a/drafts/draft-lcurley-moq-mpegts.md b/drafts/draft-lcurley-moq-mpegts.md index 0d66e9cdad0..f070598e9c2 100644 --- a/drafts/draft-lcurley-moq-mpegts.md +++ b/drafts/draft-lcurley-moq-mpegts.md @@ -255,7 +255,7 @@ type PesPacket = { Each frame MUST be exactly one PES packet, from its start code through its payload, with a `PES_packet_length` that matches the frame, or zero only where {{mpeg2}} lets a video stream leave it unbounded. The `stream_id`, the PTS and DTS when present, and every other header field ride in the frame, with the PTS and DTS on the catalog clock; a PES without a PTS carries none. The frame's timestamp is the packet's placement: the program clock at its first byte, on the same timeline. -Each frame MUST be a keyframe in its own group, as for `verbatim`, except on MPEG-1 and MPEG-2 video (0x01 and 0x02), whose pictures depend on earlier ones: there a PES carrying a `sequence_header_code` (0x000001B3) MUST be a keyframe opening a group, and the PES up to the next one MUST join it. +Each frame MUST be a keyframe in its own group, as for `verbatim`, except on MPEG-1 and MPEG-2 video (0x01 and 0x02), whose pictures depend on earlier ones: there a PES in which a `sequence_header_code` (0x000001B3) begins, including one whose last bytes the next PES completes, MUST be a keyframe opening a group, and the PES up to the next one MUST join it. A publisher drops the video PES before the first sequence header, and after a lost one until the next, rather than publish a picture without what it references. A consumer MUST refuse a frame that is not one well-formed PES packet. The record is a member beside `verbatim` rather than a `framing` value, so a consumer that predates it ignores the track instead of refusing the catalog. diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 3bcb477b2c6..b38c120979d 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -1118,6 +1118,7 @@ impl Import { if let Some(stream) = self.streams.get_mut(&pid) && let Stream::PesPacket(stream) = &mut *stream { + stream.end(); stream.settle(&self.placement); let (published, dropped) = stream.release(&self.timebase, &self.pts, generation)?; units += published; @@ -1776,6 +1777,11 @@ impl Import { for pid in pids { self.flush(pid)?; } + for stream in self.streams.values_mut() { + if let Stream::PesPacket(stream) = stream { + stream.end(); + } + } self.settle_placement()?; anyhow::ensure!( self.sections.values().all(|section| section.held.is_empty()), @@ -2289,10 +2295,12 @@ fn adjust_splice(section: &mut [u8], offset: Offset) { /// clock. A PES without a PTS keeps none. See [`Import`] for when one is held, dropped, or refuses /// its stream. /// -/// Each PES is a group of its own, except MPEG-1/2 video's: a PES carrying a sequence header opens -/// a group, where a decoder can start, and the pictures after it join that group. Those before the -/// first sequence header, or after a loss until the next, are dropped, since they depend on -/// pictures a subscriber never gets. +/// Each PES is a group of its own, except MPEG-1/2 video's: a PES in which a sequence header +/// begins opens a group, where a decoder can start, and the pictures after it join that group. +/// Those before the first sequence header, or after a loss until the next, are dropped, since +/// they depend on pictures a subscriber never gets. A video PES ending in part of a start code +/// waits for the next PES on the stream, or for the stream to end, to tell whether a sequence +/// header begins in it. struct PesPacketStream { track: crate::container::Producer, /// Held for its `Drop`, which clears this track's catalog entry. @@ -2304,6 +2312,9 @@ struct PesPacketStream { held: VecDeque<(Held, Join)>, /// The stream lost a unit since the last PES held, so the next joins no group. lost: bool, + /// The PES at the back of `held` ends in part of a start code, so whether it opens a group + /// waits for the PES after it; [`Self::release`] stops before it meanwhile. + straddles: bool, } /// Where a whole PES falls in its track's groups. @@ -2363,6 +2374,7 @@ impl PesPacketStream { stream_type, held: VecDeque::new(), lost: false, + straddles: false, }) } @@ -2371,9 +2383,22 @@ impl PesPacketStream { let span = pending.span.expect("a whole PES is spanned at its start"); let join = match mpeg_video(self.stream_type) { false => Join::Opens, - true if sequence_header(&pending.data) => Join::Opens, - true if self.lost => Join::Resyncs, - true => Join::Continues, + true => { + let payload = es_payload(&pending.data); + if std::mem::take(&mut self.straddles) { + let (before, join) = self.held.back_mut().expect("a straddling PES is held"); + if straddled_header(es_payload(&before.payload), payload) { + *join = Join::Opens; + } + } + let join = match () { + _ if memchr::memmem::find(payload, &SEQUENCE_HEADER).is_some() => Join::Opens, + _ if self.lost => Join::Resyncs, + _ => Join::Continues, + }; + self.straddles = join != Join::Opens && partial_start_code(payload); + join + } }; self.lost = false; let held = Held { @@ -2384,9 +2409,16 @@ impl PesPacketStream { self.held.push_back((held, join)); } - /// The unit in progress was lost: an MPEG-1/2 picture after it may reference it. + /// The unit in progress was lost: an MPEG-1/2 picture after it may reference it, and no + /// sequence header continues from the PES before it. fn lost(&mut self) { self.lost = true; + self.straddles = false; + } + + /// No PES follows the last held, so a start code its end began completes nothing. + fn end(&mut self) { + self.straddles = false; } /// The PCR `after` arrived on timebase `generation`: every PES held since the last is @@ -2422,6 +2454,7 @@ impl PesPacketStream { while let Some(timestamp) = self .held .front() + .filter(|_| !(self.straddles && self.held.len() == 1)) .and_then(|(held, _)| held.timestamp(pts, offset, generation)) { let (mut held, join) = self.held.pop_front().expect("a front"); @@ -3061,16 +3094,31 @@ fn mpeg_video(stream_type: u8) -> bool { matches!(stream_type, 0x01 | 0x02) } -/// Whether the whole PES `pes` carries an MPEG-1/2 `sequence_header_code` (ISO 13818-2 6.2.2.1), -/// where a decoder can start. Start codes never occur inside coded data, so any match is one. -fn sequence_header(pes: &[u8]) -> bool { - const SEQUENCE_HEADER: [u8; 4] = [0x00, 0x00, 0x01, 0xb3]; +/// An MPEG-1/2 `sequence_header_code` (ISO 13818-2 6.2.2.1), where a decoder can start. Start +/// codes never occur inside coded data, so any match is one. +const SEQUENCE_HEADER: [u8; 4] = [0x00, 0x00, 0x01, 0xb3]; + +/// The elementary stream bytes of the whole PES `pes`, after its header. +fn es_payload(pes: &[u8]) -> &[u8] { let header = match pes.get(3) { Some(&stream_id) if psi::has_optional_header(stream_id) => 9 + usize::from(pes.get(8).copied().unwrap_or(0)), _ => 6, }; - pes.get(header..) - .is_some_and(|payload| memchr::memmem::find(payload, &SEQUENCE_HEADER).is_some()) + pes.get(header..).unwrap_or_default() +} + +/// Whether `payload` ends in the first one to three bytes of a start code (`00`, `00 00`, or +/// `00 00 01`), which the next PES may complete. +fn partial_start_code(payload: &[u8]) -> bool { + matches!(payload, [.., 0x00] | [.., 0x00, 0x00, 0x01]) +} + +/// Whether a sequence header begins in `before`'s payload and ends in `after`'s, the next PES +/// on the stream. +fn straddled_header(before: &[u8], after: &[u8]) -> bool { + let before = &before[before.len().saturating_sub(SEQUENCE_HEADER.len() - 1)..]; + let after = &after[..after.len().min(SEQUENCE_HEADER.len() - 1)]; + memchr::memmem::find(&[before, after].concat(), &SEQUENCE_HEADER).is_some() } /// One elementary stream's codec importer. diff --git a/rs/moq-mux/src/container/ts/routes_test.rs b/rs/moq-mux/src/container/ts/routes_test.rs index e7ef722b5ed..1ae2ed5b8f1 100644 --- a/rs/moq-mux/src/container/ts/routes_test.rs +++ b/rs/moq-mux/src/container/ts/routes_test.rs @@ -201,6 +201,49 @@ async fn mpeg2_video_rides_whole_and_survives_a_round_trip() { assert_eq!(keyframes, opens, "and its groups"); } +/// A sequence header split across two video PES, at each of its three inner boundaries, opens +/// its group at the PES it begins in, while a picture start code split the same way opens none. +/// The last PES, ending in part of a start code with no PES after it, still goes out at the end +/// of input. +#[tokio::test(start_paused = true)] +async fn mpeg2_sequence_header_split_across_pes_opens_a_group() { + let mut mux = Mux::default(); + mux.pat(&[(1, PMT)]).pmt(PMT, 1, CLOCK, &[(0x02, VIDEO)], 0); + let start = 2_000 * MS; + // Each PES's payload after the picture start code it opens with, its last bytes completing + // in the next PES: a sequence header at `00 00 | 01 b3`, `00 | 00 01 b3`, and `00 00 01 | b3`, + // and a picture start code at `00 00 | 01 00`. + let tails: [(&[u8], &[u8]); 6] = [ + (&[], &[0, 0]), + (&[1, 0xb3], &[0]), + (&[0, 1, 0xb3], &[]), + (&[], &[0, 0, 1]), + (&[0xb3], &[0, 0]), + (&[1, 0], &[0]), + ]; + let mut pictures = Vec::new(); + for (i, (head, tail)) in tails.into_iter().enumerate() { + let ticks = start + i as u64 * 40 * MS; + mux.packet(CLOCK, Flags::pcr(ticks)); + let mut pes = vec![0, 0, 1, 0xe0, 0, 0, 0x80, 0x80, 5]; + pes.extend_from_slice(&pts_field(ticks / 300 + LEAD)); + pes.extend_from_slice(head); + pes.extend_from_slice(&[0, 0, 1, 0x00]); + pes.resize(pes.len() + 200, 0x55); + pes.extend_from_slice(tail); + mux.whole(VIDEO, &pes); + pictures.push(pes); + } + mux.packet(CLOCK, Flags::pcr(start + 6 * 40 * MS)); + + let imported = Imported::all(&mux.bytes()); + let frames = imported.frames(VIDEO).await; + let payloads: Vec<_> = frames.iter().map(|frame| frame.payload.to_vec()).collect(); + assert_eq!(payloads, pictures, "every PES goes out, the last at the end of input"); + let keyframes: Vec<_> = frames.iter().map(|frame| frame.keyframe).collect(); + assert_eq!(keyframes, [true, true, false, true, false, false]); +} + /// Each ISO 13818-6 DSM-CC type rides DSM-CC sections, so each is reassembled as sections /// rather than failing a PES start code. #[tokio::test(start_paused = true)] diff --git a/test/ts/README.md b/test/ts/README.md index 9b99132562a..efdb7afaf97 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -188,10 +188,13 @@ level defaults too. An `AVC_timing_and_HRD_descriptor` or transfer to the HRD's own schedule, which is not modelled, so such a stream is refused. -MPEG-2 video takes Rx = 1.2 Rmax and Rbx = Rmax from its level, EB from the -sequence header's `vbv_buffer_size`, and MB from what the level's VBV leaves over, -as H.222.0 2.4.2.3 specifies for the leak method a variable-rate encoder signals -with `vbv_delay` 0xFFFF. Its picture, GOP or sequence header opens each access unit. +MPEG-2 video takes Rx = 1.2 Rmax from its level, Rbx = Rmax at Low and Main level +and min(1.05 Res, Rmax) at High-1440 and High, EB from the sequence header's +`vbv_buffer_size`, and MB from what the level's VBV leaves over, as H.222.0 2.4.2.3 +specifies for the leak method. A stream whose `STD_descriptor` clears `leak_valid` +while its pictures code a `vbv_delay` other than 0xFFFF takes the `vbv_delay` +method instead, which is not modelled, so it is refused. Its picture, GOP or +sequence header opens each access unit. Opus is graded against ADTS's buffers for the same channel count: the Opus-in-TS draft gives Rx (2 Mb/s for 1-2 channels, matching ADTS) but leaves the buffer size diff --git a/test/ts/compliance.py b/test/ts/compliance.py index 9a074f57885..549928768f5 100755 --- a/test/ts/compliance.py +++ b/test/ts/compliance.py @@ -564,6 +564,7 @@ def read_programs(ts_path: str) -> list[tuple[int, list[Stream]]]: names = {d["#name"] for d in descriptors} names |= {f"registration:{d['format_identifier']}" for d in descriptors if "format_identifier" in d} names |= {"hrd_management_valid" for d in descriptors if d.get("hrd_management_valid") is True} + names |= {"leak_invalid" for d in descriptors if d["#name"] == "STD_descriptor" and d.get("leak_valid") is False} names |= {"temporal_poc_clock" for d in descriptors if d.get("fixed_frame_rate") is True and d.get("temporal_poc") is True} names |= { @@ -672,16 +673,25 @@ def _hrd(sps: dict[str, str], prefix: str, value: str) -> tuple[float, float] | return rate, size -def mpeg2_params(es: bytes) -> Params: +def mpeg2_params(es: bytes, descriptors: set[str]) -> Params: """H.222.0 2.4.2.3 buffers for H.262 video from its first sequence header and extension. - Graded by the leak method (Rbx = Rmax), which H.222.0 applies when vbv_delay is 0xFFFF, as - a variable-rate encoder codes it. + Graded by the leak method, which H.222.0 applies unless an STD_descriptor clears leak_valid + and the pictures code a vbv_delay other than 0xFFFF; that vbv_delay method is refused. + Rbx is Rmax at Low and Main level, and min(1.05 Res, Rmax) at High-1440 and High. """ head = es.find(b"\x00\x00\x01\xb3") ext = es.find(b"\x00\x00\x01\xb5", head) + picture = es.find(b"\x00\x00\x01\x00", head) if head < 0 or ext < 0 or len(es) < ext + 9 or es[ext + 4] >> 4 != 1: raise Refused("no sequence header and sequence extension, which an H.262 stream starts with") + if picture < 0 or len(es) < picture + 8: + raise Refused("no picture header after the sequence header") + # vbv_delay: 16 bits after the 10-bit temporal_reference and 3-bit picture_coding_type. H.262 + # codes it 0xFFFF in every picture of a sequence or in none, so the first decides. + vbv_delay = (es[picture + 5] & 0x07) << 13 | es[picture + 6] << 5 | es[picture + 7] >> 3 + if "leak_invalid" in descriptors and vbv_delay != 0xFFFF: + raise Refused("an STD_descriptor selects the vbv_delay method, which this model does not implement") # vbv_buffer_size_value: 10 bits after bit_rate_value and its marker; the extension's 8 bits # follow its bit_rate_extension and marker. vbv_low = (es[head + 10] & 0x1F) << 5 | es[head + 11] >> 3 @@ -693,6 +703,13 @@ def mpeg2_params(es: bytes) -> Params: raise Refused(f"H.262 profile_and_level_indication {indication:#04x} has no Table 8-13 entry") rmax, vbv_max = MPEG2_LEVELS[level] vbv = (vbv_high << 10 | vbv_low) * 16 * 1024 + if not vbv: + raise Refused("H.262 vbv_buffer_size 0 sizes no EB") + # bit_rate_value's 18 bits after frame_rate_code, and the extension's 12-bit + # bit_rate_extension above them, in units of 400 bit/s. + bit_rate = es[head + 8] << 10 | es[head + 9] << 2 | es[head + 10] >> 6 + bit_rate |= ((es[ext + 6] & 0x1F) << 7 | es[ext + 7] >> 1) << 18 + rbx = rmax if level in (10, 8) else min(1.05 * bit_rate * 400, rmax) overhead = (0.004 + 1 / 750) * max(rmax, 2_000_000) return Params( label=f"H.262 {MPEG2_LEVEL_NAMES[level]} level, vbv {vbv / 1e3:.0f} kbit", @@ -700,7 +717,7 @@ def mpeg2_params(es: bytes) -> Params: rx=1.2 * rmax, mb=(overhead + vbv_max - vbv) / 8, eb=vbv / 8, - rbx=rmax, + rbx=rbx, ) @@ -888,7 +905,7 @@ def stream_params(kind: str, stream: Stream) -> tuple[Params, object]: if kind == "avc": return avc_params(stream.sps), None if kind == "mpeg2-video": - return mpeg2_params(bytes(es)), None + return mpeg2_params(bytes(es), stream.descriptors), None if kind == "hevc": return hevc_params(stream.sps), None if kind == "teletext": diff --git a/test/ts/compliance.test.py b/test/ts/compliance.test.py index d0052b78ec5..d3ba2e47080 100644 --- a/test/ts/compliance.test.py +++ b/test/ts/compliance.test.py @@ -62,6 +62,38 @@ def tables(*pmts: tuple[int, list[int]]) -> bytes: return b"".join(out) +def h262(indication: int, rate: int, vbv: int, vbv_delay: int) -> bytes: + """A sequence header, sequence extension and picture header: profile_and_level_indication, + bit_rate in bit/s, vbv_buffer_size in 16 kbit units, and the picture's vbv_delay.""" + value = rate // 400 + head = b"\x00\x00\x01\xb3\x2d\x02\x40\x23" + head += ((value & 0x3FFFF) << 14 | 1 << 13 | (vbv & 0x3FF) << 3).to_bytes(4, "big") + ext = b"\x00\x00\x01\xb5" + ext += (1 << 44 | indication << 36 | 1 << 35 | 1 << 33 | (value >> 18) << 17 | 1 << 16 | (vbv >> 10) << 8).to_bytes(6, "big") + picture = b"\x00\x00\x01\x00" + (1 << 19 | vbv_delay << 3).to_bytes(4, "big") + return head + ext + picture + + +class Mpeg2Params(unittest.TestCase): + def test_rbx_follows_the_level(self): + # Main Profile at Main Level leaks at Rmax; at High Level, at 1.05 times the stream's rate. + for indication, rbx in ((0x48, 15_000_000), (0x44, 21_000_000)): + with self.subTest(indication=indication): + params = compliance.mpeg2_params(h262(indication, 20_000_000, 112, 0xFFFF), set()) + self.assertAlmostEqual(params.rbx, rbx) + + def test_zero_vbv_buffer_is_refused(self): + with self.assertRaises(compliance.Refused): + compliance.mpeg2_params(h262(0x48, 4_000_000, 0, 0xFFFF), set()) + + def test_vbv_delay_method_is_refused(self): + es = h262(0x48, 4_000_000, 112, 0x1000) + compliance.mpeg2_params(es, set()) + compliance.mpeg2_params(h262(0x48, 4_000_000, 112, 0xFFFF), {"leak_invalid"}) + with self.assertRaises(compliance.Refused): + compliance.mpeg2_params(es, {"leak_invalid"}) + + class PmtVersions(unittest.TestCase): def check(self, *pmts: tuple[int, list[int]]) -> compliance.Check: with tempfile.NamedTemporaryFile(suffix=".ts") as handle: diff --git a/test/ts/mpeg2.py b/test/ts/mpeg2.py index c9db77bca1e..d22503ffef5 100644 --- a/test/ts/mpeg2.py +++ b/test/ts/mpeg2.py @@ -52,15 +52,19 @@ def grade(source, capture, gop, rate): if errors: raise SystemExit(f"error: the source does not decode cleanly: {errors[:3]}") theirs, errors = pictures(capture) - index = {digest: k for k, digest in enumerate(ours)} + known = set(ours) # The pictures of the joined GOP before its next I picture may not be the source's. - lead = next((j for j, digest in enumerate(theirs) if digest in index), None) + lead = next((j for j, digest in enumerate(theirs) if digest in known), None) if lead is None: raise SystemExit("error: no decoded picture of the capture is the source's") if lead > gop: raise SystemExit(f"error: the first {lead} pictures are not the source's, past one {gop}-picture GOP") - first = index[theirs[lead]] run = theirs[lead:] + # A picture can repeat in the source, so align the run where it matches, preferring the + # alignment that ends on the source's last picture. + starts = [k for k, digest in enumerate(ours) if digest == run[0]] + matches = [k for k in starts if ours[k : k + len(run)] == run] + first = next((k for k in matches if k + len(run) == len(ours)), matches[0] if matches else starts[0]) if run != ours[first : first + len(run)]: miss = next(j for j, digest in enumerate(run) if first + j >= len(ours) or ours[first + j] != digest) raise SystemExit(f"error: decoded picture {lead + miss} is not the source's picture {first + miss}") From 89ef9692713f2b4e6489eb5e4e9bfcdc510e44cb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 09:48:46 -0700 Subject: [PATCH 15/16] fix(mux): settle a held MPEG-2 PES at a clock break A whole video PES ending in part of a start code waited for the next PES, which a clock break never delivers on the old timeline, so the break refused the import. The break now ends that lookahead, as end of input and a PMT revision already do. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/import.rs | 19 ++++++++--- rs/moq-mux/src/container/ts/routes_test.rs | 37 ++++++++++++++++------ 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index b38c120979d..8cf494649ca 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -1570,7 +1570,11 @@ impl Import { for pid in self.streams.keys().copied().collect::>() { self.salvage(pid)?; } - // Whole PES the old clock began go out on its timeline, ahead of the marker. + // Whole PES the old clock began go out on its timeline, ahead of the marker, so none waits + // for a PES on the new one to complete a start code. + for stream in self.streams.values_mut() { + stream.end(); + } self.settle_placement()?; anyhow::ensure!( self.sections.values().all(|section| section.held.is_empty()), @@ -1778,9 +1782,7 @@ impl Import { self.flush(pid)?; } for stream in self.streams.values_mut() { - if let Stream::PesPacket(stream) = stream { - stream.end(); - } + stream.end(); } self.settle_placement()?; anyhow::ensure!( @@ -2416,7 +2418,7 @@ impl PesPacketStream { self.straddles = false; } - /// No PES follows the last held, so a start code its end began completes nothing. + /// No PES follows the last held on its clock, so a start code its end began completes nothing. fn end(&mut self) { self.straddles = false; } @@ -3245,6 +3247,13 @@ impl Stream { Ok(()) } + /// No PES follows the last on this clock or input: see [`PesPacketStream::end`]. + fn end(&mut self) { + if let Stream::PesPacket(stream) = self { + stream.end(); + } + } + /// The program clock restarted: mark the break on this track. fn discontinuity(&mut self, published: bool) -> anyhow::Result<()> { match self { diff --git a/rs/moq-mux/src/container/ts/routes_test.rs b/rs/moq-mux/src/container/ts/routes_test.rs index 1ae2ed5b8f1..7ee3197b082 100644 --- a/rs/moq-mux/src/container/ts/routes_test.rs +++ b/rs/moq-mux/src/container/ts/routes_test.rs @@ -203,8 +203,8 @@ async fn mpeg2_video_rides_whole_and_survives_a_round_trip() { /// A sequence header split across two video PES, at each of its three inner boundaries, opens /// its group at the PES it begins in, while a picture start code split the same way opens none. -/// The last PES, ending in part of a start code with no PES after it, still goes out at the end -/// of input. +/// A whole PES ending in part of a start code with no whole PES after it on its clock still goes +/// out, at a clock break and at the end of input. #[tokio::test(start_paused = true)] async fn mpeg2_sequence_header_split_across_pes_opens_a_group() { let mut mux = Mux::default(); @@ -212,19 +212,29 @@ async fn mpeg2_sequence_header_split_across_pes_opens_a_group() { let start = 2_000 * MS; // Each PES's payload after the picture start code it opens with, its last bytes completing // in the next PES: a sequence header at `00 00 | 01 b3`, `00 | 00 01 b3`, and `00 00 01 | b3`, - // and a picture start code at `00 00 | 01 00`. - let tails: [(&[u8], &[u8]); 6] = [ + // and a picture start code at `00 00 | 01 00`. A clock break cuts the sixth, its length + // unbounded, so the fifth is the last whole PES on the old clock. The seventh, carrying a + // whole sequence header, ends the input. + let tails: [(&[u8], &[u8]); 7] = [ (&[], &[0, 0]), (&[1, 0xb3], &[0]), (&[0, 1, 0xb3], &[]), (&[], &[0, 0, 1]), (&[0xb3], &[0, 0]), (&[1, 0], &[0]), + (&[0, 0, 1, 0xb3], &[0, 0]), ]; + let restart = 9_000 * MS; let mut pictures = Vec::new(); for (i, (head, tail)) in tails.into_iter().enumerate() { - let ticks = start + i as u64 * 40 * MS; - mux.packet(CLOCK, Flags::pcr(ticks)); + let ticks = match i { + 6 => restart, + _ => start + i as u64 * 40 * MS, + }; + match i { + 6 => mux.packet(CLOCK, Flags::pcr(ticks).discontinuity()), + _ => mux.packet(CLOCK, Flags::pcr(ticks)), + }; let mut pes = vec![0, 0, 1, 0xe0, 0, 0, 0x80, 0x80, 5]; pes.extend_from_slice(&pts_field(ticks / 300 + LEAD)); pes.extend_from_slice(head); @@ -234,14 +244,21 @@ async fn mpeg2_sequence_header_split_across_pes_opens_a_group() { mux.whole(VIDEO, &pes); pictures.push(pes); } - mux.packet(CLOCK, Flags::pcr(start + 6 * 40 * MS)); + mux.packet(CLOCK, Flags::pcr(restart + 40 * MS)); let imported = Imported::all(&mux.bytes()); let frames = imported.frames(VIDEO).await; - let payloads: Vec<_> = frames.iter().map(|frame| frame.payload.to_vec()).collect(); - assert_eq!(payloads, pictures, "every PES goes out, the last at the end of input"); + // The PES on the new clock has its PTS rebased, so compare around it. + let unstamped = |pes: &[u8]| [&pes[..9], &pes[14..]].concat(); + let payloads: Vec<_> = frames.iter().map(|frame| unstamped(&frame.payload)).collect(); + pictures.remove(5); + let pictures: Vec<_> = pictures.iter().map(|pes| unstamped(pes)).collect(); + assert_eq!( + payloads, pictures, + "every whole PES goes out, at the clock break and the end of input" + ); let keyframes: Vec<_> = frames.iter().map(|frame| frame.keyframe).collect(); - assert_eq!(keyframes, [true, true, false, true, false, false]); + assert_eq!(keyframes, [true, true, false, true, false, true]); } /// Each ISO 13818-6 DSM-CC type rides DSM-CC sections, so each is reassembled as sections From aa1fc67b0b9ce1c74f52845de8b2da229db22127 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 17:13:14 -0700 Subject: [PATCH 16/16] fix(mux): resume whole MPEG-1/2 video at a sequence header after a late drop Export judged every pesPacket frame decodable on its own, so after dropping a late MPEG-2 picture the jitter buffer resumed at the next picture, which references the dropped one. A frame of either video class now resumes only at a keyframe. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export.rs | 5 +- rs/moq-mux/src/container/ts/export_test.rs | 91 ++++++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 25e9c7678e1..71767f63e21 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -706,6 +706,9 @@ impl Track { if video && frame.keyframe { self.keyframe = Some(frame.timestamp); } + // Video, MPEG-1/2 carried whole among it, resumes only at a keyframe, since its other + // pictures reference earlier ones. + let sync = frame.keyframe || self.kind.class() != super::stats::Class::Video; // Replaying anchors on the first frame, and the tracks arrive together rather than each // sent ahead by its send-ahead: judged as arriving that much earlier, as a live source // sends it, every track's first frames are on the clock whichever anchors it. @@ -724,7 +727,7 @@ impl Track { restart, skip, cut, - sync: frame.keyframe || !video, + sync, leading, item: Queued { frame, diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 4b38028c5e5..c282176cd5a 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -1079,6 +1079,97 @@ async fn export_trims_leading_pictures_after_a_late_drop() { assert_eq!(export.dropped(), 3, "the late picture and the two it orphaned"); } +/// MPEG-2 video carried whole resumes after a late picture at its next sequence header, not +/// at the next picture, which references the dropped one. +#[tokio::test(start_paused = true)] +async fn export_resumes_whole_mpeg2_video_at_a_sequence_header() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut aac = AudioConfig::new(AAC { profile: 2 }, 48_000, 2); + aac.container = Container::Legacy; + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + catalog.modify().unwrap().audio.renditions.insert("audio".into(), aac); + let mut audio = Producer::new(track, HangContainer::Legacy(Kind::Audio)); + let track = broadcast + .create_track("video.ts", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let mut entry = tscat::Track::new(PLACED_PID); + entry.pes_packet = Some(tscat::PesPacket::new(0x02)); + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .insert("video.ts".into(), entry); + let mut video = Producer::new(track, HangContainer::Legacy(Kind::Data)); + let mut export = open_ts( + crate::source::announced(&consumer).await, + crate::catalog::CatalogFormat::Hang, + ) + .await + .unwrap() + .with_delay(Duration::from_millis(500)); + + // (arrival, PTS) in milliseconds, and whether a sequence header opens a group. The 120 ms + // picture stalls past its deadline; the two pictures after it still reference it. + let pictures = [ + (0, 0, true), + (40, 40, false), + (80, 80, false), + (1000, 120, false), + (2000, 2000, false), + (2040, 2040, false), + (2080, 2080, true), + (2120, 2120, false), + ]; + let start = tokio::time::Instant::now(); + let (mut out, mut sample) = (Vec::new(), 0); + for (index, (at, pts, sequence)) in pictures.into_iter().enumerate() { + // The audio arrives on time throughout, so only the video is late. + while sample * AUDIO_US <= at * 1_000 { + tokio::time::sleep_until(start + Duration::from_micros(sample * AUDIO_US)).await; + out.extend(poll_frames(&mut export)); + audio + .write(Frame { + timestamp: Timestamp::from_micros(sample * AUDIO_US).unwrap(), + duration: None, + payload: Bytes::from(vec![sample as u8; 180]), + keyframe: sample.is_multiple_of(AUDIO_GROUP), + }) + .unwrap(); + sample += 1; + } + tokio::time::sleep_until(start + Duration::from_millis(at)).await; + out.extend(poll_frames(&mut export)); + let mut payload = if sequence { vec![0, 0, 1, 0xb3] } else { Vec::new() }; + payload.extend_from_slice(&[0, 0, 1, 0x00, index as u8]); + video + .write(Frame { + timestamp: Timestamp::from_millis(pts).unwrap(), + duration: None, + keyframe: sequence, + payload: Bytes::from(pes_with(0xe0, 0x80, 0x80, &pes_clock(2, pts * 90), 0, &payload)), + }) + .unwrap(); + out.extend(poll_frames(&mut export)); + } + audio.finish().unwrap(); + video.finish().unwrap(); + out.extend(drain_frames(&mut export).await); + + // Each picture's last byte is its index. + let sent: Vec = placed_out(&out).iter().map(|pes| *pes.bytes.last().unwrap()).collect(); + assert_eq!(sent, [0, 1, 2, 6, 7]); +} + /// `(PTS, DTS)` of every video PES start, in transport (decode) order. Read raw, so a capture /// that starts before its PMT still counts. fn video_pes_timing(ts: &[u8]) -> Vec<(u64, u64)> {