feat(mux): import fMP4 TTML captions onto the catalog clock

doc/lib/rs/moq-mux.md

@@ -13,7 +13,7 @@ back. This is what `moq import` / `export` and the gateways are built on.
 
 | Format | Import | Export | Notes |
 | --- | --- | --- | --- |
-| fMP4 / CMAF | yes | yes | Audio/video passthrough as `cmaf` or repackaged as `legacy`; `wvtt` subtitles import as timed WebVTT text, preserving source language and label when present. |
+| fMP4 / CMAF | yes | yes | Audio/video passthrough as `cmaf` or repackaged as `legacy`; `wvtt` subtitles import as timed WebVTT text and `stpp` TTML (IMSC timing profile) as `ttml` documents on the broadcast clock, preserving source language and label when present; other subtitle codecs are skipped. |
 | MPEG-TS | yes | yes | H.264/H.265; AAC, MP2, AC-3, E-AC-3, Opus up to 7.1; SCTE-35 and subtitle PIDs as tracks; service tables round-trip. |
 | FLV / RTMP | yes | yes | Legacy H.264 + AAC + MP3, plus enhanced-RTMP HEVC, AV1, VP9, Opus, AC-3, E-AC-3, and multitrack. |
 | Matroska / WebM | yes | yes | Timed SRT and WebVTT subtitles import as WebVTT text renditions, preserving source language and label when present; other subtitle codecs are skipped. |

drafts/draft-lcurley-moq-hang.md

@@ -359,7 +359,7 @@ The `renditions` field maps rendition IDs to text configurations, typically one
 The `format` field selects the cue serialization, and tells a consumer how to parse each frame's payload:
 
 - `vtt`: WebVTT ([W3C WebVTT](https://www.w3.org/TR/webvtt1/)). Each payload is a self-contained `WEBVTT` segment whose cues carry absolute timing. A cue's embedded start time MUST match its enclosing frame timestamp.
-- `ttml`: TTML / IMSC1 ([W3C IMSC](https://www.w3.org/TR/ttml-imsc1.1/)) fragment (XML) with absolute timing. A cue's embedded start time MUST match its enclosing frame timestamp.
+- `ttml`: TTML / IMSC1 ([W3C IMSC](https://www.w3.org/TR/ttml-imsc1.1/)). Each payload is one complete TTML document in the `media` time base with absolute timing, clipped to the frame. Its `body` begin MUST match the enclosing frame timestamp, and its `body` end bounds everything the document presents.
 - `utf8`: raw UTF-8 text with no embedded timing or styling. The cue is shown from its frame timestamp until the next cue; an empty payload clears it.
 
 A consumer MUST ignore a rendition whose `format` it does not recognize.

js/hang/src/catalog/text.ts

@@ -6,7 +6,8 @@ import { RelativeBroadcastSchema } from "./path";
 /**
  * The cue serialization format of a text track, mirroring the Rust `TextFormat`.
  *
- * Known values are `"vtt"` (WebVTT), `"ttml"` (TTML/IMSC1), and `"utf8"` (raw text). Any other
+ * Known values are `"vtt"` (WebVTT), `"ttml"` (one complete TTML/IMSC1 document per frame, clipped
+ * to the frame, whose `body` begins at the frame timestamp), and `"utf8"` (raw text). Any other
  * string is an unrecognized format the consumer should ignore. The frame timestamp always carries
  * the cue's start time, so a relay never parses the payload.
  */

quest/m2/README.md

@@ -77,7 +77,6 @@ ranked just below deterministic TS legs, which it requires.
 - [Android capture](/quest/m2/mobile-capture-android.md) - Rust captures through NDK/JNI on Android, reusing the existing codecs and setting the catalog rotation
 - [Time stretch](/quest/m2/watch-audio-time-stretch.md) - js/watch: the audio ring converges by time-stretching instead of skipping or going silent
 - [Native audio quality](/quest/m2/audio-quality-native.md) - the browser lane's profiles, budgets, and metric schema run against `moq play` on a dummy device
-- [TTML caption import](/quest/m2/captions-ttml.md) - fMP4 `stpp` tracks within a supported TTML/IMSC profile import as `ttml` renditions on the shared clock instead of being skipped
 - [Bindings size profile](/quest/m2/ffi-size-profile.md) - a benchmark decides whether the moq-ffi builds ship at opt-level "s", which halves the dylib
 - [Zero-copy byte returns](/quest/m2/ffi-bytes-return.md) - returned payloads reach the host without a copy, if the payload numbers justify it
 - [Namespaced external types](/quest/m2/uniffi-namespaces.md) - the UniFFI fork lets two crates export the same type name, reopening the per-layer crate split

rs/hang/src/catalog/text/format.rs

@@ -18,8 +18,8 @@ pub enum TextFormat {
 	#[default]
 	Vtt,
 
-	/// TTML / IMSC1 ([W3C IMSC]) fragment (XML). Each frame's payload is a document fragment with
-	/// absolute timing.
+	/// TTML / IMSC1 ([W3C IMSC]). Each frame's payload is one complete TTML document with absolute
+	/// timing, clipped to the frame: its `body` begins at the frame timestamp.
 	///
 	/// [W3C IMSC]: https://www.w3.org/TR/ttml-imsc1.1/
 	#[display("ttml")]

rs/moq-mux/Cargo.toml

@@ -29,6 +29,7 @@ moq-net = { workspace = true }
 mp4-atom = { version = "0.16.2", features = ["tokio", "bytes", "serde"] }
 mpeg2ts = "0.6.0"
 num_enum = { workspace = true }
+roxmltree = "0.21.1"
 scuffle-av1 = { version = "0.1.4" }
 scuffle-h265 = { version = "0.2.2" }
 serde = { workspace = true }

rs/moq-mux/src/container/fmp4/import.rs

@@ -27,7 +27,8 @@ use crate::catalog::Estimator;
 /// - Opus
 /// - FLAC
 ///
-/// **Text:** WebVTT (`wvtt`), reconstructed as timed text; unsupported subtitle codecs are skipped.
+/// **Text:** WebVTT (`wvtt`), reconstructed as timed text, and TTML (`stpp`) documents moved onto the
+/// catalog clock; unsupported subtitle codecs, including other XML namespaces, are skipped.
 pub struct Import<E: crate::catalog::hang::CatalogExt = ()> {
 	/// The broadcast being produced
 	broadcast: moq_net::broadcast::Producer,
@@ -370,15 +371,22 @@ impl<E: crate::catalog::hang::CatalogExt> Import<E> {
 	}
 
 	fn init_text(&mut self, trak: &Trak, moov: &Moov, reserved: &crate::catalog::Reserved<E>) -> Result<bool> {
-		let [mp4_atom::Codec::Wvtt(codec)] = trak.mdia.minf.stbl.stsd.codecs.as_slice() else {
-			return Ok(false);
+		let (format, mut config) = match trak.mdia.minf.stbl.stsd.codecs.as_slice() {
+			[mp4_atom::Codec::Wvtt(codec)] => {
+				if codec.plaintext.data_reference_index != 1 {
+					return Err(Error::InvalidSubtitle("external subtitle data reference").into());
+				}
+				let header = crate::container::text::header(&codec.config.config).map_err(Error::InvalidSubtitle)?;
+				let mut config = hang::catalog::TextConfig::new(hang::catalog::TextFormat::Vtt);
+				config.role = crate::container::text::role(&header);
+				(super::text::Format::Vtt { header }, config)
+			}
+			[mp4_atom::Codec::Unknown(kind, body)] if kind.as_ref() == b"stpp" && super::text::stpp(body)? => (
+				super::text::Format::Ttml,
+				hang::catalog::TextConfig::new(hang::catalog::TextFormat::Ttml),
+			),
+			_ => return Ok(false),
 		};
-		if codec.plaintext.data_reference_index != 1 {
-			return Err(Error::InvalidSubtitle("external subtitle data reference").into());
-		}
-		let header = crate::container::text::header(&codec.config.config).map_err(Error::InvalidSubtitle)?;
-		let mut config = hang::catalog::TextConfig::new(hang::catalog::TextFormat::Vtt);
-		config.role = crate::container::text::role(&header);
 		config.lang = Some(trak.mdia.mdhd.language.clone());
 		// The picker label is the HandlerBox name; `vlab` is an opaque source identifier.
 		config.label = (!trak.mdia.hdlr.name.is_empty()).then(|| trak.mdia.hdlr.name.clone());
@@ -392,7 +400,7 @@ impl<E: crate::catalog::hang::CatalogExt> Import<E> {
 			trak.tkhd.track_id,
 			super::text::Track {
 				producer,
-				header,
+				format,
 				defaults: sample::Defaults::from_moov(moov, trak.tkhd.track_id),
 				timescale: moq_net::Timescale::new(trak.mdia.mdhd.timescale as u64)?,
 				end_decode_time: None,

rs/moq-mux/src/container/fmp4/import_test.rs

@@ -2087,3 +2087,148 @@ fn malformed_webvtt_origins_and_comments_are_refused() {
 		assert!(importer.decode(&webvtt_input(payload, 1250)).is_err());
 	}
 }
+
+/// An `stpp` init segment. mp4-atom cannot encode an `stpp` entry, so encode a `metx` entry, whose
+/// strings line up with `stpp`'s namespace, schema location and auxiliary MIME types, and rename it.
+fn ttml_init(namespace: &str, mime: &str, timescale: u32) -> Vec<u8> {
+	let (_, moov) = decode_init(&webvtt_init("WEBVTT"));
+	let mut trak = moov.trak[0].clone();
+	trak.mdia.hdlr.handler = b"subt".into();
+	trak.mdia.mdhd.timescale = timescale;
+	trak.mdia.minf.stbl.stsd.codecs = vec![mp4_atom::Codec::Metx(mp4_atom::Metx {
+		metadata: mp4_atom::MetaData {
+			data_reference_index: 1,
+		},
+		content_encoding: namespace.into(),
+		namespace: String::new(),
+		schema_location: mime.into(),
+		btrt: None,
+	})];
+	let mut init = audio_init(trak);
+	let at = init.windows(4).position(|kind| kind == b"metx").unwrap();
+	init[at..at + 4].copy_from_slice(b"stpp");
+	init
+}
+
+const TTML: &str = r#"<tt xmlns="http://www.w3.org/ns/ttml" xml:lang="en"><body><div begin="00:01:41"><p begin="0.5s" end="2s">Hello</p></div></body></tt>"#;
+
+/// Import one TTML sample lasting 4 s from source time 100 s, returning its rendition and frame.
+async fn imported_ttml(
+	timebase: crate::catalog::Timebase,
+	broadcast: moq_net::broadcast::Producer,
+	catalog: &crate::catalog::Producer,
+	consumer: &moq_net::broadcast::Consumer,
+) -> (hang::catalog::TextConfig, crate::container::Frame) {
+	let mut input = ttml_init("http://www.w3.org/ns/ttml", "", 1000);
+	input.extend(webvtt_fragment(100_000, &[TTML.as_bytes().to_vec()], 4000));
+	let mut importer = super::Import::new(broadcast, timebase.reserve());
+	importer.decode(&input).unwrap();
+	importer.finish().unwrap();
+	let snapshot = catalog.snapshot();
+	let (name, config) = snapshot.text.renditions.iter().next().unwrap();
+	let track = consumer.track(name).unwrap().subscribe(None).await.unwrap();
+	let mut reader = crate::container::Consumer::new(
+		track,
+		crate::catalog::hang::Container::Legacy(crate::container::Kind::Data),
+	);
+	(config.clone(), reader.read_frame().await.unwrap().unwrap())
+}
+
+#[tokio::test(start_paused = true)]
+async fn ttml_samples_become_timed_text_with_source_metadata() {
+	let mut broadcast = moq_net::broadcast::Info::new().produce();
+	let consumer = broadcast.consume();
+	let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap();
+	let (config, frame) = imported_ttml(catalog.timebase(), broadcast, &catalog, &consumer).await;
+	assert_eq!(config.format, hang::catalog::TextFormat::Ttml);
+	assert_eq!(config.lang.as_deref(), Some("eng"));
+	assert_eq!(config.label.as_deref(), Some("English captions"));
+	assert_eq!(frame.timestamp.as_micros(), 100_000_000);
+	assert_eq!(
+		&frame.payload[..],
+		br#"<tt xmlns="http://www.w3.org/ns/ttml" xml:lang="en"><body begin="00:01:40" end="00:01:44"><div begin="00:00:01"><p begin="0.5s" end="2s">Hello</p></div></body></tt>"#
+	);
+}
+
+/// The source joins a catalog clock another input already fixed, so its TTML times move with it.
+#[tokio::test(start_paused = true)]
+async fn ttml_follows_a_shifted_source_clock() {
+	let wall = std::time::SystemTime::now();
+	let mut broadcast = moq_net::broadcast::Info::new().produce();
+	let consumer = broadcast.consume();
+	let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap();
+	catalog
+		.timebase()
+		.place(moq_net::Timestamp::from_secs(10).unwrap(), wall)
+		.unwrap();
+	let timebase = catalog.timebase();
+	timebase
+		.place(moq_net::Timestamp::from_secs(100).unwrap(), wall)
+		.unwrap();
+	let (_, frame) = imported_ttml(timebase, broadcast, &catalog, &consumer).await;
+	assert_eq!(frame.timestamp.as_micros(), 10_000_000);
+	assert_eq!(
+		&frame.payload[..],
+		br#"<tt xmlns="http://www.w3.org/ns/ttml" xml:lang="en"><body begin="00:00:10" end="00:00:14"><div begin="00:00:01"><p begin="0.5s" end="2s">Hello</p></div></body></tt>"#
+	);
+}
+
+/// An `stpp` entry naming another XML namespace, or carrying image subsamples, skips only its track.
+#[test]
+fn unsupported_stpp_tracks_are_skipped() {
+	for (namespace, mime) in [("urn:example:not-ttml", ""), ("http://www.w3.org/ns/ttml", "image/png")] {
+		let mut input = ttml_init(namespace, mime, 1000);
+		input.extend(webvtt_fragment(0, &[TTML.as_bytes().to_vec()], 1000));
+		let mut broadcast = moq_net::broadcast::Info::new().produce();
+		let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap();
+		let mut importer = super::Import::new(broadcast, catalog.reserve());
+		importer.decode(&input).unwrap();
+		importer.finish().unwrap();
+		assert!(catalog.snapshot().text.renditions.is_empty(), "{namespace} {mime}");
+	}
+}
+
+/// A document the profile refuses fails the import instead of publishing as TTML.
+#[test]
+fn ttml_documents_outside_the_profile_are_refused() {
+	for document in [
+		r#"<tt xmlns="urn:example:not-ttml"/>"#,
+		r#"<tt xmlns="http://www.w3.org/ns/ttml" xmlns:ttp="http://www.w3.org/ns/ttml#parameter" ttp:timeBase="smpte"/>"#,
+	] {
+		let mut input = ttml_init("http://www.w3.org/ns/ttml", "", 1000);
+		input.extend(webvtt_fragment(0, &[document.as_bytes().to_vec()], 1000));
+		let mut broadcast = moq_net::broadcast::Info::new().produce();
+		let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap();
+		let mut importer = super::Import::new(broadcast, catalog.reserve());
+		assert!(importer.decode(&input).is_err(), "{document}");
+	}
+}
+
+/// A 90 kHz sample start is not a whole microsecond; the body still begins at the serialized
+/// frame timestamp.
+#[tokio::test(start_paused = true)]
+async fn ttml_body_begins_at_the_serialized_frame_timestamp() {
+	let mut input = ttml_init("http://www.w3.org/ns/ttml", "", 90_000);
+	input.extend(webvtt_fragment(9_000_001, &[TTML.as_bytes().to_vec()], 360_000));
+
+	let mut broadcast = moq_net::broadcast::Info::new().produce();
+	let consumer = broadcast.consume();
+	let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap();
+	let mut importer = super::Import::new(broadcast, catalog.reserve());
+	importer.decode(&input).unwrap();
+	importer.finish().unwrap();
+	let snapshot = catalog.snapshot();
+	let name = snapshot.text.renditions.keys().next().unwrap();
+	let track = consumer.track(name).unwrap().subscribe(None).await.unwrap();
+	let mut reader = crate::container::Consumer::new(
+		track,
+		crate::catalog::hang::Container::Legacy(crate::container::Kind::Data),
+	);
+	let frame = reader.read_frame().await.unwrap().unwrap();
+	assert_eq!(frame.timestamp.as_micros(), 100_000_011);
+	let payload = std::str::from_utf8(&frame.payload).unwrap();
+	assert!(
+		payload.contains(r#"<body begin="00:01:40.000011" end="00:01:44.000011">"#),
+		"{payload}"
+	);
+}