From 27bfb882ccd0b5814d05761e18268b2d7f15e29e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Jan 2026 12:27:46 -0300 Subject: [PATCH] Rename minBuffer to jitter --- js/hang/src/catalog/audio.ts | 11 +++++++---- js/hang/src/catalog/video.ts | 13 +++++++++---- js/hang/src/util/latency.ts | 28 ++++++++++++++-------------- js/hang/src/watch/element.ts | 4 ++-- js/hang/src/watch/sync.ts | 10 +++++----- rs/hang/examples/video.rs | 2 +- rs/hang/src/catalog/audio/mod.rs | 13 ++++++------- rs/hang/src/catalog/root.rs | 4 ++-- rs/hang/src/catalog/video/mod.rs | 15 ++++++++------- rs/hang/src/import/aac.rs | 2 +- rs/hang/src/import/avc3.rs | 2 +- rs/hang/src/import/fmp4.rs | 32 ++++++++++++++++---------------- rs/hang/src/import/hev1.rs | 2 +- rs/hang/src/import/opus.rs | 2 +- 14 files changed, 74 insertions(+), 66 deletions(-) diff --git a/js/hang/src/catalog/audio.ts b/js/hang/src/catalog/audio.ts index 2711d9c8a..58ab02c85 100644 --- a/js/hang/src/catalog/audio.ts +++ b/js/hang/src/catalog/audio.ts @@ -32,10 +32,13 @@ export const AudioConfigSchema = z.object({ // TODO: Support up to Number.MAX_SAFE_INTEGER bitrate: u53Schema.optional(), - // Minimum buffer size in milliseconds required for smooth playback. - // This represents the minimum time the player should buffer before starting playback. - // The player should add additional buffer on top of this value. - minBuffer: u53Schema.optional(), + // The maximum jitter before the next frame is emitted in milliseconds. + // The player's jitter buffer should be larger than this value. + // If not provided, the player should assume each frame is flushed immediately. + // + // NOTE: The audio "frame" duration depends on the codec, sample rate, etc. + // ex: AAC often uses 1024 samples per frame, so at 44100Hz, this would be 1024/44100 = 23ms + jitter: u53Schema.optional(), }); export const AudioSchema = z diff --git a/js/hang/src/catalog/video.ts b/js/hang/src/catalog/video.ts index 28b942d58..fdd5b62d2 100644 --- a/js/hang/src/catalog/video.ts +++ b/js/hang/src/catalog/video.ts @@ -43,10 +43,15 @@ export const VideoConfigSchema = z.object({ // Default: true optimizeForLatency: z.boolean().optional(), - // Minimum buffer size in milliseconds required for smooth playback. - // This represents the minimum time the player should buffer before starting playback. - // The player should add additional buffer on top of this value. - minBuffer: u53Schema.optional(), + // The maximum jitter before the next frame is emitted in milliseconds. + // The player's jitter buffer should be larger than this value. + // If not provided, the player should assume each frame is flushed immediately. + // + // ex: + // - If each frame is flushed immediately, this would be 1000/fps. + // - If there can be up to 3 b-frames in a row, this would be 3 * 1000/fps. + // - If frames are buffered into 2s segments, this would be 2s. + jitter: u53Schema.optional(), }); // Mirrors VideoDecoderConfig diff --git a/js/hang/src/util/latency.ts b/js/hang/src/util/latency.ts index 16cda5eb6..29d9122b8 100644 --- a/js/hang/src/util/latency.ts +++ b/js/hang/src/util/latency.ts @@ -1,22 +1,22 @@ import type * as Moq from "@moq/lite"; import { Effect, type Getter, Signal } from "@moq/signals"; -type ConfigWithMinBuffer = { minBuffer?: number; framerate?: number }; +type ConfigWithJitter = { jitter?: number; framerate?: number }; export interface LatencyProps { buffer: Signal; - config: Getter; + config: Getter; } /** - * A helper class that computes the final latency based on the catalog's minBuffer and the user's buffer. - * If the minBuffer is not present, then we use framerate to estimate a default. + * A helper class that computes the final latency based on the catalog's jitter and the user's buffer. + * If the jitter is not present, then we use framerate to estimate a default. * - * Effective latency = catalog.minBuffer + buffer + * Effective latency = catalog.jitter + buffer */ export class Latency { buffer: Signal; - config: Getter; + config: Getter; signals = new Effect(); @@ -33,18 +33,18 @@ export class Latency { #run(effect: Effect): void { const buffer = effect.get(this.buffer); - // Compute the latency based on the catalog's minBuffer and the user's buffer. + // Compute the latency based on the catalog's jitter and the user's buffer. const config = effect.get(this.config); - // Use minBuffer from catalog if available, otherwise estimate from framerate - let minBuffer: number | undefined = config?.minBuffer; - if (minBuffer === undefined && config?.framerate !== undefined && config.framerate > 0) { - // Estimate minBuffer as one frame duration if framerate is available - minBuffer = 1000 / config.framerate; + // Use jitter from catalog if available, otherwise estimate from framerate + let jitter: number | undefined = config?.jitter; + if (jitter === undefined && config?.framerate !== undefined && config.framerate > 0) { + // Estimate jitter as one frame duration if framerate is available + jitter = 1000 / config.framerate; } - minBuffer ??= 0; + jitter ??= 0; - const latency = (minBuffer + buffer) as Moq.Time.Milli; + const latency = (jitter + buffer) as Moq.Time.Milli; this.#combined.set(latency); } diff --git a/js/hang/src/watch/element.ts b/js/hang/src/watch/element.ts index 6e6b64208..87de181ce 100644 --- a/js/hang/src/watch/element.ts +++ b/js/hang/src/watch/element.ts @@ -52,8 +52,8 @@ export default class HangWatch extends HTMLElement { // TODO: Temporarily defaults to false because Cloudflare doesn't support it yet. reload = new Signal(false); - // Additional buffer in milliseconds on top of the catalog's minBuffer (default: 100ms). - // The effective latency = catalog.minBuffer + buffer + // Additional buffer in milliseconds on top of the catalog's jitter (default: 100ms). + // The effective latency = catalog.jitter + buffer buffer = new Signal(100 as Time.Milli); // Set when the element is connected to the DOM. diff --git a/js/hang/src/watch/sync.ts b/js/hang/src/watch/sync.ts index d0ad15b1b..391ac0f25 100644 --- a/js/hang/src/watch/sync.ts +++ b/js/hang/src/watch/sync.ts @@ -44,20 +44,20 @@ export class Sync { #runLatency(effect: Effect): void { const buffer = effect.get(this.buffer); - // Compute the latency based on the catalog's minBuffer and the user's buffer. + // Compute the latency based on the catalog's jitter and the user's buffer. const video = effect.get(this.video); - // Use minBuffer from catalog if available, otherwise estimate from framerate - let videoBuffer: number | undefined = video?.minBuffer; + // Use jitter from catalog if available, otherwise estimate from framerate + let videoBuffer: number | undefined = video?.jitter; if (videoBuffer === undefined && video?.framerate !== undefined && video.framerate > 0) { - // Estimate minBuffer as one frame duration if framerate is available + // Estimate jitter as one frame duration if framerate is available videoBuffer = 1000 / video.framerate; } videoBuffer ??= 0; const audio = effect.get(this.audio); // TODO if there's no explicit buffer, estimate the audio buffer based on the sample rate and codec? - const audioBuffer = audio?.minBuffer ?? 0; + const audioBuffer = audio?.jitter ?? 0; const latency = (Math.max(videoBuffer, audioBuffer) + buffer) as Time.Milli; this.#latency.set(latency); diff --git a/rs/hang/examples/video.rs b/rs/hang/examples/video.rs index eecabbedc..04dc0721e 100644 --- a/rs/hang/examples/video.rs +++ b/rs/hang/examples/video.rs @@ -67,7 +67,7 @@ fn create_track(broadcast: &mut moq_lite::BroadcastProducer) -> hang::TrackProdu display_ratio_height: None, optimize_for_latency: None, container: hang::catalog::Container::Legacy, - min_buffer: None, + jitter: None, }; // Create a map of video renditions diff --git a/rs/hang/src/catalog/audio/mod.rs b/rs/hang/src/catalog/audio/mod.rs index e95730d89..89b7202f5 100644 --- a/rs/hang/src/catalog/audio/mod.rs +++ b/rs/hang/src/catalog/audio/mod.rs @@ -68,13 +68,12 @@ pub struct AudioConfig { #[serde(default)] pub container: Container, - /// Minimum buffer size in milliseconds required for smooth playback. + /// The maximum jitter before the next frame is emitted in milliseconds. + /// The player's jitter buffer should be larger than this value. + /// If not provided, the player should assume each frame is flushed immediately. /// - /// This represents the minimum time the player should buffer before starting playback. - /// For HLS imports, this is typically the segment duration. - /// For fMP4 imports, this is detected from the fragment duration. - /// - /// The player should add additional jitter buffer on top of this value. + /// NOTE: The audio "frame" duration depends on the codec, sample rate, etc. + /// ex: AAC often uses 1024 samples per frame, so at 44100Hz, this would be 1024/44100 = 23ms #[serde(default)] - pub min_buffer: Option, + pub jitter: Option, } diff --git a/rs/hang/src/catalog/root.rs b/rs/hang/src/catalog/root.rs index 5c97e4aa7..dafe2744d 100644 --- a/rs/hang/src/catalog/root.rs +++ b/rs/hang/src/catalog/root.rs @@ -332,7 +332,7 @@ mod test { framerate: Some(30.0), optimize_for_latency: None, container: Container::Legacy, - min_buffer: None, + jitter: None, }, ); @@ -346,7 +346,7 @@ mod test { bitrate: Some(128_000), description: None, container: Container::Legacy, - min_buffer: None, + jitter: None, }, ); diff --git a/rs/hang/src/catalog/video/mod.rs b/rs/hang/src/catalog/video/mod.rs index 86bf8bfb8..08999a5bb 100644 --- a/rs/hang/src/catalog/video/mod.rs +++ b/rs/hang/src/catalog/video/mod.rs @@ -117,13 +117,14 @@ pub struct VideoConfig { #[serde(default)] pub container: Container, - /// Minimum buffer size in milliseconds required for smooth playback. + /// The maximum jitter before the next frame is emitted in milliseconds. + /// The player's jitter buffer should be larger than this value. + /// If not provided, the player should assume each frame is flushed immediately. /// - /// This represents the minimum time the player should buffer before starting playback. - /// For HLS imports, this is typically the segment duration. - /// For fMP4 imports, this is detected from the fragment duration. - /// - /// The player should add additional jitter buffer on top of this value. + /// ex: + /// - If each frame is flushed immediately, this would be 1000/fps. + /// - If there can be up to 3 b-frames in a row, this would be 3 * 1000/fps. + /// - If frames are buffered into 2s segments, this would be 2s. #[serde(default)] - pub min_buffer: Option, + pub jitter: Option, } diff --git a/rs/hang/src/import/aac.rs b/rs/hang/src/import/aac.rs index 240e1e970..a74773167 100644 --- a/rs/hang/src/import/aac.rs +++ b/rs/hang/src/import/aac.rs @@ -108,7 +108,7 @@ impl Aac { bitrate: None, description: None, container: hang::catalog::Container::Legacy, - min_buffer: None, + jitter: None, }; tracing::debug!(name = ?track.name, ?config, "starting track"); diff --git a/rs/hang/src/import/avc3.rs b/rs/hang/src/import/avc3.rs index a4bdf6fe4..4620efc79 100644 --- a/rs/hang/src/import/avc3.rs +++ b/rs/hang/src/import/avc3.rs @@ -64,7 +64,7 @@ impl Avc3 { display_ratio_height: None, optimize_for_latency: None, container: hang::catalog::Container::Legacy, - min_buffer: None, + jitter: None, }; if let Some(old) = &self.config diff --git a/rs/hang/src/import/fmp4.rs b/rs/hang/src/import/fmp4.rs index 986897260..2de366ba2 100644 --- a/rs/hang/src/import/fmp4.rs +++ b/rs/hang/src/import/fmp4.rs @@ -69,7 +69,7 @@ struct Fmp4Track { group: Option, // The minimum buffer required for the track. - min_buffer: Option, + jitter: Option, // The last timestamp seen for this track. last_timestamp: Option, @@ -84,7 +84,7 @@ impl Fmp4Track { kind, producer, group: None, - min_buffer: None, + jitter: None, last_timestamp: None, min_duration: None, } @@ -272,7 +272,7 @@ impl Fmp4 { display_ratio_height: None, optimize_for_latency: None, container, - min_buffer: None, + jitter: None, } } mp4_atom::Codec::Hev1(hev1) => self.init_h265(true, &hev1.hvcc, &hev1.visual, container)?, @@ -289,7 +289,7 @@ impl Fmp4 { display_ratio_height: None, optimize_for_latency: None, container, - min_buffer: None, + jitter: None, }, mp4_atom::Codec::Vp09(vp09) => { // https://github.com/gpac/mp4box.js/blob/325741b592d910297bf609bc7c400fc76101077b/src/box-codecs.js#L238 @@ -317,7 +317,7 @@ impl Fmp4 { bitrate: None, framerate: None, container, - min_buffer: None, + jitter: None, } } mp4_atom::Codec::Av01(av01) => { @@ -351,7 +351,7 @@ impl Fmp4 { bitrate: None, framerate: None, container, - min_buffer: None, + jitter: None, } } mp4_atom::Codec::Unknown(unknown) => anyhow::bail!("unknown codec: {:?}", unknown), @@ -393,7 +393,7 @@ impl Fmp4 { display_ratio_height: None, optimize_for_latency: None, container, - min_buffer: None, + jitter: None, }) } @@ -428,7 +428,7 @@ impl Fmp4 { bitrate: Some(bitrate.into()), description: None, // TODO? container, - min_buffer: None, + jitter: None, } } mp4_atom::Codec::Opus(opus) => { @@ -439,7 +439,7 @@ impl Fmp4 { bitrate: None, description: None, // TODO? container, - min_buffer: None, + jitter: None, } } mp4_atom::Codec::Unknown(unknown) => anyhow::bail!("unknown codec: {:?}", unknown), @@ -487,7 +487,7 @@ impl Fmp4 { anyhow::bail!("missing trun box"); } - // Keep track of the minimum and maximum timestamp for this track to compute the min_buffer. + // Keep track of the minimum and maximum timestamp for this track to compute the jitter. // Ideally these should both be the same value (a single frame lul). let mut min_timestamp = None; let mut max_timestamp = None; @@ -643,12 +643,12 @@ impl Fmp4 { // We report the minimum buffer required as the difference between the min and max frames. // We also add the duration between frames to account for the frame rate. // ex. for 2s fragments, this should be exactly 2s if we did everything correctly. - let min_buffer = max - min + min_duration; + let jitter = max - min + min_duration; - if min_buffer < track.min_buffer.unwrap_or(Timestamp::MAX) { - track.min_buffer = Some(min_buffer); + if jitter < track.jitter.unwrap_or(Timestamp::MAX) { + track.jitter = Some(jitter); - // Update the catalog with the new min_buffer + // Update the catalog with the new jitter let mut catalog = self.broadcast.catalog.lock(); match track.kind { @@ -658,7 +658,7 @@ impl Fmp4 { .renditions .get_mut(&track.producer.info.name) .context("missing video config")?; - config.min_buffer = Some(min_buffer.convert()?); + config.jitter = Some(jitter.convert()?); } TrackKind::Audio => { let audio = catalog.audio.as_mut().context("missing audio")?; @@ -666,7 +666,7 @@ impl Fmp4 { .renditions .get_mut(&track.producer.info.name) .context("missing audio config")?; - config.min_buffer = Some(min_buffer.convert()?); + config.jitter = Some(jitter.convert()?); } } } diff --git a/rs/hang/src/import/hev1.rs b/rs/hang/src/import/hev1.rs index 571c3d51b..5a4d8dfe4 100644 --- a/rs/hang/src/import/hev1.rs +++ b/rs/hang/src/import/hev1.rs @@ -63,7 +63,7 @@ impl Hev1 { display_ratio_height: vui_data.display_ratio_height, optimize_for_latency: None, container: hang::catalog::Container::Legacy, - min_buffer: None, + jitter: None, }; if let Some(old) = &self.config diff --git a/rs/hang/src/import/opus.rs b/rs/hang/src/import/opus.rs index cfbf17159..7003b202e 100644 --- a/rs/hang/src/import/opus.rs +++ b/rs/hang/src/import/opus.rs @@ -54,7 +54,7 @@ impl Opus { bitrate: None, description: None, container: hang::catalog::Container::Legacy, - min_buffer: None, + jitter: None, }; tracing::debug!(name = ?track.name, ?config, "starting track");