From d263a0ff3e28deb38b2cc5020d2d1a74fbbed792 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 00:05:56 -0700 Subject: [PATCH 01/20] chore(quest): claim quest/m1/subscribe-ranges/model Co-Authored-By: Claude Opus 5.5 From 36c1f47d4a49712e8dd0331c995eb21c0e949f76 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 08:02:46 -0700 Subject: [PATCH 02/20] feat(net)!: carry group ranges on track::Subscription Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/benches/group.rs | 2 +- rs/moq-net/src/ietf/publisher.rs | 18 +- rs/moq-net/src/ietf/subscriber.rs | 8 +- rs/moq-net/src/lite/publisher.rs | 18 +- rs/moq-net/src/lite/subscriber.rs | 51 +- rs/moq-net/src/model/resume.rs | 25 +- rs/moq-net/src/model/subscription.rs | 584 ++++++++++++------ rs/moq-net/src/model/track.rs | 64 +- rs/moq-net/tests/announce_to_serve.rs | 2 +- rs/moq-net/tests/finished_broadcast_mock.rs | 2 +- rs/moq-net/tests/group_order.rs | 4 +- rs/moq-net/tests/head_fetch.rs | 7 +- rs/moq-net/tests/history_groups.rs | 6 +- rs/moq-net/tests/late_lower_group.rs | 14 +- rs/moq-net/tests/loom.rs | 4 +- rs/moq-net/tests/quiet_catalog.rs | 6 +- .../tests/settled_end_survives_abort.rs | 2 +- .../tests/subscription_end_integrity.rs | 4 +- rs/moq-net/tests/track_tail.rs | 8 +- rs/moq-net/tests/untimed.rs | 2 +- 20 files changed, 545 insertions(+), 286 deletions(-) diff --git a/rs/moq-net/benches/group.rs b/rs/moq-net/benches/group.rs index 1446fbcd405..ec9cd38d45c 100644 --- a/rs/moq-net/benches/group.rs +++ b/rs/moq-net/benches/group.rs @@ -216,7 +216,7 @@ fn filled_track(n: usize, payload: &Bytes) -> TrackCtx { fn replay() -> track::Subscription { track::Subscription::default() .with_max_delay(Duration::MAX) - .with_start(track::Position::group(0)) + .with_groups(0..) } /// Drain N cached groups from a track, in arrival order and in sequence order. diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index ec5fef0dc40..edaccc7392b 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -1067,14 +1067,16 @@ where // cover the group with no gap and no overlap. let edge = live_edge(&cache); let range = subscribe_range(&msg, edge, self.version); - subscription.start = range.start.map(|start| track::Position { - group: start.group, - frame: start.object, - }); - subscription.end = range.end.and_then(|end| match end.object { - Some(object) => track::Position::after(end.group, object), - None => track::Position::after_group(end.group), - }); + subscription.set_bounds( + range.start.map(|start| track::Position { + group: start.group, + frame: start.object, + }), + range.end.and_then(|end| match end.object { + Some(object) => track::Position::after(end.group, object), + None => track::Position::after_group(end.group), + }), + ); let _ = track.update(subscription); // A Timestamp goes out wherever the draft can declare its units and the track // has them. A subscriber that opted out of this SUBSCRIBE_OK's properties learns diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 98e00ea0427..d47040dfd8f 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -2480,12 +2480,12 @@ where ) -> Option { let validity = self.validity.start(); let subscription = target.subscription(); - let start = subscription.as_ref().and_then(|s| s.start); + let start = subscription.as_ref().and_then(|s| s.start()); // A live join delivers nothing below the group SUBSCRIBE_OK names as Largest. let live = start.is_none(); let join = match subscribe_join( - subscription.as_ref().and_then(|s| s.start), - subscription.as_ref().and_then(|s| s.end), + subscription.as_ref().and_then(|s| s.start()), + subscription.as_ref().and_then(|s| s.end()), self.version, ) { Ok(join) => join, @@ -2529,7 +2529,7 @@ where TrackState { resume: subscription .as_ref() - .and_then(|s| s.start) + .and_then(|s| s.start()) .filter(|start| start.frame != 0), // A resumed copy is the session's to abort from here on. producer: target.producer().cloned(), diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 75a6cb6fa70..a3ed9603609 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3332,7 +3332,7 @@ impl TrackRun { // Feed the full update into the model subscriber so the producer's // aggregate reflects it (and a relay re-forwards it upstream). // Read first: `update` replaces these preferences. - let requested = self.track.subscription().start.map(|start| start.group); + let requested = self.track.subscription().start().map(|start| start.group); let floored = requested.is_some_and(|group| group > 0); let bounds = Bounds::from(&upd); let _ = self.track.update(crate::track::Subscription { @@ -3437,7 +3437,7 @@ impl TrackRun { if self.emit_range && self.start.is_none() && self.ctx.version.has_largest() - && let Some(start) = self.track.subscription().start + && let Some(start) = self.track.subscription().start() && let Poll::Ready(Some(largest)) = self.track.poll_live(waiter) && start > largest { @@ -3538,7 +3538,7 @@ impl TrackRun { return Ok(()); }; // A cap below the end bounds what the subscriber is owed, as on its side. - let end = final_sequence(self.track.subscription().end.map_or(fin, |end| end.min(fin))); + let end = final_sequence(self.track.subscription().end().map_or(fin, |end| end.min(fin))); served.expire(self.ctx.runtime.now()); for gap in served.gaps(start..end) { writer.buffer(&lite::SubscribeResponse::Drop(lite::SubscribeDrop { @@ -3594,7 +3594,7 @@ impl TrackRun { // floor. Lite-06 encodes a floor of group 0 as 0, which decodes as no named floor; // that 0 is group 0, so it is not pinned. Raised, not assigned: an update that // landed while the group was held may already have raised it past. - if self.track.subscription().start.is_none() && !self.ctx.version.resolves_start() { + if self.track.subscription().start().is_none() && !self.ctx.version.resolves_start() { self.track.raise_start_to(start); } self.serve(group); @@ -3956,13 +3956,13 @@ mod serve_group_test { end_group: Some(5), end_frame: None, }; - assert_eq!(whole.positions().end, Some(track::Position::group(6))); + assert_eq!(whole.positions().end(), Some(track::Position::group(6))); let capped = Bounds { end_frame: Some(2), ..whole }; - assert_eq!(capped.positions().end, Some(track::Position { group: 5, frame: 3 })); + assert_eq!(capped.positions().end(), Some(track::Position { group: 5, frame: 3 })); let started = Bounds { start_group: Some(5), @@ -3971,8 +3971,8 @@ mod serve_group_test { end_frame: None, }; let positions = started.positions(); - assert_eq!(positions.start, Some(track::Position { group: 5, frame: 3 })); - assert_eq!(positions.end, None); + assert_eq!(positions.start(), Some(track::Position { group: 5, frame: 3 })); + assert_eq!(positions.end(), None); // A frame bound the peer sent without its group has nothing to count from, so it // cannot reach the model at all. @@ -3982,7 +3982,7 @@ mod serve_group_test { end_group: None, end_frame: Some(7), }; - assert_eq!((orphan.positions().start, orphan.positions().end), (None, None)); + assert_eq!((orphan.positions().start(), orphan.positions().end()), (None, None)); } /// A group whose head the publisher no longer holds is skipped, not served short: diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index cd9948f28f6..051581646d1 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -2917,15 +2917,15 @@ mod tests { #[test] fn wire_bounds_match_the_builders() { let whole = Subscription::default().with_end(Position::after_group(5)); - let bounds = WireBounds::new(whole.start, whole.end); + let bounds = WireBounds::new(whole.start(), whole.end()); assert_eq!((bounds.end_group, bounds.end_frame), (Some(5), None)); let capped = Subscription::default().with_end(Position::after(5, 2)); - let bounds = WireBounds::new(capped.start, capped.end); + let bounds = WireBounds::new(capped.start(), capped.end()); assert_eq!((bounds.end_group, bounds.end_frame), (Some(5), Some(2))); let started = Subscription::default().with_start(Position { group: 5, frame: 3 }); - let bounds = WireBounds::new(started.start, started.end); + let bounds = WireBounds::new(started.start(), started.end()); assert_eq!((bounds.start_group, bounds.start_frame), (Some(5), 3)); } @@ -3052,10 +3052,10 @@ mod tests { .with_end(Position::after(u64::MAX, 5)); h.serve.widen_frame_bounds(&mut subscription); - assert_eq!(subscription.start, Some(Position::group(u64::MAX))); + assert_eq!(subscription.start(), Some(Position::group(u64::MAX))); // Past the last group there is no position to round up to, and unbounded is the // wider request. - assert_eq!(subscription.end, None); + assert_eq!(subscription.end(), None); } /// The widening covers SUBSCRIBE_UPDATE too: a downstream peer asking for a frame @@ -4527,7 +4527,10 @@ impl TrackServe { // subscriber holds, so ask from the head of its group instead: the first frame then // arrives at once and says where the live feed is. if !self.subscriber.version.has_largest() { - subscription.start = subscription.start.map(|start| Position::group(start.group)); + subscription.set_bounds( + subscription.start().map(|start| Position::group(start.group)), + subscription.end(), + ); } if self.subscriber.version.has_frame_bounds() { return; @@ -4536,23 +4539,22 @@ impl TrackServe { // Round both bounds outward to the enclosing group, so the peer sends at least // what was asked for and never less. Rounding the end down instead would be able // to empty a non-empty range, which has no wire encoding at all. - let start = subscription.start.map(|start| Position::group(start.group)); - let end = subscription.end.and_then(|end| match end.frame { + let start = subscription.start().map(|start| Position::group(start.group)); + let end = subscription.end().and_then(|end| match end.frame { 0 => Some(end), // Past the last group there is no position to round up to, and unbounded is // the wider request. _ => Position::after_group(end.group), }); - if (start, end) != (subscription.start, subscription.end) { + if (start, end) != (subscription.start(), subscription.end()) { tracing::debug!( track = %self.name, version = ?self.subscriber.version, "widening frame bounds to whole groups for an older peer" ); } - subscription.start = start; - subscription.end = end; + subscription.set_bounds(start, end); } /// Apply a subscription-demand change: hand back an [`Establish`] to open the @@ -4572,18 +4574,21 @@ impl TrackServe { // demand at all is the faithful translation. An absent start is the live edge, wherever that lands, so the only end that is // certainly empty is the very first position, which is what `Position::default()` // stands in for. - let pref = pref.filter(|sub| sub.end.is_none_or(|end| end > sub.start.unwrap_or_default())); + let pref = pref.filter(|sub| sub.end().is_none_or(|end| end > sub.start().unwrap_or_default())); match pref { Some(mut subscription) => { + // The wire carries one range: the span of every range asked for, which the + // subscribers' own reads narrow again. + subscription.set_bounds(subscription.start(), subscription.end()); self.widen_frame_bounds(&mut subscription); match sub { Sub::None => { // An idle copy is asked from the head of the newest group it cached when // the subscriber has no start of its own: a quiet track (a catalog) sends // that group again, which says the cache is current. - if subscription.start.is_none() { - subscription.start = producer.idle_newest().map(Position::group); + if subscription.start().is_none() { + subscription.set_bounds(producer.idle_newest().map(Position::group), subscription.end()); } // Open an upstream SUBSCRIBE for the first subscriber. Ok(Begin::Establish(self.prepare_establish( @@ -4595,19 +4600,19 @@ impl TrackServe { Sub::Active(active) => { // Downstream preferences changed: forward them upstream as a // SUBSCRIBE_UPDATE (Lite03+ only; older peers can't carry one). - let start_moved = active.start != subscription.start; + let start_moved = active.start != subscription.start(); active.priority = subscription.priority; active.order = subscription.order; active.max_delay = subscription.max_delay; if let Ok(mut tail) = active.tail.write() { tail.set_grace(tail::grace(subscription.max_delay)); // A lowered floor owes groups nobody asked for until now. - if let Some(start) = subscription.start { + if let Some(start) = subscription.start() { let floor = active.start.map(|start| start.group).or(active.served); tail.demand(start.group..floor.unwrap_or(u64::MAX), self.subscriber.runtime.now()); } } - active.start = subscription.start; + active.start = subscription.start(); if supports_update { // The floor follows the requested start, in both directions: // moving below a declared SUBSCRIBE_START reopens those groups @@ -4620,7 +4625,7 @@ impl TrackServe { if start_moved { let _ = producer.start_at(active.start.map(|start| start.group)); } - buffer_update(active, subscription.end)?; + buffer_update(active, subscription.end())?; } Ok(Begin::None) } @@ -4669,7 +4674,7 @@ impl TrackServe { // previous subscription's declaration must not outlive its demand, and // live-edge demand (None) starts with no floor at all. Lite-06+ only resolves // the start with its SUBSCRIBE_START, so until then the floor is just a request. - let floor = subscription.start.map(|start| start.group); + let floor = subscription.start().map(|start| start.group); let _ = match self.subscriber.version.resolves_start() { true => producer.request_start(floor), false => producer.start_at(floor), @@ -4827,7 +4832,7 @@ impl Establish { &mut cx ))?; - let bounds = WireBounds::new(self.subscription.start, self.subscription.end); + let bounds = WireBounds::new(self.subscription.start(), self.subscription.end()); let msg = lite::Subscribe { epoch: self.serve.epoch.clone(), id: self.id, @@ -4888,10 +4893,10 @@ impl Establish { stream, id: self.id, max_delay: self.subscription.max_delay, - start: self.subscription.start, + start: self.subscription.start(), priority: self.subscription.priority, order: self.subscription.order, - requested: self.subscription.start, + requested: self.subscription.start(), tail: self.tail.clone(), served: None, end: None, @@ -5640,7 +5645,7 @@ impl ServeLoop { // parks, since a cap can be raised). Those streams can still // be in flight, so wait for the tail before finishing. let subscription = self.serving.subscription(); - let requested_end = subscription.as_ref().and_then(|sub| sub.end).map(|end| match end + let requested_end = subscription.as_ref().and_then(|sub| sub.end()).map(|end| match end .frame { 0 => end.group, diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 51e5f021e36..6bbfd6630b4 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -24,7 +24,7 @@ use std::task::{Poll, ready}; use crate::{Error, Result, StreamError, group, stats, track}; use super::delivered::Delivered; -use super::subscription::{Cap, Position, Subscription, max_some}; +use super::subscription::{Cap, Position, Subscription}; /// What the front says about a logical track: the copy to read now, and how it ended. #[derive(Default)] @@ -263,14 +263,7 @@ impl Copy { /// What this copy is asked for: the reader's own preferences, from no earlier than /// where it was asked to start, and through no later than where it was replaced. fn wire(&self, reader: &Subscription) -> Subscription { - let mut wire = Subscription { - start: max_some(reader.start, self.floor), - ..reader.clone() - }; - if let Some(end) = self.end().map(Position::group) { - wire.end = Some(wire.end.map_or(end, |requested| requested.min(end))); - } - wire + reader.clamped(self.floor, self.end().map(Position::group)) } /// The first group past where this copy was replaced, if any: one replaced before @@ -491,10 +484,7 @@ impl Reader { let floor = self.newest.map(Position::group); // The wire request carries the floor from the first message: some sessions read // the demand only once. - let wire = Subscription { - start: max_some(self.mirrored.start, floor), - ..self.mirrored.clone() - }; + let wire = self.mirrored.clamped(floor, None); self.copies.push(Copy { generation, sub: Sub::Pending(track.subscribe(wire)), @@ -1572,13 +1562,10 @@ mod test { let mut reader = sub.reader.lock().unwrap(); let old = &mut reader.copies[0]; old.until = Some(last); - assert_eq!(old.wire(&Subscription::default()).end, end); + assert_eq!(old.wire(&Subscription::default()).end(), end); // A reader's narrower bound survives an unbounded replacement cap. - let requested = Subscription { - end: Some(Position::group(7)), - ..Default::default() - }; - assert_eq!(old.wire(&requested).end, Some(end.unwrap_or(Position::group(7)))); + let requested = Subscription::default().with_end(Position::group(7)); + assert_eq!(old.wire(&requested).end(), Some(end.unwrap_or(Position::group(7)))); } } diff --git a/rs/moq-net/src/model/subscription.rs b/rs/moq-net/src/model/subscription.rs index c851b90d451..d44e23501fc 100644 --- a/rs/moq-net/src/model/subscription.rs +++ b/rs/moq-net/src/model/subscription.rs @@ -28,6 +28,9 @@ pub struct Subscription { /// the publisher's [`Info::max_age`](crate::track::Info::max_age), since waiting for /// a group longer than it is kept around cannot produce it. /// + /// It caps every one of [`Self::ranges`]: a range names what may be delivered, and this + /// budget skips what inside it has gone stale. + /// /// # Where it is enforced /// /// At both ends, and neither alone is enough. The publisher skips a group that has @@ -61,36 +64,19 @@ pub struct Subscription { /// standalone track needs [`Pool::gc`](crate::cache::Pool::gc) called, as its idle /// expiry already does. Otherwise it is judged only when something else wakes the read. pub max_delay: Duration, - /// The lowest [`Position`] the publisher may deliver, or `None` for no floor. + /// The spans of the track to deliver; see [`Range`]. /// - /// A floor, not a request: only [`Self::max_delay`] asks for data, and the floor bounds - /// how far back it may reach. `None` and a floor of group 0 mean the same thing, since - /// nothing sits below group 0. Delivery starts at the oldest group at or above the floor that the budget still - /// considers fresh, so a floor above the live edge simply waits there (a resumed - /// subscription naming where it left off). + /// The default is [`Range::oldest`]: everything [`Self::max_delay`] still considers + /// fresh. [`Range::live`] starts at the latest group instead, and an empty list asks + /// for nothing. /// - /// Aggregated across every live subscriber (the loosest floor wins, and any subscriber - /// without one clears it), so it says what the publisher sends, not what any one - /// subscriber sees. [`crate::track::Subscriber::set_groups`] is the local read cursor; - /// setting one does not imply the other. See [Local cursor vs wire + /// A request to the publisher, aggregated across every live subscriber as the union of + /// their ranges, so it says what the publisher sends, not what any one subscriber sees. + /// The one place it touches this subscriber's reads is where its cursor starts: at the + /// lowest group any of its ranges names. [`crate::track::Subscriber::set_groups`] is the + /// local read filter. See [Local cursor vs wire /// preference](crate::track::Subscriber#local-cursor-vs-wire-preference). - pub start: Option, - /// First [`Position`] the publisher should *not* deliver, or `None` for no end. - /// - /// Exclusive, like the end of a [`std::ops::Range`], which is what lets one field - /// carry both "through the end of group 5" ([`Position::after_group(5)`](Position::after_group)) - /// and "up to frame 2 of group 5" ([`Position::after(5, 2)`](Position::after)). An - /// inclusive end cannot express the first without a sentinel frame, and the ordering - /// falls out for free: group 6's head sorts above any frame of group 5, so a - /// whole-group subscriber correctly absorbs a frame-capped one in the aggregate. - /// - /// The wire agrees: `Group End` and `Frame End` are both encoded as `absolute + 1`. - /// - /// A request, aggregated across every live subscriber (any unbounded subscriber makes - /// the aggregate unbounded). [`crate::track::Subscriber::set_groups`] is the local read - /// cursor; [`Position::group_end`] translates this field into its bound. Setting one - /// does not imply the other. - pub end: Option, + pub ranges: Vec, /// Which pending group the publisher sends first; see [`Order`]. /// /// Aggregated across every live subscriber: any [`Order::Descending`] subscriber makes @@ -114,13 +100,152 @@ pub enum Order { Descending, } +/// Where one end of a [`Range`] falls: a fixed position, or a count of groups back from the +/// latest one. +/// +/// A relative anchor resolves against the latest group of the hop that serves it, once, +/// when it starts serving. Before any group exists it resolves at the first one, so +/// [`Range::live`] starts there. A relay that cannot resolve it forwards it upstream as is. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Anchor { + /// This exact position. + Absolute(Position), + /// The group this many before the latest, clamped to group 0: `Relative(0)` is the latest group. + Relative(u64), +} + +/// A span of a track to deliver: from `start` up to `end`, or onward when `end` is `None`. +/// +/// Relative anchors cover whole groups: a relative start is frame 0 of its group, and a +/// relative end includes its group, so `Absolute(0)..Relative(0)` delivers through the latest +/// group. An absolute end is exclusive, like the end of a [`std::ops::Range`]: see +/// [`Position::after`] and [`Position::after_group`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Range { + /// Where delivery starts. + pub start: Anchor, + /// Where delivery stops, or `None` to keep following the live edge. + pub end: Option, +} + +impl Range { + /// Everything from the latest group onward. + pub fn live() -> Self { + Self { + start: Anchor::Relative(0), + end: None, + } + } + + /// Everything from the oldest group still around onward, the default. + /// + /// Unlike `Absolute(0)..` this names no floor, so an older wire asks for the live edge + /// and lets [`Subscription::max_delay`] decide how far back to reach. + pub fn oldest() -> Self { + Self { + start: Anchor::Relative(u64::MAX), + end: None, + } + } + + /// The whole groups in `groups`, such as `2..=5`, `2..`, or `..6`. + /// + /// An omitted start is [`Self::oldest`]'s, and an inclusive end past the last group is + /// no end, as [`Position::after_group`] spells it. + pub fn groups(groups: impl RangeBounds) -> Self { + let unbounded_start = matches!(groups.start_bound(), Bound::Unbounded); + let (start, end) = sequence_bounds(groups); + Self { + start: match unbounded_start { + true => Self::oldest().start, + false => Anchor::Absolute(Position::group(start)), + }, + end: end.map(|end| Anchor::Absolute(Position::group(end))), + } + } + + /// The first position this range delivers, or `None` while it waits on a latest group. + fn resolve_start(&self, latest: Option) -> Option { + match self.start { + Anchor::Absolute(start) => Some(start), + // Nothing sits below group 0, so the farthest offset needs no latest group. + Anchor::Relative(u64::MAX) => Some(Position::default()), + Anchor::Relative(back) => Some(Position::group(latest?.saturating_sub(back))), + } + } + + /// Whether this range certainly delivers nothing, wherever the latest group lands. + fn is_empty(&self) -> bool { + match (self.start, self.end) { + (Anchor::Absolute(start), Some(Anchor::Absolute(end))) => end <= start, + _ => false, + } + } +} + +/// The union of `ranges` in a canonical order, without empty or redundant ranges. +/// +/// Exact only where the latest group cannot matter, which is enough for an aggregate to +/// compare equal across subscribers asking for the same thing in different words. +pub(super) fn union(ranges: impl IntoIterator) -> Vec { + let mut ranges: Vec = ranges.into_iter().filter(|range| !range.is_empty()).collect(); + if ranges.contains(&Range::oldest()) { + return vec![Range::oldest()]; + } + + // An open range covers every range starting at or after it, whatever its end. + let open_absolute = ranges + .iter() + .filter_map(|range| match (range.start, range.end) { + (Anchor::Absolute(start), None) => Some(start), + _ => None, + }) + .min(); + let open_relative = ranges + .iter() + .filter_map(|range| match (range.start, range.end) { + (Anchor::Relative(back), None) => Some(back), + _ => None, + }) + .max(); + ranges.retain(|range| match range.start { + Anchor::Absolute(start) => open_absolute.is_none_or(|open| start < open || (start == open && range.end.is_none())), + Anchor::Relative(back) => open_relative.is_none_or(|open| back > open || (back == open && range.end.is_none())), + }); + ranges.sort_unstable(); + ranges.dedup(); + + // Merge overlapping or adjacent absolute ranges. Sorted by start, so one pass suffices. + let mut merged: Vec = Vec::with_capacity(ranges.len()); + for range in ranges { + if let Some(last) = merged.last_mut() + && let Anchor::Absolute(start) = range.start + && let Some(Anchor::Absolute(end)) = last.end + && start <= end + { + match range.end { + Some(Anchor::Absolute(next)) => { + last.end = Some(Anchor::Absolute(end.max(next))); + continue; + } + None => { + last.end = None; + continue; + } + Some(Anchor::Relative(_)) => {} + } + } + merged.push(range); + } + merged +} + impl Default for Subscription { fn default() -> Self { Self { priority: 0, max_delay: Duration::ZERO, - start: None, - end: None, + ranges: vec![Range::oldest()], order: Order::Descending, } } @@ -140,25 +265,16 @@ impl Subscription { self } - /// Floor delivery at `start`, or leave it unfloored when `None`. Returns `self` for - /// chaining. - /// - /// A floor bounds how far back [`Self::max_delay`] may reach; it does not request data - /// on its own. [`Position::group`] is the whole-group form. - pub fn with_start(mut self, start: impl Into>) -> Self { - self.start = start.into(); + /// Replace [`Self::ranges`], returning `self` for chaining. + pub fn with_ranges(mut self, ranges: impl IntoIterator) -> Self { + self.ranges = ranges.into_iter().collect(); self } - /// Stop delivery at `end`, or leave the subscription unbounded when `None`. Returns - /// `self` for chaining. - /// - /// Exclusive, matching [`Self::end`], so pass the position *after* the last one you - /// want. [`Position::after`] and [`Position::after_group`] name that conversion so no - /// call site has to write the `+ 1` itself. - pub fn with_end(mut self, end: impl Into>) -> Self { - self.end = end.into(); - self + /// Request the whole groups in `groups` as the only range; see [`Range::groups`]. + /// Returns `self` for chaining. + pub fn with_groups(self, groups: impl RangeBounds) -> Self { + self.with_ranges([Range::groups(groups)]) } /// Set which pending group the publisher sends first, returning `self` for chaining. @@ -167,37 +283,115 @@ impl Subscription { self } - /// Request the whole groups in `groups`, replacing both [`Self::start`] and - /// [`Self::end`]. Returns `self` for chaining. + /// The lowest absolute position any range asks for, or `None` when one counts back + /// from the latest group (or none is asked for). /// - /// Any range of group sequences works: `2..=5`, `2..6`, `..6`, `2..`, or `..` to - /// clear both bounds. An inclusive end past the last group is unbounded, as - /// [`Position::after_group`] spells it. - pub fn with_groups(mut self, groups: impl RangeBounds) -> Self { - let unbounded_start = matches!(groups.start_bound(), Bound::Unbounded); - let (start, end) = sequence_bounds(groups); - self.start = (!unbounded_start).then(|| Position::group(start)); - self.end = end.map(Position::group); + /// The form a wire without ranges carries: a floor below which nothing is requested. + pub(crate) fn start(&self) -> Option { + let mut floor: Option = None; + for range in &self.ranges { + let Anchor::Absolute(start) = range.start else { + return None; + }; + floor = Some(floor.map_or(start, |floor| floor.min(start))); + } + floor + } + + /// The exclusive end past every range, or `None` when one is open or counts back from + /// the latest group. An empty list is the empty range, `Position::group(0)`. + /// + /// The form a wire without ranges carries, alongside [`Self::start`]. + pub(crate) fn end(&self) -> Option { + let mut end = Some(Position::default()); + for range in &self.ranges { + let Some(Anchor::Absolute(next)) = range.end else { + return None; + }; + end = end.max(Some(next)); + } + end + } + + /// Replace the ranges with one from `start` (or [`Range::oldest`]'s) to `end`: the form + /// a wire without ranges carries. + pub(crate) fn set_bounds(&mut self, start: Option, end: Option) { + self.ranges = vec![Range { + start: start.map_or(Range::oldest().start, Anchor::Absolute), + end: end.map(Anchor::Absolute), + }]; + } + + /// Replace the start of [`Self::set_bounds`]'s one range, keeping its end. + pub(crate) fn with_start(mut self, start: impl Into>) -> Self { + self.set_bounds(start.into(), self.end()); + self + } + + /// Replace the end of [`Self::set_bounds`]'s one range, keeping its start. + pub(crate) fn with_end(mut self, end: impl Into>) -> Self { + self.set_bounds(self.start(), end.into()); self } + /// These preferences with every range raised to start no earlier than `floor` and + /// stop no later than `end`, the exclusive end. + /// + /// A relative anchor becomes the bound itself, since which of the two is tighter + /// depends on a latest group this hop may not know. That asks for no less than the + /// range would. + pub(crate) fn clamped(&self, floor: Option, end: Option) -> Self { + let ranges = self + .ranges + .iter() + .map(|range| Range { + start: match (range.start, floor) { + (start, None) => start, + (Anchor::Absolute(start), Some(floor)) => Anchor::Absolute(start.max(floor)), + (Anchor::Relative(_), Some(floor)) => Anchor::Absolute(floor), + }, + end: match (range.end, end) { + (requested, None) => requested, + (Some(Anchor::Absolute(requested)), Some(end)) => Some(Anchor::Absolute(requested.min(end))), + (_, Some(end)) => Some(Anchor::Absolute(end)), + }, + }) + .collect(); + Self { + ranges, + ..self.clone() + } + } + + /// The lowest group this subscription's ranges start at, resolving relative starts + /// against `latest`. `None` while one waits on a latest group that does not exist yet. + /// + /// No ranges at all floor nowhere: group 0. + pub(crate) fn floor(&self, latest: Option) -> Option { + let mut floor = None; + for range in &self.ranges { + let start = range.resolve_start(latest)?.group; + floor = Some(floor.map_or(start, |floor: u64| floor.min(start))); + } + Some(floor.unwrap_or(0)) + } + // Fold this subscription into the running aggregate: Ready with the merged // result when it demands more than `combined`, Pending when it's a subset // (so callers can skip a redundant broadcast of the same aggregate). pub(super) fn poll_combined(&self, combined: &Option) -> Poll { let Some(combined) = combined else { - return Poll::Ready(self.clone()); + return Poll::Ready(Subscription { + ranges: union(self.ranges.iter().copied()), + ..self.clone() + }); }; let merged = Subscription { priority: self.priority.max(combined.priority), // Sequence-first prioritization is enabled only when every subscriber wants it. max_delay: self.max_delay.max(combined.max_delay), - // Bounds fold as whole positions. Two subscribers starting in the same group - // are separated only by their frame, so folding group and frame independently - // would invent a bound neither asked for. - start: min_floored(self.start, combined.start), - end: max_unbounded(self.end, combined.end), + ranges: union(self.ranges.iter().chain(&combined.ranges).copied()), order: match (self.order, combined.order) { (Order::Ascending, Order::Ascending) => Order::Ascending, _ => Order::Descending, @@ -221,7 +415,7 @@ impl Subscription { /// Pairing the two is the point: a frame index counts from the start of a group, so it /// means nothing on its own. Carrying them together makes "frame 5 of nothing" /// unrepresentable rather than merely undefined. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Position { /// The group sequence. pub group: u64, @@ -341,41 +535,6 @@ pub(super) fn before_end(sequence: u64, end: Option) -> bool { end.is_none_or(|end| sequence < end) } -// Combining two optional bounds comes in two families, and they disagree on what `None` -// means. `_some` treats it as the neutral element (the other side wins), for intersecting -// two ranges that each restrict independently. `_floored` / `_unbounded` treat it as -// absorbing (the result is `None` too), for aggregating across subscribers, where one -// subscriber asking for everything makes the aggregate everything. Picking the wrong -// family silently narrows or widens what the publisher sends, so the suffix, not the -// `min`/`max`, is the part to read. - -/// The higher of two optional bounds, `None` neutral. -pub(super) fn max_some(a: Option, b: Option) -> Option { - match (a, b) { - (Some(a), Some(b)) => Some(a.max(b)), - (Some(a), None) | (None, Some(a)) => Some(a), - (None, None) => None, - } -} - -/// The lower of two optional floors, `None` absorbing (no floor). The mirror of -/// [`max_unbounded`]: both bounds only ever *restrict*, so a subscriber without one keeps -/// the aggregate unrestricted. -pub(super) fn min_floored(a: Option, b: Option) -> Option { - match (a, b) { - (Some(a), Some(b)) => Some(a.min(b)), - (None, _) | (_, None) => None, - } -} - -/// The higher of two optional bounds, `None` absorbing (unbounded). -pub(super) fn max_unbounded(a: Option, b: Option) -> Option { - match (a, b) { - (Some(a), Some(b)) => Some(a.max(b)), - (None, _) | (_, None) => None, - } -} - #[cfg(test)] mod tests { use super::*; @@ -390,32 +549,35 @@ mod tests { combined } - /// The exclusive representation runs out at both extremes, and `Option` says so - /// rather than saturating into a bound that contradicts the request. - /// A group range spells both positions at once, in whichever form the caller has. - #[test] - fn group_ranges_build_whole_group_positions() { - let sub = Subscription::default().with_groups(2..=5); - assert_eq!(sub.start, Some(Position::group(2))); - assert_eq!(sub.end, Some(Position::group(6))); + fn absolute(group: u64, frame: u64) -> Anchor { + Anchor::Absolute(Position { group, frame }) + } - let sub = Subscription::default().with_groups(2..6); - assert_eq!(sub.end, Some(Position::group(6))); + fn range(start: Anchor, end: impl Into>) -> Range { + Range { + start, + end: end.into(), + } + } - let sub = Subscription::default().with_groups(..6); - assert_eq!(sub.start, None); - assert_eq!(sub.end, Some(Position::group(6))); + fn ranges(subscriptions: &[Subscription]) -> Vec { + combine(subscriptions).unwrap().ranges + } - let sub = Subscription::default().with_groups(2..); - assert_eq!(sub.start, Some(Position::group(2))); - assert_eq!(sub.end, None); + /// A group range spells both positions at once, in whichever form the caller has. + #[test] + fn group_ranges_build_whole_group_positions() { + assert_eq!(Range::groups(2..=5), range(absolute(2, 0), absolute(6, 0))); + assert_eq!(Range::groups(2..6), range(absolute(2, 0), absolute(6, 0))); + assert_eq!(Range::groups(..6), range(Range::oldest().start, absolute(6, 0))); + assert_eq!(Range::groups(2..), range(absolute(2, 0), None)); + assert_eq!(Range::groups(..), Range::oldest()); // Through the last group is unbounded, as `after_group` spells it. - let sub = Subscription::default().with_groups(..=u64::MAX); - assert_eq!(sub.end, None); + assert_eq!(Range::groups(..=u64::MAX).end, None); let sub = Subscription::default().with_groups(2..=5).with_groups(..); - assert_eq!((sub.start, sub.end), (None, None)); + assert_eq!(sub, Subscription::default()); } #[test] @@ -424,14 +586,10 @@ mod tests { // in the same group. assert_eq!(Position::after(5, u64::MAX), Some(Position::group(6))); - // Past everything has no position. `Subscription::end` spells that `None` too, - // so the meanings coincide and the group is included rather than dropped. + // Past everything has no position. An open range spells that `None` too, so the + // meanings coincide and the group is included rather than dropped. assert_eq!(Position::after_group(u64::MAX), None); assert_eq!(Position::after(u64::MAX, u64::MAX), None); - assert_eq!( - Subscription::default().with_end(Position::after_group(u64::MAX)).end, - None - ); // A head-of-group end excludes that group; a mid-group end includes it. assert_eq!(Position::group(0).group_end(), Bound::Excluded(0)); @@ -448,68 +606,152 @@ mod tests { assert_eq!(Cap::from(Bound::Unbounded), Cap::from(..)); } + /// A relative start counts back from the latest group and clamps at group 0. Before any + /// group exists only the farthest offset resolves, since it is group 0 regardless. #[test] - fn combined_group_start_keeps_the_loosest_floor() { - // A floor only restricts, so the lowest one wins across floored subscribers. - let catchup = Subscription::default().with_start(Position::group(10)); - let older_catchup = Subscription::default().with_start(Position::group(5)); - let combined = combine(&[catchup.clone(), older_catchup]).unwrap(); - assert_eq!(combined.start, Some(Position::group(5))); + fn relative_starts_resolve_against_the_latest_group() { + let live = Subscription::default().with_ranges([Range::live()]); + assert_eq!(live.floor(Some(7)), Some(7)); + assert_eq!(live.floor(None), None); + + let buffering = Subscription::default().with_ranges([range(Anchor::Relative(3), None)]); + assert_eq!(buffering.floor(Some(7)), Some(4)); + assert_eq!(buffering.floor(Some(1)), Some(0), "an offset past the start clamps"); + + assert_eq!(Subscription::default().floor(None), Some(0)); + + // The lowest range wins, and one unresolved range leaves the whole floor unresolved. + let mixed = Subscription::default().with_ranges([range(absolute(4, 0), None), Range::live()]); + assert_eq!(mixed.floor(Some(3)), Some(3)); + assert_eq!(mixed.floor(Some(9)), Some(4)); + assert_eq!(mixed.floor(None), None); - // A subscriber with no floor at all clears the aggregate: its budget may reach - // below any floor the others set. - let unfloored = Subscription::default(); - let combined = combine(&[catchup, unfloored]).unwrap(); - assert_eq!(combined.start, None); + // Nothing asked for floors nowhere. + assert_eq!(Subscription::default().with_ranges([]).floor(None), Some(0)); } + /// The bounds a wire without ranges carries: the lowest absolute start and the highest + /// absolute end, either one lost to a range that counts back from the latest group. #[test] - fn combined_group_end_keeps_live_subscription_unbounded() { - // No end at all is unbounded, which absorbs any explicit one. - let live = Subscription::default(); - let bounded = Subscription::default().with_end(Position::after_group(10)); + fn bounds_project_the_ranges_for_older_wires() { + let sub = Subscription::default(); + assert_eq!((sub.start(), sub.end()), (None, None)); + + let sub = Subscription::default().with_ranges([Range::groups(2..4), range(absolute(7, 1), absolute(9, 0))]); + assert_eq!( + (sub.start(), sub.end()), + (Some(Position::group(2)), Some(Position::group(9))) + ); + + // An explicit group 0 is a floor, unlike the default. + let sub = Subscription::default().with_groups(0..); + assert_eq!((sub.start(), sub.end()), (Some(Position::group(0)), None)); - let combined = combine(&[live, bounded]).unwrap(); + let sub = Subscription::default().with_ranges([Range::groups(2..4), Range::live()]); + assert_eq!((sub.start(), sub.end()), (None, None)); - assert_eq!(combined.end, None); + let sub = Subscription::default().with_ranges([range(absolute(2, 0), Anchor::Relative(0))]); + assert_eq!((sub.start(), sub.end()), (Some(Position::group(2)), None)); + + // Nothing asked for is the empty range. + let sub = Subscription::default().with_ranges([]); + assert_eq!(sub.end(), Some(Position::group(0))); + + let mut sub = Subscription::default(); + sub.set_bounds(Some(Position { group: 5, frame: 2 }), Position::after_group(9)); + assert_eq!( + (sub.start(), sub.end()), + (Some(Position { group: 5, frame: 2 }), Some(Position::group(10))) + ); + sub.set_bounds(None, None); + assert_eq!(sub, Subscription::default()); } + /// The aggregate is the union: overlapping and adjacent absolute ranges merge, an open + /// range absorbs what starts inside it, and the oldest range absorbs everything. #[test] - fn combined_start_folds_the_whole_position() { - let early_frame = Subscription::default().with_start(Position { group: 5, frame: 2 }); - let late_frame = Subscription::default().with_start(Position { group: 5, frame: 9 }); + fn combined_ranges_are_the_union() { + let sub = |ranges: &[Range]| Subscription::default().with_ranges(ranges.iter().copied()); - // Same group: the earlier frame wins. - let combined = combine(&[late_frame.clone(), early_frame.clone()]).unwrap(); - assert_eq!(combined.start, Some(Position { group: 5, frame: 2 })); + assert_eq!( + ranges(&[sub(&[Range::groups(2..5)]), sub(&[Range::groups(4..8)])]), + vec![Range::groups(2..8)] + ); + assert_eq!( + ranges(&[sub(&[Range::groups(2..5)]), sub(&[Range::groups(5..8)])]), + vec![Range::groups(2..8)], + "adjacent ranges merge" + ); + assert_eq!( + ranges(&[sub(&[Range::groups(2..5)]), sub(&[Range::groups(6..8)])]), + vec![Range::groups(2..5), Range::groups(6..8)], + "a gap stays a gap" + ); + assert_eq!( + ranges(&[sub(&[Range::groups(2..5), Range::groups(9..)]), sub(&[Range::groups(4..)])]), + vec![Range::groups(2..)] + ); + assert_eq!( + ranges(&[sub(&[Range::groups(2..5)]), Subscription::default()]), + vec![Range::oldest()] + ); + + // Relative ranges count back from the same latest group, so the widest open one covers + // every narrower one, whatever its end. + assert_eq!( + ranges(&[ + sub(&[Range::live()]), + sub(&[range(Anchor::Relative(3), None)]), + sub(&[range(Anchor::Relative(1), Anchor::Relative(0))]), + ]), + vec![range(Anchor::Relative(3), None)] + ); - // An earlier group wins outright, carrying its own frame rather than the - // smallest frame across the two. - let earlier_group = Subscription::default().with_start(Position { group: 4, frame: 7 }); - let combined = combine(&[early_frame, earlier_group]).unwrap(); - assert_eq!(combined.start, Some(Position { group: 4, frame: 7 })); + // An absolute floor above the live edge does not hide the live edge: both stay. + assert_eq!( + ranges(&[sub(&[Range::groups(4..)]), sub(&[Range::live()])]), + vec![Range::groups(4..), Range::live()] + ); + + // Empty ranges ask for nothing. + assert_eq!(ranges(&[sub(&[Range::groups(5..5)])]), Vec::::new()); + assert_eq!( + ranges(&[sub(&[]), sub(&[Range::groups(1..2)])]), + vec![Range::groups(1..2)] + ); } + /// Ranges fold as whole positions: two ranges in the same group are separated only by + /// their frames, so folding group and frame independently would invent a bound. #[test] - fn combined_end_folds_the_whole_position() { - let short = Subscription::default().with_end(Position::after(5, 2)); - let long = Subscription::default().with_end(Position::after(5, 9)); + fn combined_ranges_fold_whole_positions() { + let sub = |range: Range| Subscription::default().with_ranges([range]); - // Same group: the later frame wins. Ends are exclusive, so an inclusive frame 9 - // is stored as 10. - let combined = combine(&[short.clone(), long.clone()]).unwrap(); - assert_eq!(combined.end, Some(Position { group: 5, frame: 10 })); + assert_eq!( + ranges(&[ + sub(range(absolute(5, 9), absolute(5, 10))), + sub(range(absolute(5, 2), absolute(5, 3))) + ]), + vec![range(absolute(5, 2), absolute(5, 3)), range(absolute(5, 9), absolute(5, 10))] + ); + assert_eq!( + ranges(&[ + sub(range(absolute(5, 2), absolute(5, 9))), + sub(range(absolute(5, 4), absolute(6, 0))) + ]), + vec![range(absolute(5, 2), absolute(6, 0))] + ); + } - // The whole group is the head of the next one, which outsorts every frame of - // this one, so it absorbs any capped end without a sentinel. - let whole = Subscription::default().with_end(Position::after_group(5)); - let combined = combine(&[long, whole]).unwrap(); - assert_eq!(combined.end, Some(Position::group(6))); + /// Combining is order-independent and stops changing once it covers everyone. + #[test] + fn combined_ranges_settle() { + let a = Subscription::default().with_ranges([Range::groups(2..5), Range::live()]); + let b = Subscription::default().with_ranges([Range::groups(4..9)]); + assert_eq!(combine(&[a.clone(), b.clone()]), combine(&[b.clone(), a.clone()])); - // A later group wins outright, carrying its own frame. - let later_group = Subscription::default().with_end(Position::after(6, 1)); - let combined = combine(&[short, later_group]).unwrap(); - assert_eq!(combined.end, Some(Position { group: 6, frame: 2 })); + let combined = combine(&[a.clone(), b]); + assert!(a.poll_combined(&combined).is_pending(), "a subset adds nothing"); } #[test] @@ -531,14 +773,4 @@ mod tests { ); assert_eq!(combine(&[descending, ascending]).unwrap().order, Order::Descending); } - - #[test] - fn combined_group_end_uses_latest_bounded_end() { - let early = Subscription::default().with_end(Position::after_group(10)); - let late = Subscription::default().with_end(Position::after_group(20)); - - let combined = combine(&[early, late]).unwrap(); - - assert_eq!(combined.end, Some(Position::group(21))); - } } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index cf7eecc97e0..458b43a269a 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -20,7 +20,7 @@ use super::delivered::Delivered; use super::{Datagram, Requests}; use super::Cap; -pub use super::subscription::{Order, Position, Subscription}; +pub use super::subscription::{Anchor, Order, Position, Range, Subscription}; use std::{ collections::{BTreeMap, VecDeque}, @@ -2776,18 +2776,6 @@ fn snapshot_subscription(subs: &kio::Shared, bound: Option u64 { - subscription.start.map(|start| start.group).unwrap_or(0) -} - /// Clamp a drift budget to the publisher's retention window: nobody can wait for a late /// group longer than the publisher keeps it around. /// @@ -3939,6 +3927,9 @@ struct Cursor { datagram_index: usize, /// Minimum sequence to return from any `recv` method. Set by `start_at`. min_sequence: u64, + /// Whether the subscription's floor still waits on the track's first group: a range + /// counting back from the latest group cannot resolve before one exists. + floor_pending: bool, /// One past the highest sequence returned by `next_group`, or `None` once it returned /// `u64::MAX` and no group can follow. /// Used only by that method to skip late arrivals; does not affect `recv_group`. @@ -3976,12 +3967,13 @@ struct Cursor { impl Cursor { fn new(state: kio::Consumer, subscription: kio::Producer) -> Self { - // The budget picks where to start, timed or not: delivery skips the cached groups - // it already judges stale, on whichever clock convicts them. - let (min_sequence, datagram_index) = { + // The ranges floor the cursor, and the budget picks where above it to start, timed + // or not: delivery skips the cached groups it already judges stale, on whichever + // clock convicts them. + let (floor, datagram_index) = { let preferences = subscription.read(); let state = state.read(); - let min_sequence = floor_of(&preferences); + let floor = preferences.floor(state.max_sequence); // Arrival order need not be timestamp order. Only skip the initial stale prefix; // subsequent delivery stays in arrival order, including late datagrams. let newest = state.datagrams.iter().filter_map(|datagram| datagram.timestamp).max(); @@ -3997,12 +3989,13 @@ impl Cursor { }) .expect("the newest datagram is within max delay") }); - (min_sequence, state.datagram_offset + start) + (floor, state.datagram_offset + start) }; Self { state, subscription, - min_sequence, + min_sequence: floor.unwrap_or(0), + floor_pending: floor.is_none(), index: 0, datagram_index, next_sequence: Some(0), @@ -4025,6 +4018,19 @@ impl Cursor { } } + /// Raise the floor to where the subscription's ranges start, once a group exists to + /// count a relative start back from. Returns whether the floor is settled. + fn resolve_floor(&mut self) -> bool { + if self.floor_pending { + let latest = self.state.read().max_sequence; + if let Some(floor) = self.subscription.read().floor(latest) { + self.min_sequence = self.min_sequence.max(floor); + self.floor_pending = false; + } + } + !self.floor_pending + } + // A helper to automatically apply Dropped if the state is closed without an error. fn poll(&self, waiter: &kio::Waiter, f: F) -> Poll> where @@ -4143,6 +4149,7 @@ impl Cursor { } fn poll_recv_group(&mut self, waiter: &kio::Waiter) -> Poll>> { + self.resolve_floor(); if let Some(aborts) = &self.aborts { // A sequence may name both a live group and its fetched head. Only the // consumer we parked decides whether this notification invalidates it. @@ -4199,6 +4206,12 @@ impl Cursor { consumer.cache_refresh(); self.index = found_index + 1; + // The first group to arrive resolves a relative floor, which may land + // above it. + if self.floor_pending && self.resolve_floor() && consumer.sequence < self.min_sequence { + continue; + } + // Park a group beyond the cap instead of dropping it, and keep scanning // so an in-range group that arrived behind it still flows. if !super::subscription::before_end(consumer.sequence, self.end_sequence) { @@ -4234,9 +4247,10 @@ impl Cursor { return Poll::Ready(Ok(None)); }; self.datagram_index = found_index + 1; + self.resolve_floor(); let sequence = datagram.sequence; - let mid_group = self.subscription.read().start.is_some_and(|start| { + let mid_group = self.subscription.read().start().is_some_and(|start| { start.group == sequence && start.frame > datagram.wire.as_ref().and_then(|wire| wire.object_id).unwrap_or(0) }); @@ -4255,6 +4269,7 @@ impl Cursor { let Some(next) = self.next_sequence else { return Poll::Ready(Ok(None)); }; + self.resolve_floor(); let mut floor = next.max(self.min_sequence); let end = self.end_sequence; // One scan for the whole poll, so walking a backlog off stays linear in its size. @@ -4270,6 +4285,12 @@ impl Cursor { }; let group = producer.consume(); + // The first group to arrive resolves a relative floor, which may land above it. + if self.floor_pending && self.resolve_floor() && group.sequence < self.min_sequence { + floor = floor.max(self.min_sequence); + continue; + } + // Skip a group the budget has given up on and keep scanning, so one poll // walks a whole backlog off rather than handing it out group by group. if ready!(self.poll_stale(&group, drift, waiter))? { @@ -4611,6 +4632,7 @@ impl Subscriber { Inner::Resume(resume, _) => return resume.poll_start(waiter), }; let start = ready!(Consumer::poll_state_start(&cursor.state, waiter)); + cursor.resolve_floor(); Poll::Ready(start.map(|start| start.max(cursor.min_sequence))) } @@ -8205,7 +8227,7 @@ mod test { "the widest subscriber changing must wake the aggregate watcher", ); match producer.poll_subscription_changed(&waiter) { - Poll::Ready(Ok(Some(sub))) => assert_eq!(sub.end, Position::after_group(5)), + Poll::Ready(Ok(Some(sub))) => assert_eq!(sub.end(), Position::after_group(5)), other => panic!("expected the narrowed aggregate, got {other:?}"), } } diff --git a/rs/moq-net/tests/announce_to_serve.rs b/rs/moq-net/tests/announce_to_serve.rs index ac28e54e59f..a6d09099b9e 100644 --- a/rs/moq-net/tests/announce_to_serve.rs +++ b/rs/moq-net/tests/announce_to_serve.rs @@ -78,7 +78,7 @@ fn outcome(result: &Result) -> String { /// group's frames and then how the subscription ended. Read continuously, as a /// real subscriber would, so the subscription is driven the whole time. async fn read(broadcast: &moq_net::broadcast::Consumer, name: &str) -> mpsc::UnboundedReceiver { - let subscription = moq_net::track::Subscription::default().with_start(moq_net::track::Position::group(0)); + let subscription = moq_net::track::Subscription::default().with_groups(0..); let mut sub = broadcast .track(name) .unwrap() diff --git a/rs/moq-net/tests/finished_broadcast_mock.rs b/rs/moq-net/tests/finished_broadcast_mock.rs index 6f45072b9a5..ea8eda8873b 100644 --- a/rs/moq-net/tests/finished_broadcast_mock.rs +++ b/rs/moq-net/tests/finished_broadcast_mock.rs @@ -60,7 +60,7 @@ async fn round(finish_broadcast: bool) -> (Vec>, Option) .expect("broadcast resolves"); let reader = moq_net_sim::spawn(async move { - let subscription = moq_net::track::Subscription::default().with_start(moq_net::track::Position::group(0)); + let subscription = moq_net::track::Subscription::default().with_groups(0..); let mut sub = remote .track("video") .unwrap() diff --git a/rs/moq-net/tests/group_order.rs b/rs/moq-net/tests/group_order.rs index 0baba6f72fd..8d28acd0b47 100644 --- a/rs/moq-net/tests/group_order.rs +++ b/rs/moq-net/tests/group_order.rs @@ -11,7 +11,7 @@ mod support; use std::time::Duration; -use moq_net::track::{Info, Order, Position, Subscription}; +use moq_net::track::{Info, Order, Subscription}; use moq_net::{Hop, Timescale, Timestamp, Version}; use support::harness::{MockConnectOptions, MockPair, connect_mock}; @@ -57,7 +57,7 @@ fn read_arrivals(origin: moq_net::origin::Producer, order: Order) -> moq_net_sim .expect("broadcast resolves"); let subscription = Subscription::default() .with_max_delay(FOREVER) - .with_start(Position::group(0)) + .with_groups(0..) .with_order(order); let mut sub = remote .track("ordered") diff --git a/rs/moq-net/tests/head_fetch.rs b/rs/moq-net/tests/head_fetch.rs index 6a44bb04038..2d8143b7161 100644 --- a/rs/moq-net/tests/head_fetch.rs +++ b/rs/moq-net/tests/head_fetch.rs @@ -12,7 +12,7 @@ mod support; use std::time::Duration; -use moq_net::track::{Position, Subscription}; +use moq_net::track::{Anchor, Position, Range, Subscription}; use moq_net::{Hop, Timestamp, Version}; use support::harness::{MockConnectOptions, connect_mock}; @@ -99,7 +99,10 @@ async fn round(version: &str) { let mut peer_sub = peer_remote .track("catalog.json") .unwrap() - .subscribe(Subscription::default().with_start(start)) + .subscribe(Subscription::default().with_ranges([Range { + start: Anchor::Absolute(start), + end: None, + }])) .await .expect("peer subscribe"); let mut peer_group = moq_net_sim::timeout(TIMEOUT, peer_sub.recv_group()) diff --git a/rs/moq-net/tests/history_groups.rs b/rs/moq-net/tests/history_groups.rs index 9bedb5efa54..4668c147126 100644 --- a/rs/moq-net/tests/history_groups.rs +++ b/rs/moq-net/tests/history_groups.rs @@ -12,7 +12,7 @@ mod support; use std::time::Duration; -use moq_net::track::{Info, Position, Subscription}; +use moq_net::track::{Info, Subscription}; use moq_net::{Hop, Timescale, Timestamp, Version}; use support::harness::{MockConnectOptions, connect_mock}; @@ -84,7 +84,7 @@ async fn round(version: &str, relay: bool, newest_first: bool) -> Vec<(u64, usiz let reader = moq_net_sim::spawn(async move { let subscription = Subscription::default() .with_max_delay(FOREVER) - .with_start(Position::group(0)); + .with_groups(0..); let mut sub = remote .track("history") .unwrap() @@ -227,7 +227,7 @@ async fn a_late_subscriber_receives_the_relays_cached_history() { let remote = consumer.request_broadcast("bcast", None).await.unwrap(); let subscription = Subscription::default() .with_max_delay(FOREVER) - .with_start(Position::group(0)); + .with_groups(0..); let first_subscription = subscription.clone(); let first_reader = moq_net_sim::spawn(async move { let mut sub = remote diff --git a/rs/moq-net/tests/late_lower_group.rs b/rs/moq-net/tests/late_lower_group.rs index 98cb74e523a..9406bfadfe4 100644 --- a/rs/moq-net/tests/late_lower_group.rs +++ b/rs/moq-net/tests/late_lower_group.rs @@ -11,7 +11,7 @@ mod support; use std::time::Duration; -use moq_net::track::{Info, Position, Subscription}; +use moq_net::track::{Anchor, Info, Position, Range, Subscription}; use moq_net::{Timestamp, Version}; use support::harness::{MockConnectOptions, connect_mock}; @@ -27,6 +27,14 @@ const LITE: &[&str] = &["moq-lite-05", "moq-lite-06", "moq-lite-07-wip"]; const TRANSPORT: &[&str] = &["moq-transport-14", "moq-transport-17", "moq-transport-22"]; +/// From `start`, or the oldest group still around when there is none. +fn floored(start: Option) -> Range { + Range { + start: start.map_or(Range::oldest().start, Anchor::Absolute), + end: None, + } +} + fn produce_origin(hop: u64) -> moq_net::origin::Producer { let (producer, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::new(moq_net::Hop::new(hop).unwrap())); @@ -112,7 +120,7 @@ async fn session(version: &str, relay: bool, start: Option) -> Vec<(u6 .expect("resolve timeout") .expect("broadcast resolves"); - let subscription = Subscription::default().with_max_delay(BUDGET).with_start(start); + let subscription = Subscription::default().with_max_delay(BUDGET).with_ranges([floored(start)]); let reader = moq_net_sim::spawn(async move { let mut sub = remote .track("late") @@ -154,7 +162,7 @@ async fn in_process(start: Option) -> Vec<(u64, usize)> { .with_max_age(FOREVER), ) .unwrap(); - let subscription = Subscription::default().with_max_delay(BUDGET).with_start(start); + let subscription = Subscription::default().with_max_delay(BUDGET).with_ranges([floored(start)]); let mut sub = track.subscribe(subscription); let reader = moq_net_sim::spawn(async move { read_all(&mut sub).await }); moq_net_sim::sleep(Duration::from_millis(10)).await; diff --git a/rs/moq-net/tests/loom.rs b/rs/moq-net/tests/loom.rs index 4acb7790e3e..71c3f7dca81 100644 --- a/rs/moq-net/tests/loom.rs +++ b/rs/moq-net/tests/loom.rs @@ -102,7 +102,7 @@ fn back_to_back_groups_arrive_in_order() { }); let from_start = moq_net::track::Subscription::default() - .with_start(moq_net::track::Position::group(0)) + .with_groups(0..) .with_max_delay(std::time::Duration::MAX); let mut subscriber = block_on(track_consumer.subscribe(from_start)).expect("subscribe"); let first = block_on(subscriber.recv_group()).expect("recv").expect("first group"); @@ -514,7 +514,7 @@ fn an_aborted_parked_group_releases_a_finished_track() { let group = track.append_group().unwrap(); group.clone().finish().unwrap(); let mut subscriber = - track.subscribe(moq_net::track::Subscription::default().with_start(moq_net::track::Position::group(0))); + track.subscribe(moq_net::track::Subscription::default().with_groups(0..)); subscriber.set_groups(..0); track.finish().unwrap(); let aborting = thread::spawn(move || group.abort(Error::Old).unwrap()); diff --git a/rs/moq-net/tests/quiet_catalog.rs b/rs/moq-net/tests/quiet_catalog.rs index f8f95ed7c98..ba421705bc8 100644 --- a/rs/moq-net/tests/quiet_catalog.rs +++ b/rs/moq-net/tests/quiet_catalog.rs @@ -174,7 +174,7 @@ async fn resume_then_fresh(version: Version, in_process: bool) -> Result<(), Str let _sub = resume_remote .track("catalog.json") .unwrap() - .subscribe(track::Subscription::default().with_start(track::Position::group(1))) + .subscribe(track::Subscription::default().with_groups(1..)) .await .expect("resume subscribe"); moq_net_sim::sleep(Duration::from_secs(60)).await; @@ -182,9 +182,9 @@ async fn resume_then_fresh(version: Version, in_process: bool) -> Result<(), Str // Wait for the resume to reach the publisher, so the fresh reader widens it // rather than opening its own floorless subscription. - let resumed = Some(track::Position::group(1)); + let resumed = vec![track::Range::groups(1..)]; moq_net_sim::timeout(TIMEOUT, async { - while track.subscription().map(|sub| sub.start) != Some(resumed) { + while track.subscription().map(|sub| sub.ranges).as_ref() != Some(&resumed) { track.subscription_changed().await.unwrap(); } }) diff --git a/rs/moq-net/tests/settled_end_survives_abort.rs b/rs/moq-net/tests/settled_end_survives_abort.rs index 91e376ecc7c..1f2722008bf 100644 --- a/rs/moq-net/tests/settled_end_survives_abort.rs +++ b/rs/moq-net/tests/settled_end_survives_abort.rs @@ -32,7 +32,7 @@ async fn round(abort: bool, ordered: bool) -> (Vec, Option) { // From the first group with a replay window: a late reader is owed the whole track, // not the live edge. let subscription = moq_net::track::Subscription::default() - .with_start(moq_net::track::Position::group(0)) + .with_groups(0..) .with_max_delay(std::time::Duration::from_secs(30)); let mut consumer = track.subscribe(subscription); diff --git a/rs/moq-net/tests/subscription_end_integrity.rs b/rs/moq-net/tests/subscription_end_integrity.rs index 26ba2a08acc..eae2d93be94 100644 --- a/rs/moq-net/tests/subscription_end_integrity.rs +++ b/rs/moq-net/tests/subscription_end_integrity.rs @@ -83,7 +83,7 @@ async fn round(drop_session: bool) -> (Vec>, Option) { let (go_tx, go_rx) = futures::channel::oneshot::channel::<()>(); let reader = moq_net_sim::spawn(async move { - let subscription = moq_net::track::Subscription::default().with_start(moq_net::track::Position::group(0)); + let subscription = moq_net::track::Subscription::default().with_groups(0..); let mut sub = remote .track("video") .unwrap() @@ -225,7 +225,7 @@ async fn killed(version: &str, local: bool) -> (Option, Option Outcome { } = connect(version).await; let reader = moq_net_sim::spawn(async move { - let subscription = moq_net::track::Subscription::default().with_start(moq_net::track::Position::group(0)); + let subscription = moq_net::track::Subscription::default().with_groups(0..); let mut sub = remote .track("video") .unwrap() @@ -275,7 +275,7 @@ async fn ietf_leaving_cancels_a_blocked_end_of_track() { _keep, } = connect(version).await; - let subscription = moq_net::track::Subscription::default().with_start(moq_net::track::Position::group(0)); + let subscription = moq_net::track::Subscription::default().with_groups(0..); let mut sub = remote .track("video") .unwrap() @@ -334,7 +334,7 @@ async fn a_lost_datagram_delays_the_end_only_without_a_stream_count() { } = connect(version).await; let reader = moq_net_sim::spawn(async move { - let subscription = moq_net::track::Subscription::default().with_start(moq_net::track::Position::group(0)); + let subscription = moq_net::track::Subscription::default().with_groups(0..); let mut sub = remote .track("video") .unwrap() @@ -397,7 +397,7 @@ async fn a_group_whose_first_frame_trails_its_header_is_delivered() { let reader = moq_net_sim::spawn(async move { let subscription = moq_net::track::Subscription::default() - .with_start(moq_net::track::Position::group(0)) + .with_groups(0..) .with_max_delay(TIMEOUT); let mut sub = remote .track("video") diff --git a/rs/moq-net/tests/untimed.rs b/rs/moq-net/tests/untimed.rs index 883ca9a332c..2c13e03708b 100644 --- a/rs/moq-net/tests/untimed.rs +++ b/rs/moq-net/tests/untimed.rs @@ -194,7 +194,7 @@ async fn an_explicit_start_holds_on_an_untimed_track() { let consumer = chain.nodes[1].consume().request_broadcast("room", None).await.unwrap(); let subscription = track::Subscription::default() - .with_start(track::Position::group(2)) + .with_groups(2..) .with_max_delay(Duration::from_secs(30)); let mut subscriber = moq_net_sim::timeout(TIMEOUT, consumer.track("data").unwrap().subscribe(subscription)) .await From 03e68a9f59c5af319c3fa8f4a809f798f002e2b2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 08:15:06 -0700 Subject: [PATCH 03/20] feat(ffi)!: carry subscription ranges and order through moq-ffi and moq-c Co-Authored-By: Claude Opus 5.5 --- rs/moq-archive/src/reader/tests.rs | 2 +- rs/moq-audio/src/decode/consumer.rs | 2 +- rs/moq-c/src/api.rs | 143 ++++++++++++++++----- rs/moq-c/src/consume.rs | 28 +++- rs/moq-c/src/test.rs | 47 ++++--- rs/moq-ffi/src/consumer.rs | 93 ++++++++++++-- rs/moq-ffi/src/test.rs | 65 +++++++++- rs/moq-hls/src/export/history.rs | 2 +- rs/moq-mux/src/container/consumer.rs | 18 ++- rs/moq-mux/src/container/ts/export_test.rs | 2 +- rs/moq-net/src/lite/subscriber.rs | 2 +- rs/moq-net/src/model/subscription.rs | 4 +- rs/moq-net/src/model/track.rs | 24 ++-- rs/moq-video/src/decode/consumer.rs | 6 +- 14 files changed, 341 insertions(+), 97 deletions(-) diff --git a/rs/moq-archive/src/reader/tests.rs b/rs/moq-archive/src/reader/tests.rs index 217d99f2e22..53b70c3fde2 100644 --- a/rs/moq-archive/src/reader/tests.rs +++ b/rs/moq-archive/src/reader/tests.rs @@ -808,7 +808,7 @@ async fn benchmark_timeline_fetch_fanout() { broadcast.info().pool.gc(now, &kio::Waiter::noop()); } let subscription = moq_net::track::Subscription::default() - .with_start(moq_net::track::Position::group(0)) + .with_groups(0..) .with_max_delay(Duration::from_secs(60)); let mut retained = broadcast .consume() diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index d2b30e0eb1e..caf222c792b 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -167,7 +167,7 @@ impl Consumer { ) .await?; // Latest skips cached history unless decoder warmup needs it. Move the - // local cursor, not Subscription::group_start: the latter is aggregated + // local cursor, not Subscription::ranges: the latter is aggregated // across subscribers and can ask the publisher to rewind for everyone. if options.start == Start::Latest && warmup.is_zero() diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index ea76d27838f..821b00c8bf7 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -557,7 +557,7 @@ pub enum moq_demand { /// Subscriber-side raw track delivery preferences. /// /// A null [moq_consume_track] or [moq_consume_track_update] `subscription` -/// pointer uses the moq-net defaults. +/// pointer uses the moq-net defaults, and so does a zeroed struct. #[repr(C)] #[allow(non_camel_case_types)] pub struct moq_subscription { @@ -566,35 +566,110 @@ pub struct moq_subscription { /// How far a non-latest group may fall behind the live edge before it is skipped, in microseconds. /// Zero skips immediately. Enforced by the publisher's cache and by any local buffering. + /// It caps every range. pub max_delay_us: u64, - /// The lowest group to deliver (a floor). A floor is not a request: `max_delay_us` is - /// what asks for data, and delivery starts at the oldest group at or above the floor - /// within that budget (the latest group at the default budget of 0). - pub group_start: u64, - /// Whether `group_start` is present. When false, there is no floor. - pub group_start_present: bool, - - /// First group not to deliver (exclusive), or ignored when `group_end_present` is - /// false. `0` is the empty range. - pub group_end: u64, - /// Whether `group_end` is present. When false, there is no end cap. - pub group_end_present: bool, -} - -impl From<&moq_subscription> for moq_net::track::Subscription { - fn from(subscription: &moq_subscription) -> Self { - let mut out = moq_net::track::Subscription::default() - .with_priority(subscription.priority) - .with_max_delay(std::time::Duration::from_micros(subscription.max_delay_us)); - if subscription.group_start_present { - out = out.with_start(moq_net::track::Position::group(subscription.group_start)); - } - if subscription.group_end_present { - out = out.with_end(moq_net::track::Position::group(subscription.group_end)); - } - out + /// The spans of the track to deliver, or NULL for everything `max_delay_us` still + /// considers fresh. A non-NULL pointer with `ranges_len` 0 asks for nothing. + /// + /// A request to the publisher, aggregated across every subscriber. Reads start at the + /// lowest group a range starts at, and stop past the last absolute end. + pub ranges: *const moq_range, + pub ranges_len: usize, + + /// `moq_order` discriminant: which pending group the publisher sends first. + pub order: u32, +} + +/// A span of a track to deliver: from `start` up to `end`, or onward without one. +/// +/// A relative start is the first frame of its group, and a relative end includes its whole +/// group. An absolute end is exclusive. +#[repr(C)] +#[allow(non_camel_case_types)] +#[derive(Clone, Copy)] +pub struct moq_range { + /// Where delivery starts. + pub start: moq_anchor, + /// Where delivery stops, read only when `end_present` is true. + pub end: moq_anchor, + /// Whether `end` is present. When false, the range follows the live edge. + pub end_present: bool, +} + +/// Where one end of a [moq_range] falls. Zeroing it means group 0, frame 0. +#[repr(C)] +#[allow(non_camel_case_types)] +#[derive(Clone, Copy)] +pub struct moq_anchor { + /// `moq_anchor_kind` discriminant. + pub kind: u32, + /// The group, or with `MOQ_ANCHOR_RELATIVE` how many groups before the latest one + /// (0 is the latest, and an offset past the first group clamps to it). + pub group: u64, + /// The frame within `group`, read only with `MOQ_ANCHOR_ABSOLUTE`. + pub frame: u64, +} + +/// How a [moq_anchor] names its position. +#[repr(C)] +#[allow(non_camel_case_types)] +#[derive(Clone, Copy, Debug)] +pub enum moq_anchor_kind { + /// A fixed group and frame. + MOQ_ANCHOR_ABSOLUTE = 0, + /// A count of groups back from the latest one. + MOQ_ANCHOR_RELATIVE = 1, +} + +/// Which of a subscription's pending groups the publisher sends first. Zero is the default. +#[repr(C)] +#[allow(non_camel_case_types)] +#[derive(Clone, Copy, Debug)] +pub enum moq_order { + /// Newest first, so congestion sheds the backlog rather than the live edge. + MOQ_ORDER_DESCENDING = 0, + /// Oldest first, draining a backlog in sequence order. + MOQ_ORDER_ASCENDING = 1, +} + +fn parse_anchor(anchor: &moq_anchor) -> Result { + Ok(match anchor.kind { + v if v == moq_anchor_kind::MOQ_ANCHOR_ABSOLUTE as u32 => moq_net::track::Anchor::Absolute(moq_net::track::Position { + group: anchor.group, + frame: anchor.frame, + }), + v if v == moq_anchor_kind::MOQ_ANCHOR_RELATIVE as u32 => moq_net::track::Anchor::Relative(anchor.group), + _ => return Err(Error::InvalidCode), + }) +} + +/// # Safety +/// - `subscription->ranges` must be NULL or point to `subscription->ranges_len` ranges. +pub(crate) unsafe fn parse_subscription(subscription: &moq_subscription) -> Result { + let order = match subscription.order { + v if v == moq_order::MOQ_ORDER_DESCENDING as u32 => moq_net::track::Order::Descending, + v if v == moq_order::MOQ_ORDER_ASCENDING as u32 => moq_net::track::Order::Ascending, + _ => return Err(Error::InvalidCode), + }; + let mut out = moq_net::track::Subscription::default() + .with_priority(subscription.priority) + .with_max_delay(std::time::Duration::from_micros(subscription.max_delay_us)) + .with_order(order); + if !subscription.ranges.is_null() { + let ranges = unsafe { std::slice::from_raw_parts(subscription.ranges, subscription.ranges_len) }; + let ranges = ranges + .iter() + .map(|range| { + Ok(moq_net::track::Range { + start: parse_anchor(&range.start)?, + end: range.end_present.then(|| parse_anchor(&range.end)).transpose()?, + }) + }) + .collect::, Error>>()?; + out = out.with_ranges(ranges); } + Ok(out) } /// A borrowed UTF-8 string slice, NOT NULL terminated. @@ -3557,7 +3632,8 @@ pub extern "C" fn moq_consume_close(consume: u32) -> i32 { /// /// # Safety /// - The caller must ensure that name is a valid pointer to name_len bytes of data. -/// - The caller must ensure that subscription is either NULL or a valid pointer to a [moq_subscription] struct. +/// - The caller must ensure that subscription is either NULL or a valid pointer to a [moq_subscription] struct, +/// whose `ranges` is NULL or points to `ranges_len` ranges. /// - The caller must keep `user_data` valid until the terminal (`<= 0`) `on_frame` callback. #[unsafe(no_mangle)] pub unsafe extern "C" fn moq_consume_track( @@ -3571,7 +3647,9 @@ pub unsafe extern "C" fn moq_consume_track( ffi::enter(move || { let broadcast = ffi::parse_id(broadcast)?; let name = unsafe { ffi::parse_str(name, name_len)? }; - let subscription = unsafe { subscription.as_ref() }.map(moq_net::track::Subscription::from); + let subscription = unsafe { subscription.as_ref() } + .map(|subscription| unsafe { parse_subscription(subscription) }) + .transpose()?; let on_frame = unsafe { ffi::OnStatus::new(user_data, on_frame)? }; State::lock().consume.raw_track(broadcast, name, subscription, on_frame) }) @@ -3584,12 +3662,15 @@ pub unsafe extern "C" fn moq_consume_track( /// Returns a zero on success, or a negative code on failure. /// /// # Safety -/// - The caller must ensure that subscription is either NULL or a valid pointer to a [moq_subscription] struct. +/// - The caller must ensure that subscription is either NULL or a valid pointer to a [moq_subscription] struct, +/// whose `ranges` is NULL or points to `ranges_len` ranges. #[unsafe(no_mangle)] pub unsafe extern "C" fn moq_consume_track_update(track: u32, subscription: *const moq_subscription) -> i32 { ffi::enter(move || { let track = ffi::parse_id(track)?; - let subscription = unsafe { subscription.as_ref() }.map(moq_net::track::Subscription::from); + let subscription = unsafe { subscription.as_ref() } + .map(|subscription| unsafe { parse_subscription(subscription) }) + .transpose()?; State::lock().consume.raw_track_update(track, subscription) }) } diff --git a/rs/moq-c/src/consume.rs b/rs/moq-c/src/consume.rs index 48e2c6acbf4..59ffee9abd4 100644 --- a/rs/moq-c/src/consume.rs +++ b/rs/moq-c/src/consume.rs @@ -677,12 +677,32 @@ impl Consume { fn apply_raw_subscription(track: &mut moq_net::track::Ordered, subscription: Option) { let subscription = subscription.unwrap_or_default(); - let start = subscription.start.map(|start| start.group).or_else(|| track.latest()); + // Reads start where the lowest range does. Without a floor of its own (the oldest + // range, the default) a raw reader joins at the latest group. + let latest = track.latest(); + let start = subscription + .ranges + .iter() + .map(|range| match range.start { + moq_net::track::Anchor::Absolute(start) => Some(start.group), + moq_net::track::Anchor::Relative(u64::MAX) => latest, + moq_net::track::Anchor::Relative(back) => latest.map(|latest| latest.saturating_sub(back)), + }) + .min() + .flatten(); + // They stop past the last absolute end; an open or relative one follows the live edge. + let end = subscription + .ranges + .iter() + .map(|range| match range.end { + Some(moq_net::track::Anchor::Absolute(end)) => Some(end), + _ => None, + }) + .collect::>>() + .map(|ends| ends.into_iter().max().unwrap_or_default()); track.set_groups(( start.map_or(Bound::Unbounded, Bound::Included), - subscription - .end - .map_or(Bound::Unbounded, moq_net::track::Position::group_end), + end.map_or(Bound::Unbounded, moq_net::track::Position::group_end), )); // A closed track makes the update meaningless; the reader already sees the close. let _ = track.update(subscription); diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index b4ce355b71a..1c1537ae691 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -1411,10 +1411,9 @@ fn publish_track_invalid_broadcast() { let subscription = moq_subscription { priority: 1, max_delay_us: 0, - group_start: 0, - group_start_present: false, - group_end: 0, - group_end_present: false, + ranges: std::ptr::null(), + ranges_len: 0, + order: 0, }; assert!(unsafe { moq_consume_track_update(9999, &subscription) } < 0); } @@ -1518,10 +1517,9 @@ fn raw_track_publish_consume() { let subscription = moq_subscription { priority: 0, max_delay_us: 1_000_000, - group_start: 0, - group_start_present: false, - group_end: 0, - group_end_present: false, + ranges: std::ptr::null(), + ranges_len: 0, + order: 0, }; let consumer = id(unsafe { moq_consume_track( @@ -1720,13 +1718,25 @@ fn raw_track_subscription_options_and_update() { let consume = request_broadcast(origin, path); let frame_cb = Callback::new(); + let ranges = [moq_range { + start: moq_anchor { + kind: moq_anchor_kind::MOQ_ANCHOR_ABSOLUTE as u32, + group: 1, + frame: 0, + }, + end: moq_anchor { + kind: moq_anchor_kind::MOQ_ANCHOR_ABSOLUTE as u32, + group: 2, + frame: 0, + }, + end_present: true, + }]; let subscription = moq_subscription { priority: 5, max_delay_us: 25_000, - group_start: 1, - group_start_present: true, - group_end: 2, - group_end_present: true, + ranges: ranges.as_ptr(), + ranges_len: ranges.len(), + order: 0, }; let consumer = id(unsafe { moq_consume_track( @@ -1753,8 +1763,12 @@ fn raw_track_subscription_options_and_update() { assert_eq!(frame.timestamp_us, 20_000); assert_eq!(moq_consume_track_frame_free(frame_id), 0); + let widened = [moq_range { + end: moq_anchor { group: 3, ..ranges[0].end }, + ..ranges[0] + }]; let update = moq_subscription { - group_end: 3, + ranges: widened.as_ptr(), ..subscription }; assert_eq!(unsafe { moq_consume_track_update(consumer, &update) }, 0); @@ -4930,10 +4944,9 @@ fn read_raw_frames(consume: u32, name: &[u8], count: usize) -> Vec> { let subscription = moq_subscription { priority: 0, max_delay_us: 1_000_000, - group_start: 0, - group_start_present: false, - group_end: 0, - group_end_present: false, + ranges: std::ptr::null(), + ranges_len: 0, + order: 0, }; let track = id(unsafe { moq_consume_track( diff --git a/rs/moq-ffi/src/consumer.rs b/rs/moq-ffi/src/consumer.rs index cca2ad2cd15..502ee8fbadc 100644 --- a/rs/moq-ffi/src/consumer.rs +++ b/rs/moq-ffi/src/consumer.rs @@ -22,7 +22,7 @@ fn raw_frame(frame: moq_net::frame::Frame) -> Result { /// Subscriber-side delivery preferences, mirroring [`moq_net::track::Subscription`]. /// /// Construct with the fields you care about; the rest default to moq-net's defaults -/// (priority 0, no staleness tolerance, full group range). +/// (priority 0, no staleness tolerance, everything still fresh, newest first). #[derive(Clone, uniffi::Record)] pub struct MoqSubscription { /// Delivery priority; higher values preempt lower ones under bandwidth contention. @@ -32,19 +32,79 @@ pub struct MoqSubscription { /// `0` skips immediately; a larger value tolerates that much reordering. /// /// Enforced both by the publisher's cache (sent on the wire) and by any local - /// buffering, such as [`MoqMediaTrackConsumer`]'s jitter buffer. + /// buffering, such as [`MoqMediaTrackConsumer`]'s jitter buffer. It caps every range. #[uniffi(default = 0)] pub max_delay_us: u64, - /// The lowest group to deliver (a floor), or null for none. A floor is not a - /// request: `max_delay_us` is what asks for data, and delivery starts at the oldest - /// group at or above the floor within that budget (the latest group at the default - /// budget of 0). + /// The spans of the track to deliver, or null for everything `max_delay_us` still + /// considers fresh. An empty list asks for nothing. + /// + /// A request to the publisher, aggregated across every subscriber. The one place it + /// touches this subscription's reads is where they start: at the lowest group any + /// range starts at. + #[uniffi(default = None)] + pub ranges: Option>, + /// Which pending group the publisher sends first, or null for newest first. #[uniffi(default = None)] - pub group_start: Option, - /// First group not to deliver (exclusive), or null for no end. `Some(0)` is the - /// empty range. + pub order: Option, +} + +/// A span of a track to deliver: from `start` up to `end`, or onward when `end` is null. +/// +/// A relative start is the first frame of its group, and a relative end includes its whole +/// group. An absolute end is exclusive. +#[derive(Clone, Copy, uniffi::Record)] +pub struct MoqRange { + /// Where delivery starts. + pub start: MoqAnchor, + /// Where delivery stops, or null to keep following the live edge. #[uniffi(default = None)] - pub group_end: Option, + pub end: Option, +} + +/// Where one end of a [`MoqRange`] falls, mirroring [`moq_net::track::Anchor`]. +#[derive(Clone, Copy, uniffi::Enum)] +pub enum MoqAnchor { + /// This exact group and frame. + Absolute { group: u64, frame: u64 }, + /// The group this many before the latest, clamped to group 0: `0` is the latest group. + Relative { groups: u64 }, +} + +/// Which of a subscription's pending groups the publisher sends first, mirroring +/// [`moq_net::track::Order`]. +#[derive(Clone, Copy, uniffi::Enum)] +pub enum MoqOrder { + /// Oldest first, draining a backlog in sequence order. + Ascending, + /// Newest first, so congestion sheds the backlog rather than the live edge. + Descending, +} + +impl From for moq_net::track::Anchor { + fn from(anchor: MoqAnchor) -> Self { + match anchor { + MoqAnchor::Absolute { group, frame } => Self::Absolute(moq_net::track::Position { group, frame }), + MoqAnchor::Relative { groups } => Self::Relative(groups), + } + } +} + +impl From for moq_net::track::Range { + fn from(range: MoqRange) -> Self { + Self { + start: range.start.into(), + end: range.end.map(Into::into), + } + } +} + +impl From for moq_net::track::Order { + fn from(order: MoqOrder) -> Self { + match order { + MoqOrder::Ascending => Self::Ascending, + MoqOrder::Descending => Self::Descending, + } + } } /// Options for fetching one past group by sequence. @@ -63,11 +123,16 @@ impl From for moq_net::group::Fetch { impl From for moq_net::track::Subscription { fn from(s: MoqSubscription) -> Self { - moq_net::track::Subscription::default() + let mut subscription = moq_net::track::Subscription::default() .with_priority(s.priority) - .with_max_delay(std::time::Duration::from_micros(s.max_delay_us)) - .with_start(s.group_start.map(moq_net::track::Position::group)) - .with_end(s.group_end.map(moq_net::track::Position::group)) + .with_max_delay(std::time::Duration::from_micros(s.max_delay_us)); + if let Some(ranges) = s.ranges { + subscription = subscription.with_ranges(ranges.into_iter().map(Into::into)); + } + if let Some(order) = s.order { + subscription = subscription.with_order(order.into()); + } + subscription } } diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index 633ec70bba1..5b7da429b52 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -5,7 +5,7 @@ use super::session::{MoqClient, MoqClientConfig, MoqClientTls, MoqSession}; use crate::consumer::MoqBroadcastConsumer; use crate::consumer::MoqFetchGroupOptions; use crate::consumer::MoqGroupConsumer; -use crate::consumer::MoqSubscription; +use crate::consumer::{MoqAnchor, MoqOrder, MoqRange, MoqSubscription}; use crate::consumer::MoqTrackConsumer; use crate::error::{MoqError, MoqProtocolError, MoqProtocolKind}; use crate::flate::{MoqFlateConfig, MoqFlateSnapshotProducer, MoqFlateStreamProducer}; @@ -771,8 +771,11 @@ async fn raw_track_update_does_not_wait_for_pending_read() { consumer.update(MoqSubscription { priority: 10, max_delay_us: 25_000, - group_start: Some(0), - group_end: None, + ranges: Some(vec![MoqRange { + start: MoqAnchor::Absolute { group: 0, frame: 0 }, + end: None, + }]), + order: None, }); let payload = b"updated subscription".to_vec(); @@ -3628,8 +3631,11 @@ async fn raw_track_update_during_pending_group_still_reads_datagram() { consumer.update(MoqSubscription { priority: 10, max_delay_us: 25_000, - group_start: Some(0), - group_end: None, + ranges: Some(vec![MoqRange { + start: MoqAnchor::Absolute { group: 0, frame: 0 }, + end: None, + }]), + order: None, }); let sequence = track.append_datagram(datagram_frame(b"after-update")).unwrap(); @@ -5819,3 +5825,52 @@ async fn shutdown_times_out_on_an_unfinished_track() { server.cancel(); } } + +/// An ascending subscription reaching back over a cached backlog reads it oldest first, +/// and the order and ranges reach moq-net as asked. +#[tokio::test] +async fn ascending_subscription_reads_a_cached_backlog_oldest_first() { + let broadcast = MoqBroadcastProducer::new().unwrap(); + let track = broadcast.publish_track("events".into(), Some(timed_info())).unwrap(); + for timestamp_us in [0, 1_000, 2_000] { + let group = track.append_group().unwrap(); + group + .write_frame(MoqFrame { + payload: b"x".to_vec(), + timestamp_us: Some(timestamp_us), + }) + .unwrap(); + group.finish().unwrap(); + } + + let subscription = MoqSubscription { + priority: 0, + max_delay_us: 10_000_000, + ranges: Some(vec![MoqRange { + start: MoqAnchor::Relative { groups: 2 }, + end: None, + }]), + order: Some(MoqOrder::Ascending), + }; + let converted = moq_net::track::Subscription::from(subscription.clone()); + assert_eq!(converted.order, moq_net::track::Order::Ascending); + assert_eq!( + converted.ranges, + vec![moq_net::track::Range { + start: moq_net::track::Anchor::Relative(2), + end: None, + }] + ); + + let consumer = track.consume(Some(subscription)).unwrap(); + let mut sequences = Vec::new(); + for _ in 0..3 { + let group = tokio::time::timeout(TIMEOUT, consumer.recv_group()) + .await + .expect("timed out waiting for a cached group") + .unwrap() + .expect("expected a group"); + sequences.push(group.sequence()); + } + assert_eq!(sequences, vec![0, 1, 2]); +} diff --git a/rs/moq-hls/src/export/history.rs b/rs/moq-hls/src/export/history.rs index d6e64fcaca6..27139885e3f 100644 --- a/rs/moq-hls/src/export/history.rs +++ b/rs/moq-hls/src/export/history.rs @@ -32,7 +32,7 @@ pub(super) async fn read( // Effectively unbounded, so no history group is skipped as late: 2^53 - 1 ms is the largest // delay every implementation decodes (js/net reads it as a safe integer). let subscription = moq_net::track::Subscription::default() - .with_start(moq_net::track::Position::group(0)) + .with_groups(0..) .with_max_delay(Duration::from_millis((1_u64 << 53) - 1)); let mut live = track.subscribe(subscription).await?.ordered(); let mut decoder = window::Decoder::::new(window::ConsumerConfig::default().with_compression(true)); diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 243e5020b47..4b769596c2f 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -44,8 +44,8 @@ pub(crate) enum Item { /// skipped. A missing sequence gets the same tolerance: there is no way to tell a stream /// that lost the delivery race from one the cache evicted. /// -/// Delivery starts at the [`Subscription::start`](moq_net::track::Subscription::start) -/// floor when one is named, waiting for that group under the budget; without one it +/// Delivery starts at the lowest absolute start among the +/// [`Subscription::ranges`](moq_net::track::Subscription::ranges) when every range names one, waiting for that group under the budget; without one it /// starts wherever the publisher does, adopted from the first served groups. From there /// the same budget is the one rule that catches the consumer up to the live edge, and /// history the publisher no longer serves expires like any other gap, since its reach @@ -156,7 +156,17 @@ impl Consumer { // edge from there. The budget is clamped to the track's retention window, // since history the publisher no longer keeps can't be waited for and would // otherwise stall the catch-up by the excess. - let start = subscription.start.map(|position| position.group); + // Only absolute starts name a floor: a relative one counts back from wherever the + // publisher is, which the first arrivals say. + let start = subscription + .ranges + .iter() + .map(|range| match range.start { + moq_net::track::Anchor::Absolute(start) => Some(start.group), + moq_net::track::Anchor::Relative(_) => None, + }) + .collect::>>() + .and_then(|starts| starts.into_iter().min()); let max_delay = subscription .max_delay .min(track.info().max_age.unwrap_or(std::time::Duration::MAX)); @@ -2112,7 +2122,7 @@ mod tests { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); let consumer_track = track.subscribe( moq_net::track::Subscription::default() - .with_start(moq_net::track::Position::group(0)) + .with_groups(0..) .with_max_delay(Duration::from_millis(500)), ); let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index e77cd97475e..447e75d551b 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -2803,7 +2803,7 @@ async fn read_si_groups(consumer: &moq_net::broadcast::Consumer, name: &str) -> .unwrap() .subscribe( moq_net::track::Subscription::default() - .with_start(moq_net::track::Position::group(0)) + .with_groups(0..) .with_max_delay(Duration::from_secs(5)), ) .await diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 051581646d1..aee63264717 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -2889,7 +2889,7 @@ mod tests { /// The model's exclusive end maps back to the wire's inclusive pair. /// - /// The two disagree deliberately (see [`Subscription::end`]), so this pins the seam: + /// The two disagree deliberately (see [`Position::after_group`]), so this pins the seam: /// an end at the head of a group means the group below it, served whole, while one /// mid-group caps the frame below it. Off by one here would silently drop or /// duplicate a frame at every relay hop. diff --git a/rs/moq-net/src/model/subscription.rs b/rs/moq-net/src/model/subscription.rs index d44e23501fc..127288c5764 100644 --- a/rs/moq-net/src/model/subscription.rs +++ b/rs/moq-net/src/model/subscription.rs @@ -435,8 +435,8 @@ impl Position { /// The position just past `frame` of `group`: the exclusive end that includes it. /// /// `None` when there is no such position, i.e. the very last frame of the very last - /// group. That is past everything, which [`Subscription::end`] spells `None` too, so - /// the two meanings line up and `with_end` can take this directly. + /// group. That is past everything, which an open [`Range::end`] spells `None` too, so + /// the two meanings line up. pub fn after(group: u64, frame: u64) -> Option { match frame.checked_add(1) { Some(frame) => Some(Self { group, frame }), diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 458b43a269a..34dc29a937a 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -3777,10 +3777,9 @@ impl kio::Task for Fetching { /// /// - [`Self::set_groups`] limits **this subscriber's reads**, filtering exactly what /// this handle returns without changing the publisher's demand. -/// - [`Subscription::start`] / [`Subscription::end`], set via [`Self::update`], -/// are a **request to the publisher**. They're aggregated across every live subscriber -/// (earliest start, widest end), so they say what the publisher should send, not what -/// this subscriber sees. +/// - [`Subscription::ranges`], set via [`Self::update`], are a **request to the +/// publisher**. They're aggregated across every live subscriber (their union), so they +/// say what the publisher should send, not what this subscriber sees. /// /// They stay separate because their scopes differ: a subscriber can't filter by the /// aggregate, since another subscriber can widen it, and the publisher can't honor a @@ -3789,7 +3788,8 @@ impl kio::Task for Fetching { /// for. Set both to skip them *and* avoid the transfer. /// /// The one place they meet is where the cursor comes from. A new subscriber's cursor is -/// floored at the group its own subscription named (or 0), and its +/// floored at the lowest group its own ranges start at, a relative start counted back from +/// the latest group (or the first to arrive, before any exists), and its /// [`Subscription::max_delay`] decides what above the floor is worth delivering. Every later /// move is the caller's. pub struct Subscriber { @@ -3949,7 +3949,7 @@ struct Cursor { /// [`Self::end_sequence`], shared with the groups this cursor hands out so their /// expiry measures drift against the same servable window after a cap moves. /// - /// Deliberately not [`Subscription::end`]. That is a *request to the publisher*, + /// Deliberately not where [`Subscription::ranges`] end. That is a *request to the publisher*, /// folded in with every other subscriber's, and it does not filter this handle (see /// [`Subscriber`]): another unbounded subscriber widens the aggregate and the groups /// arrive here anyway. Capping the drift edge with it would pin the live edge at the @@ -4427,7 +4427,7 @@ impl Subscriber { /// skipped rather than handed over, so a single poll walks off a whole backlog. The /// default is [`Duration::ZERO`], which takes the live /// edge and writes the rest off; raise it to read history. [`Self::set_groups`] and - /// [`Subscription::start`] are filters, not exemptions: backfill needs a budget that + /// [`Subscription::ranges`] are filters, not exemptions: backfill needs a budget that /// covers it. [`Consumer::fetch_group`] is the way to ask for one old group outright. /// The budget remains attached to a returned group: if it stalls while newer data /// advances, its pending frame read ends with [`Error::Old`]. @@ -4682,7 +4682,7 @@ impl Subscriber { /// `2..=5` includes groups 2 through 5; `2..5` excludes group 5. An omitted /// start preserves the current floor, and an omitted end removes the cap. /// Groups above the end remain buffered and can be read after raising the cap. - /// This changes only local delivery; use [`Subscription::with_groups`] and + /// This changes only local delivery; use [`Subscription::ranges`] and /// [`Self::update`] to change the requested groups. pub fn set_groups(&mut self, groups: impl RangeBounds) { let (start, end) = super::subscription::sequence_bounds(groups); @@ -4694,7 +4694,7 @@ impl Subscriber { /// /// A local filter, not a request: it doesn't tell the publisher anything, so the /// skipped groups are still delivered and simply not returned. To ask the publisher - /// to start there instead, set [`Subscription::start`] via [`Self::update`]. + /// to start there instead, set [`Subscription::ranges`] via [`Self::update`]. /// See [Local cursor vs wire preference](Self#local-cursor-vs-wire-preference). #[cfg_attr(not(feature = "async"), allow(dead_code))] pub(crate) fn start_at(&mut self, sequence: u64) { @@ -4719,9 +4719,9 @@ impl Subscriber { /// /// The range says whether its sequence is delivered: `..=5` serves through group 5, /// `..5` stops before it. `..0` is the empty range: no group is delivered. - /// [`Position::group_end`] translates a [`Subscription::end`]. + /// [`Position::group_end`] translates a range's absolute end. /// - /// A local filter, not a request; [`Subscription::end`] is the wire-level + /// A local filter, not a request; [`Subscription::ranges`] are the wire-level /// counterpart. See [Local cursor vs wire preference](Self#local-cursor-vs-wire-preference). /// /// Groups beyond the cap are held rather than skipped past, so a later call to @@ -8088,7 +8088,7 @@ mod test { append_at(&mut producer, second * 1000); } - // `Subscription::end` is a request to the publisher, folded in with every other + // A range's end is a request to the publisher, folded in with every other // subscriber's, and it does not filter this handle: the groups above it arrive // anyway. Capping the live edge with it would pin the edge below them, leaving // everything past it with nothing newer to be late against. diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index f47c1443d41..6988c1bc610 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -358,15 +358,15 @@ mod tests { ); // The skip is the local read cursor and nothing else. Asking for it - // through `Subscription::start` would look equivalent and is not: the + // through `Subscription::ranges` would look equivalent and is not: the // floor is aggregated across every live subscriber and tells the // publisher what to send, so naming a cached sequence there rewinds the // track for everyone reading it. A rendition switched away from and back // to is the case that bites, because its cached sequence is stale by // then and the publisher resends the broadcast from it. assert_eq!( - published.subscription().and_then(|sub| sub.start), - None, + published.subscription().map(|sub| sub.ranges), + Some(vec![moq_net::track::Range::oldest()]), "the publisher was asked to rewind the track", ); } From 3fe7b3fafac62cec7a5d81e117a673cf476d6bf3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 08:23:44 -0700 Subject: [PATCH 04/20] test(net): cover merged subscription ranges in-process and through a relay A relative range on an empty track now resolves against the first group to arrive rather than the latest one at the first read, so a live subscriber that reads late still starts at the first group. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 105 ++++++++++++++++++- rs/moq-net/tests/subscribe_ranges.rs | 147 +++++++++++++++++++++++++++ 2 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 rs/moq-net/tests/subscribe_ranges.rs diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 34dc29a937a..f32d0fe812e 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -4022,7 +4022,12 @@ impl Cursor { /// count a relative start back from. Returns whether the floor is settled. fn resolve_floor(&mut self) -> bool { if self.floor_pending { - let latest = self.state.read().max_sequence; + // The track was empty at subscribe, so the latest group then was the first to + // arrive, however many have arrived since. + let latest = { + let state = self.state.read(); + state.arrival.front().map(|(sequence, _)| *sequence).or(state.max_sequence) + }; if let Some(floor) = self.subscription.read().floor(latest) { self.min_sequence = self.min_sequence.max(floor); self.floor_pending = false; @@ -11659,4 +11664,102 @@ mod test { } assert_eq!(subscriber.delivered_runs(), 1000); } + + /// A track whose newest group is `newest`, every group fresh to [`replay`]. + #[cfg(feature = "async")] + fn quiet_track(newest: u64) -> Producer { + let mut producer = track_producer("test", Info::timed()); + for millis in 0..=newest { + append_at(&mut producer, millis); + } + producer + } + + /// A live range merged with one above the live edge still starts at the newest group, + /// and the aggregate keeps both rather than flooring the live one at the higher start. + #[test] + #[cfg(feature = "async")] + fn a_live_range_merged_with_one_above_the_live_edge_gets_the_newest_group() { + let mut producer = quiet_track(3); + let mut ahead = producer.subscribe(replay().with_groups(4..)); + let mut live = producer.subscribe(replay().with_ranges([Range::live()])); + assert_eq!( + producer.subscription().unwrap().ranges, + vec![Range::groups(4..), Range::live()] + ); + assert_eq!(drain(&mut ahead), Vec::::new()); + assert_eq!(drain(&mut live), vec![3]); + + append_at(&mut producer, 4); + assert_eq!(drain(&mut ahead), vec![4]); + assert_eq!(drain(&mut live), vec![4]); + } + + /// A live range on a track with no groups yet starts at the first one to arrive. + #[test] + #[cfg(feature = "async")] + fn a_live_range_on_an_empty_track_starts_at_the_first_group() { + let mut producer = track_producer("test", Info::timed()); + let mut live = producer.subscribe(replay().with_ranges([Range::live()])); + assert_eq!(drain(&mut live), Vec::::new()); + append_at(&mut producer, 0); + append_at(&mut producer, 1); + assert_eq!(drain(&mut live), vec![0, 1]); + } + + /// A relative start buffers that many groups behind the live edge, whatever a range + /// above the edge merged beside it asks for. + #[test] + #[cfg(feature = "async")] + fn a_buffering_range_merged_with_one_above_the_live_edge_keeps_its_backlog() { + let mut producer = quiet_track(5); + let mut ahead = producer.subscribe(replay().with_groups(8..)); + let buffering = Range { + start: Anchor::Relative(2), + end: None, + }; + let mut behind = producer.subscribe(replay().with_ranges([buffering])); + assert_eq!( + producer.subscription().unwrap().ranges, + vec![Range::groups(8..), buffering] + ); + assert_eq!(drain(&mut behind), vec![3, 4, 5]); + + for millis in 6..=8 { + append_at(&mut producer, millis); + } + assert_eq!(drain(&mut behind), vec![6, 7, 8]); + assert_eq!(drain(&mut ahead), vec![8]); + } + + /// A frame-precise start survives the merge with a live range, for the publisher to + /// trim on the wire. Locally it floors reads at its group. + #[test] + #[cfg(feature = "async")] + fn a_frame_precise_start_merged_with_live_keeps_its_frame() { + let producer = quiet_track(3); + let precise = Range { + start: Anchor::Absolute(Position { group: 1, frame: 2 }), + end: None, + }; + let mut from_frame = producer.subscribe(replay().with_ranges([precise])); + let mut live = producer.subscribe(replay().with_ranges([Range::live()])); + assert_eq!(producer.subscription().unwrap().ranges, vec![precise, Range::live()]); + assert_eq!(drain(&mut from_frame), vec![1, 2, 3]); + assert_eq!(drain(&mut live), vec![3]); + } + + /// An untimed track replays from group 0 only what the wall clock hasn't made stale. + #[test] + #[cfg(feature = "async")] + fn an_untimed_range_from_group_zero_replays_what_is_still_fresh() { + let mut producer = untimed_producer(); + for _ in 0..5 { + append_untimed(&mut producer); + elapse(&producer, Duration::from_secs(1)); + } + let budget = Duration::from_millis(1500); + let mut subscriber = producer.subscribe(Subscription::default().with_max_delay(budget).with_groups(0..)); + assert_eq!(drain(&mut subscriber), vec![3, 4]); + } } diff --git a/rs/moq-net/tests/subscribe_ranges.rs b/rs/moq-net/tests/subscribe_ranges.rs new file mode 100644 index 00000000000..a075f9dd867 --- /dev/null +++ b/rs/moq-net/tests/subscribe_ranges.rs @@ -0,0 +1,147 @@ +#![cfg(feature = "async")] + +//! Subscribers asking for different ranges of one track share a single upstream +//! subscription, and each still gets the groups its own range asks for. + +mod support; + +use std::time::Duration; + +use moq_net::track::{Info, Range, Subscription}; +use moq_net::{Timestamp, Version}; +use support::harness::{MockConnectOptions, connect_mock}; + +const TIMEOUT: Duration = Duration::from_secs(10); + +const LITE: &[&str] = &["moq-lite-05", "moq-lite-06", "moq-lite-07-wip"]; + +/// Draft-20 and later are left out: the first subscriber's absolute start goes upstream +/// in the SUBSCRIBE filter, and the subscriber never updates it when a live one joins +/// (`quest/m1/subscribe-ranges/ietf-update.md`), so group 3 never arrives. +const TRANSPORT: &[&str] = &["moq-transport-14", "moq-transport-17"]; + +fn produce_origin(hop: u64) -> moq_net::origin::Producer { + let (producer, driver) = + moq_net::origin::Producer::new(moq_net::origin::Config::new(moq_net::Hop::new(hop).unwrap())); + support::harness::spawn(driver); + producer +} + +fn write_group(track: &moq_net::track::Producer, sequence: u64) { + let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + group.write_frame(Timestamp::now(), &b"x"[..]).unwrap(); + group.finish().unwrap(); +} + +/// The next group's sequence, or `None` once the track ends or nothing arrives in time. +async fn next(sub: &mut moq_net::track::Subscriber) -> Option { + let group = moq_net_sim::timeout(TIMEOUT, sub.recv_group()).await.ok()?; + group.expect("recv_group").map(|group| group.sequence) +} + +async fn read_all(sub: &mut moq_net::track::Subscriber) -> Vec { + let mut got = Vec::new(); + while let Some(sequence) = next(sub).await { + got.push(sequence); + } + got +} + +async fn subscribe(remote: &moq_net::broadcast::Consumer, subscription: Subscription) -> moq_net::track::Subscriber { + remote + .track("quiet") + .unwrap() + .subscribe(subscription) + .await + .expect("subscribe") +} + +/// What a subscriber from group 4 and a live one each read from a quiet track whose +/// newest group is 3, then from group 4 once it arrives. `version` connects them to the +/// publisher, through a relay hop when `relay` is set; `None` reads in-process. The one +/// from group 4 subscribes first, and its request reaches the publisher before the live +/// one joins. +async fn ahead_then_live(version: Option<&str>, relay: bool) -> (Vec, Vec) { + let publisher = produce_origin(1); + let broadcast = publisher.create_broadcast("bcast").unwrap(); + let track = broadcast + .create_track("quiet", Info::default().with_timescale(moq_net::Timescale::MILLI)) + .unwrap(); + broadcast.announce(Default::default()).unwrap(); + for sequence in 0..=3 { + write_group(&track, sequence); + } + + let relay_origin = produce_origin(2); + let subscriber = produce_origin(3); + let mut sessions = Vec::new(); + let remote = match version { + None => broadcast.consume(), + Some(version) => { + let version: Version = version.parse().unwrap(); + if relay { + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(publisher.consume()); + options.client_subscribe = Some(relay_origin.clone()); + sessions.push(connect_mock(options).await); + } + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(match relay { + true => relay_origin.consume(), + false => publisher.consume(), + }); + options.client_subscribe = Some(subscriber.clone()); + sessions.push(connect_mock(options).await); + + let consumer = subscriber.consume(); + moq_net_sim::timeout(TIMEOUT, consumer.routed("bcast")) + .await + .expect("announce timeout") + .expect("routed"); + moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("bcast", None)) + .await + .expect("resolve timeout") + .expect("broadcast resolves") + } + }; + + let mut ahead = subscribe(&remote, Subscription::default().with_groups(4..)).await; + moq_net_sim::timeout(TIMEOUT, track.demand().used()) + .await + .expect("no subscriber appeared") + .unwrap(); + moq_net_sim::sleep(Duration::from_millis(10)).await; + let mut live = subscribe(&remote, Subscription::default().with_ranges([Range::live()])).await; + + // The live subscriber reads group 3 before group 4 supersedes it. + let mut live_got: Vec = next(&mut live).await.into_iter().collect(); + write_group(&track, 4); + track.finish().unwrap(); + + live_got.extend(read_all(&mut live).await); + let ahead_got = read_all(&mut ahead).await; + drop((track, sessions, relay_origin, subscriber, broadcast, publisher)); + (ahead_got, live_got) +} + +/// A subscriber from group 4 merged with a live one doesn't floor the shared subscription +/// at 4: the live one still gets group 3. +#[moq_net_sim::test] +async fn a_live_subscriber_merged_with_one_ahead_still_gets_the_newest_group() { + let want = (vec![4], vec![3, 4]); + let mut failures = Vec::new(); + + let got = ahead_then_live(None, false).await; + if got != want { + failures.push(format!("in-process: {got:?}")); + } + for version in LITE.iter().chain(TRANSPORT) { + for relay in [false, true] { + let got = ahead_then_live(Some(version), relay).await; + if got != want { + failures.push(format!("{version} relay={relay}: {got:?}")); + } + } + } + assert!(failures.is_empty(), "{failures:#?}"); +} From 209a1ff16543b16c50676e3feead67aef862e328 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 08:29:09 -0700 Subject: [PATCH 05/20] perf(net): union subscription ranges once per aggregate Folding each subscriber re-merged the growing aggregate, so N disjoint ranges cost O(N^2 log N): 150ms for 4096. Collect every subscriber's ranges and union them once instead: 110us. track_subscription_ranges sweeps range count and span, and span stays flat. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/benches/track.rs | 37 +++++++++++++++++ rs/moq-net/src/model/subscription.rs | 61 ++++++++++++++-------------- rs/moq-net/src/model/track.rs | 16 +++----- 3 files changed, 73 insertions(+), 41 deletions(-) diff --git a/rs/moq-net/benches/track.rs b/rs/moq-net/benches/track.rs index d2f4c0c1c3b..504a863008b 100644 --- a/rs/moq-net/benches/track.rs +++ b/rs/moq-net/benches/track.rs @@ -41,6 +41,11 @@ //! wall clock: each empty group is handed out like any other, so a walk over the rest of //! the run per group shows up as a slope over it. //! +//! `track_subscription_ranges` aggregates subscribers that each ask for a disjoint range, +//! swept over how many ranges and how many groups each spans: the union must cost the +//! same per range whatever it spans, and fold every subscriber's ranges at once rather +//! than re-merging the aggregate per subscriber, which would show up as a slope over them. +//! //! Run with `cargo bench -p moq-net --bench track`. use std::hint::black_box; @@ -81,6 +86,12 @@ const JOIN: [usize; 3] = [64, 1024, 16384]; /// Viewers that joined and left before the churn, leaving the list room for that many. const PEAK: [usize; 3] = [0, 1024, 16384]; +/// Disjoint ranges in one aggregate, one per subscriber. +const RANGES: [u64; 4] = [1, 16, 256, 4096]; + +/// Groups each aggregated range spans. +const SPAN: [u64; 3] = [1, 64, 4096]; + /// Concurrent publishers sharing one relay-style cache pool. const WRITERS: [usize; 4] = [1, 2, 4, 8]; @@ -519,6 +530,31 @@ fn bench_subscriber_join(c: &mut Criterion) { group.finish(); } +fn bench_subscription_ranges(c: &mut Criterion) { + let mut group = c.benchmark_group("track_subscription_ranges"); + for span in SPAN { + for ranges in RANGES { + let broadcast = broadcast::Info::default().produce(); + let track = broadcast.create_track("bench", None).unwrap(); + // A gap between neighbors keeps every range distinct in the union. + let viewers: Vec<_> = (0..ranges) + .map(|i| { + let start = i * (span + 1); + track.subscribe(track::Subscription::default().with_groups(start..start + span)) + }) + .collect(); + assert_eq!(track.subscription().unwrap().ranges.len(), ranges as usize); + + group.throughput(Throughput::Elements(ranges)); + group.bench_function(BenchmarkId::new(format!("span_{span}"), ranges), |b| { + b.iter(|| black_box(track.subscription())); + }); + drop((viewers, track, broadcast)); + } + } + group.finish(); +} + /// Late joins scan a bounded datagram ring; sweep both the backlog and the joining audience. fn bench_datagram_subscriber_join(c: &mut Criterion) { let mut group = c.benchmark_group("track_datagram_subscriber_join"); @@ -917,6 +953,7 @@ criterion_group!( bench_subscriber_churn, bench_subscriber_churn_after_peak, bench_subscriber_join, + bench_subscription_ranges, bench_datagram_subscriber_join, bench_parked_read, bench_sparse_read, diff --git a/rs/moq-net/src/model/subscription.rs b/rs/moq-net/src/model/subscription.rs index 127288c5764..3293c957b8b 100644 --- a/rs/moq-net/src/model/subscription.rs +++ b/rs/moq-net/src/model/subscription.rs @@ -1,6 +1,5 @@ use std::{ ops::{Bound, RangeBounds, RangeFull, RangeTo, RangeToInclusive}, - task::Poll, time::Duration, }; @@ -375,34 +374,36 @@ impl Subscription { } Some(floor.unwrap_or(0)) } +} - // Fold this subscription into the running aggregate: Ready with the merged - // result when it demands more than `combined`, Pending when it's a subset - // (so callers can skip a redundant broadcast of the same aggregate). - pub(super) fn poll_combined(&self, combined: &Option) -> Poll { - let Some(combined) = combined else { - return Poll::Ready(Subscription { - ranges: union(self.ranges.iter().copied()), - ..self.clone() - }); - }; - - let merged = Subscription { - priority: self.priority.max(combined.priority), - // Sequence-first prioritization is enabled only when every subscriber wants it. - max_delay: self.max_delay.max(combined.max_delay), - ranges: union(self.ranges.iter().chain(&combined.ranges).copied()), - order: match (self.order, combined.order) { - (Order::Ascending, Order::Ascending) => Order::Ascending, - _ => Order::Descending, - }, +/// The most demanding request across subscribers, folded in one at a time. +/// +/// Ranges are only collected per subscriber and merged once in [`Self::finish`], so an +/// aggregate of many disjoint ranges costs one union rather than one per subscriber. +#[derive(Default)] +pub(super) struct Combined(Option); + +impl Combined { + /// Fold in one subscriber's preferences. + pub(super) fn add(&mut self, sub: &Subscription) { + let Some(combined) = &mut self.0 else { + self.0 = Some(sub.clone()); + return; }; - - if &merged != combined { - return Poll::Ready(merged); + combined.priority = combined.priority.max(sub.priority); + combined.max_delay = combined.max_delay.max(sub.max_delay); + combined.ranges.extend_from_slice(&sub.ranges); + // Sequence-first prioritization is enabled only when every subscriber wants it. + if let Order::Descending = sub.order { + combined.order = Order::Descending; } + } - Poll::Pending + /// The aggregate, or `None` without a subscriber. + pub(super) fn finish(self) -> Option { + let mut combined = self.0?; + combined.ranges = union(std::mem::take(&mut combined.ranges)); + Some(combined) } } @@ -540,13 +541,11 @@ mod tests { use super::*; fn combine(subscriptions: &[Subscription]) -> Option { - let mut combined = None; + let mut combined = Combined::default(); for sub in subscriptions { - if let Poll::Ready(merged) = sub.poll_combined(&combined) { - combined = Some(merged); - } + combined.add(sub); } - combined + combined.finish() } fn absolute(group: u64, frame: u64) -> Anchor { @@ -751,7 +750,7 @@ mod tests { assert_eq!(combine(&[a.clone(), b.clone()]), combine(&[b.clone(), a.clone()])); let combined = combine(&[a.clone(), b]); - assert!(a.poll_combined(&combined).is_pending(), "a subset adds nothing"); + assert_eq!(combine(&[combined.clone().unwrap(), a]), combined, "a subset adds nothing"); } #[test] diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index f32d0fe812e..e95780794be 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -2735,7 +2735,7 @@ fn combined_subscription( bound: Option, waiter: &kio::Waiter, ) -> (Option, bool) { - let mut combined = None; + let mut combined = super::subscription::Combined::default(); let mut departed = false; for sub in subs.iter() { // A closed consumer means the subscriber dropped: it holds no live demand. @@ -2754,26 +2754,22 @@ fn combined_subscription( // upstream parked at that cap for good. let _ = sub.poll_closed(waiter); let _ = sub.poll(waiter, |_| Poll::<()>::Pending); - if let Poll::Ready(merged) = sub.read().poll_combined(&combined) { - combined = Some(merged); - } + combined.add(&sub.read()); } - (clamp_combined(combined, bound), departed) + (clamp_combined(combined.finish(), bound), departed) } /// A non-blocking aggregate of the current subscriptions, without arming any waiter. fn snapshot_subscription(subs: &kio::Shared, bound: Option) -> Option { - let mut combined: Option = None; + let mut combined = super::subscription::Combined::default(); for sub in subs.read().iter() { // Skip dropped subscribers, matching `combined_subscription`. if sub.is_closed() { continue; } - if let Poll::Ready(merged) = sub.read().poll_combined(&combined) { - combined = Some(merged); - } + combined.add(&sub.read()); } - clamp_combined(combined, bound) + clamp_combined(combined.finish(), bound) } /// Clamp a drift budget to the publisher's retention window: nobody can wait for a late From 39ee36976987201b684d3a0516583713e1be76e5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 08:51:21 -0700 Subject: [PATCH 06/20] feat(bindings)!: carry subscription ranges and order through every wrapper Python, Go, Kotlin, Dart, C++ and Swift replace group_start/group_end with ranges and order, alias Range, Anchor and Order, and test that an ascending subscription reads a cached backlog oldest first. Swift nests the aliases under Subscription, since a top-level Range would shadow Swift.Range. Co-Authored-By: Claude Opus 5.5 --- cpp/moq/include/moq/moq.hpp | 3 + dart/moq/lib/src/aliases.dart | 11 +- dart/moq/test/moq_test.dart | 33 ++ dart/moq_ffi/lib/src/moq.dart | 453 +++++++++++++++++- doc/concept/moq-lite.md | 6 + go/wrapper/moq_test.go | 53 ++ go/wrapper/records.go | 22 +- go/wrapper/types.go | 20 + .../kotlin/dev/moq/Aliases.kt | 12 +- .../kotlin/dev/moq/SmokeTest.kt | 21 + py/moq-rs/moq/__init__.py | 6 + py/moq-rs/moq/_records.py | 15 +- py/moq-rs/moq/types.py | 12 + py/moq-rs/tests/test_local.py | 39 +- swift/Sources/Moq/Aliases.swift | 16 +- swift/Tests/MoqTests/SmokeTests.swift | 21 + 16 files changed, 699 insertions(+), 44 deletions(-) diff --git a/cpp/moq/include/moq/moq.hpp b/cpp/moq/include/moq/moq.hpp index 67a403ee551..f28c4de9ff5 100644 --- a/cpp/moq/include/moq/moq.hpp +++ b/cpp/moq/include/moq/moq.hpp @@ -47,6 +47,7 @@ namespace moq::detail { namespace moq { // Every generated `moq::MoqFoo` is also `moq::Foo`. `just cpp check` fails when one is missing. +using Anchor = MoqAnchor; using AnnounceConsumer = MoqAnnounceConsumer; using AnnouncedBroadcast = MoqAnnouncedBroadcast; using AudioCodec = MoqAudioCodec; @@ -119,8 +120,10 @@ using MediaTrackConfig = MoqMediaTrackConfig; using MediaGroupConfig = MoqMediaGroupConfig; using MediaFrame = MoqMediaFrame; using OriginConfig = MoqOriginConfig; +using Order = MoqOrder; using ProtocolError = MoqProtocolError; using QuicConfig = MoqQuicConfig; +using Range = MoqRange; using Route = MoqRoute; using ServerConfig = MoqServerConfig; using ServerTls = MoqServerTls; diff --git a/dart/moq/lib/src/aliases.dart b/dart/moq/lib/src/aliases.dart index e98f7ca7f33..ed46400d648 100644 --- a/dart/moq/lib/src/aliases.dart +++ b/dart/moq/lib/src/aliases.dart @@ -111,9 +111,18 @@ typedef Datagram = MoqDatagram; /// A payload plus the timestamp it should be presented at. typedef Frame = MoqFrame; -/// Tunes how a track subscription is delivered: priority, group ordering, and range. +/// Tunes how a track subscription is delivered: priority, latency budget, ranges, and group ordering. typedef Subscription = MoqSubscription; +/// A span of a track to deliver: from `start` up to `end`, or onward when `end` is null. +typedef Range = MoqRange; + +/// Where one end of a [Range] falls: absolute or relative to the latest group. +typedef Anchor = MoqAnchor; + +/// Which pending group the publisher sends first. +typedef Order = MoqOrder; + /// Options for fetching one past group by sequence. typedef FetchGroupOptions = MoqFetchGroupOptions; diff --git a/dart/moq/test/moq_test.dart b/dart/moq/test/moq_test.dart index 8793cfc4d29..f25353b275a 100644 --- a/dart/moq/test/moq_test.dart +++ b/dart/moq/test/moq_test.dart @@ -113,6 +113,39 @@ void main() { broadcast.close(); }); + test( + 'an ascending subscription reads a cached backlog oldest first', + () async { + final broadcast = BroadcastProducer(); + final track = broadcast.publishTrack( + name: 'backlog', + info: TrackInfo(timescale: 1000000), + ); + for (var ms = 0; ms < 3; ms++) { + final group = track.appendGroup(); + group.writeFrame( + frame: Frame(payload: utf8.encode('x'), timestampUs: ms * 1000), + ); + group.finish(); + } + + final consumer = track.consume( + subscription: Subscription( + maxDelayUs: 10000000, + ranges: [Range(start: RelativeMoqAnchor(2))], + order: Order.ascending, + ), + ); + final sequences = []; + for (var i = 0; i < 3; i++) { + final group = await consumer.recvGroup().timeout(timeout); + sequences.add(group!.sequence()); + } + expect(sequences, [0, 1, 2]); + broadcast.close(); + }, + ); + test('connects, announces, subscribes, and delivers a frame', () async { final relay = MoqOriginProducer(config: MoqOriginConfig()); final server = MoqServer( diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index dc820ae71be..8db54e35d0d 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -8,7 +8,9 @@ import "dart:ffi"; import "dart:io" show Platform, File, Directory; import "dart:isolate"; import "dart:typed_data"; + import "package:ffi/ffi.dart"; + import "uniffi_runtime.dart"; export "uniffi_runtime.dart"; @@ -56,16 +58,75 @@ class FfiConverterMoqFetchGroupOptions { } } +class MoqRange { + final MoqAnchor start; + final MoqAnchor? end; + MoqRange({required this.start, this.end = null}); +} + +class FfiConverterMoqRange { + static MoqRange lift(RustBuffer buf) { + return FfiConverterMoqRange.read(buf.asUint8List()).value; + } + + static LiftRetVal read(Uint8List buf) { + int new_offset = buf.offsetInBytes; + final start_lifted = FfiConverterMoqAnchor.read( + Uint8List.view(buf.buffer, new_offset), + ); + final start = start_lifted.value; + new_offset += start_lifted.bytesRead; + final end_lifted = FfiConverterOptionalMoqAnchor.read( + Uint8List.view(buf.buffer, new_offset), + ); + final end = end_lifted.value; + new_offset += end_lifted.bytesRead; + return LiftRetVal( + MoqRange(start: start, end: end), + new_offset - buf.offsetInBytes, + ); + } + + static RustBuffer lower(MoqRange value) { + final total_length = + FfiConverterMoqAnchor.allocationSize(value.start) + + FfiConverterOptionalMoqAnchor.allocationSize(value.end) + + 0; + final buf = Uint8List(total_length); + write(value, buf); + return toRustBuffer(buf); + } + + static int write(MoqRange value, Uint8List buf) { + int new_offset = buf.offsetInBytes; + new_offset += FfiConverterMoqAnchor.write( + value.start, + Uint8List.view(buf.buffer, new_offset), + ); + new_offset += FfiConverterOptionalMoqAnchor.write( + value.end, + Uint8List.view(buf.buffer, new_offset), + ); + return new_offset - buf.offsetInBytes; + } + + static int allocationSize(MoqRange value) { + return FfiConverterMoqAnchor.allocationSize(value.start) + + FfiConverterOptionalMoqAnchor.allocationSize(value.end) + + 0; + } +} + class MoqSubscription { final int priority; final int maxDelayUs; - final int? groupStart; - final int? groupEnd; + final List? ranges; + final MoqOrder? order; MoqSubscription({ this.priority = 0, this.maxDelayUs = 0, - this.groupStart = null, - this.groupEnd = null, + this.ranges = null, + this.order = null, }); } @@ -86,22 +147,22 @@ class FfiConverterMoqSubscription { ); final maxDelayUs = maxDelayUs_lifted.value; new_offset += maxDelayUs_lifted.bytesRead; - final groupStart_lifted = FfiConverterOptionalUInt64.read( + final ranges_lifted = FfiConverterOptionalSequenceMoqRange.read( Uint8List.view(buf.buffer, new_offset), ); - final groupStart = groupStart_lifted.value; - new_offset += groupStart_lifted.bytesRead; - final groupEnd_lifted = FfiConverterOptionalUInt64.read( + final ranges = ranges_lifted.value; + new_offset += ranges_lifted.bytesRead; + final order_lifted = FfiConverterOptionalMoqOrder.read( Uint8List.view(buf.buffer, new_offset), ); - final groupEnd = groupEnd_lifted.value; - new_offset += groupEnd_lifted.bytesRead; + final order = order_lifted.value; + new_offset += order_lifted.bytesRead; return LiftRetVal( MoqSubscription( priority: priority, maxDelayUs: maxDelayUs, - groupStart: groupStart, - groupEnd: groupEnd, + ranges: ranges, + order: order, ), new_offset - buf.offsetInBytes, ); @@ -111,8 +172,8 @@ class FfiConverterMoqSubscription { final total_length = FfiConverterUInt8.allocationSize(value.priority) + FfiConverterUInt64.allocationSize(value.maxDelayUs) + - FfiConverterOptionalUInt64.allocationSize(value.groupStart) + - FfiConverterOptionalUInt64.allocationSize(value.groupEnd) + + FfiConverterOptionalSequenceMoqRange.allocationSize(value.ranges) + + FfiConverterOptionalMoqOrder.allocationSize(value.order) + 0; final buf = Uint8List(total_length); write(value, buf); @@ -129,12 +190,12 @@ class FfiConverterMoqSubscription { value.maxDelayUs, Uint8List.view(buf.buffer, new_offset), ); - new_offset += FfiConverterOptionalUInt64.write( - value.groupStart, + new_offset += FfiConverterOptionalSequenceMoqRange.write( + value.ranges, Uint8List.view(buf.buffer, new_offset), ); - new_offset += FfiConverterOptionalUInt64.write( - value.groupEnd, + new_offset += FfiConverterOptionalMoqOrder.write( + value.order, Uint8List.view(buf.buffer, new_offset), ); return new_offset - buf.offsetInBytes; @@ -143,8 +204,8 @@ class FfiConverterMoqSubscription { static int allocationSize(MoqSubscription value) { return FfiConverterUInt8.allocationSize(value.priority) + FfiConverterUInt64.allocationSize(value.maxDelayUs) + - FfiConverterOptionalUInt64.allocationSize(value.groupStart) + - FfiConverterOptionalUInt64.allocationSize(value.groupEnd) + + FfiConverterOptionalSequenceMoqRange.allocationSize(value.ranges) + + FfiConverterOptionalMoqOrder.allocationSize(value.order) + 0; } } @@ -3015,6 +3076,178 @@ class FfiConverterMoqWebSocketConfig { } } +abstract class MoqAnchor { + RustBuffer lower(); + int allocationSize(); + int write(Uint8List buf); +} + +class FfiConverterMoqAnchor { + static MoqAnchor lift(RustBuffer buffer) { + return FfiConverterMoqAnchor.read(buffer.asUint8List()).value; + } + + static LiftRetVal read(Uint8List buf) { + final index = buf.buffer.asByteData(buf.offsetInBytes).getInt32(0); + final subview = Uint8List.view(buf.buffer, buf.offsetInBytes + 4); + switch (index) { + case 1: + final lifted = AbsoluteMoqAnchor.read(subview); + return LiftRetVal( + lifted.value, + lifted.bytesRead - subview.offsetInBytes + 4, + ); + case 2: + final lifted = RelativeMoqAnchor.read(subview); + return LiftRetVal( + lifted.value, + lifted.bytesRead - subview.offsetInBytes + 4, + ); + default: + throw UniffiInternalError( + UniffiInternalError.unexpectedEnumCase, + "Unable to determine enum variant", + ); + } + } + + static RustBuffer lower(MoqAnchor value) { + return value.lower(); + } + + static int allocationSize(MoqAnchor value) { + return value.allocationSize(); + } + + static int write(MoqAnchor value, Uint8List buf) { + return value.write(buf) - buf.offsetInBytes; + } +} + +class AbsoluteMoqAnchor extends MoqAnchor { + final int group; + final int frame; + AbsoluteMoqAnchor({required int this.group, required int this.frame}); + AbsoluteMoqAnchor._(int this.group, int this.frame); + static LiftRetVal read(Uint8List buf) { + int new_offset = buf.offsetInBytes; + final group_lifted = FfiConverterUInt64.read( + Uint8List.view(buf.buffer, new_offset), + ); + final group = group_lifted.value; + new_offset += group_lifted.bytesRead; + final frame_lifted = FfiConverterUInt64.read( + Uint8List.view(buf.buffer, new_offset), + ); + final frame = frame_lifted.value; + new_offset += frame_lifted.bytesRead; + return LiftRetVal(AbsoluteMoqAnchor._(group, frame), new_offset); + } + + @override + RustBuffer lower() { + final buf = Uint8List(allocationSize()); + write(buf); + return toRustBuffer(buf); + } + + @override + int allocationSize() { + return FfiConverterUInt64.allocationSize(group) + + FfiConverterUInt64.allocationSize(frame) + + 4; + } + + @override + int write(Uint8List buf) { + buf.buffer.asByteData(buf.offsetInBytes).setInt32(0, 1); + int new_offset = buf.offsetInBytes + 4; + new_offset += FfiConverterUInt64.write( + group, + Uint8List.view(buf.buffer, new_offset), + ); + new_offset += FfiConverterUInt64.write( + frame, + Uint8List.view(buf.buffer, new_offset), + ); + return new_offset; + } +} + +class RelativeMoqAnchor extends MoqAnchor { + final int groups; + RelativeMoqAnchor(int this.groups); + RelativeMoqAnchor._(int this.groups); + static LiftRetVal read(Uint8List buf) { + int new_offset = buf.offsetInBytes; + final groups_lifted = FfiConverterUInt64.read( + Uint8List.view(buf.buffer, new_offset), + ); + final groups = groups_lifted.value; + new_offset += groups_lifted.bytesRead; + return LiftRetVal(RelativeMoqAnchor._(groups), new_offset); + } + + @override + RustBuffer lower() { + final buf = Uint8List(allocationSize()); + write(buf); + return toRustBuffer(buf); + } + + @override + int allocationSize() { + return FfiConverterUInt64.allocationSize(groups) + 4; + } + + @override + int write(Uint8List buf) { + buf.buffer.asByteData(buf.offsetInBytes).setInt32(0, 2); + int new_offset = buf.offsetInBytes + 4; + new_offset += FfiConverterUInt64.write( + groups, + Uint8List.view(buf.buffer, new_offset), + ); + return new_offset; + } +} + +enum MoqOrder { ascending, descending } + +class FfiConverterMoqOrder { + static LiftRetVal read(Uint8List buf) { + final index = buf.buffer.asByteData(buf.offsetInBytes).getInt32(0); + switch (index) { + case 1: + return LiftRetVal(MoqOrder.ascending, 4); + case 2: + return LiftRetVal(MoqOrder.descending, 4); + default: + throw UniffiInternalError( + UniffiInternalError.unexpectedEnumCase, + "Unable to determine enum variant", + ); + } + } + + static MoqOrder lift(RustBuffer buffer) { + return FfiConverterMoqOrder.read(buffer.asUint8List()).value; + } + + static RustBuffer lower(MoqOrder input) { + return toRustBuffer(createUint8ListFromInt(input.index + 1)); + } + + static int allocationSize(MoqOrder _value) { + return 4; + } + + static int write(MoqOrder value, Uint8List buf) { + buf.buffer.asByteData(buf.offsetInBytes).setInt32(0, value.index + 1); + return 4; + } +} + abstract class MoqException implements Exception { RustBuffer lower(); int allocationSize(); @@ -9836,6 +10069,51 @@ class FfiConverterOptionalDouble64 { } } +class FfiConverterOptionalMoqAnchor { + static MoqAnchor? lift(RustBuffer buf) { + return FfiConverterOptionalMoqAnchor.read(buf.asUint8List()).value; + } + + static LiftRetVal read(Uint8List buf) { + if (ByteData.view(buf.buffer, buf.offsetInBytes).getInt8(0) == 0) { + return LiftRetVal(null, 1); + } + final result = FfiConverterMoqAnchor.read( + Uint8List.view(buf.buffer, buf.offsetInBytes + 1), + ); + return LiftRetVal(result.value, result.bytesRead + 1); + } + + static int allocationSize([MoqAnchor? value]) { + if (value == null) { + return 1; + } + return FfiConverterMoqAnchor.allocationSize(value) + 1; + } + + static RustBuffer lower(MoqAnchor? value) { + if (value == null) { + return toRustBuffer(Uint8List.fromList([0])); + } + final buf = Uint8List(FfiConverterOptionalMoqAnchor.allocationSize(value)); + FfiConverterOptionalMoqAnchor.write(value, buf); + return toRustBuffer(buf); + } + + static int write(MoqAnchor? value, Uint8List buf) { + if (value == null) { + buf[0] = 0; + return 1; + } + buf[0] = 1; + return FfiConverterMoqAnchor.write( + value, + Uint8List.view(buf.buffer, buf.offsetInBytes + 1), + ) + + 1; + } +} + class FfiConverterOptionalMoqAnnounceEvent { static MoqAnnounceEvent? lift(RustBuffer buf) { return FfiConverterOptionalMoqAnnounceEvent.read(buf.asUint8List()).value; @@ -10260,6 +10538,51 @@ class FfiConverterOptionalMoqMediaTimebase { } } +class FfiConverterOptionalMoqOrder { + static MoqOrder? lift(RustBuffer buf) { + return FfiConverterOptionalMoqOrder.read(buf.asUint8List()).value; + } + + static LiftRetVal read(Uint8List buf) { + if (ByteData.view(buf.buffer, buf.offsetInBytes).getInt8(0) == 0) { + return LiftRetVal(null, 1); + } + final result = FfiConverterMoqOrder.read( + Uint8List.view(buf.buffer, buf.offsetInBytes + 1), + ); + return LiftRetVal(result.value, result.bytesRead + 1); + } + + static int allocationSize([MoqOrder? value]) { + if (value == null) { + return 1; + } + return FfiConverterMoqOrder.allocationSize(value) + 1; + } + + static RustBuffer lower(MoqOrder? value) { + if (value == null) { + return toRustBuffer(Uint8List.fromList([0])); + } + final buf = Uint8List(FfiConverterOptionalMoqOrder.allocationSize(value)); + FfiConverterOptionalMoqOrder.write(value, buf); + return toRustBuffer(buf); + } + + static int write(MoqOrder? value, Uint8List buf) { + if (value == null) { + buf[0] = 0; + return 1; + } + buf[0] = 1; + return FfiConverterMoqOrder.write( + value, + Uint8List.view(buf.buffer, buf.offsetInBytes + 1), + ) + + 1; + } +} + class FfiConverterOptionalMoqOriginProducer { static MoqOriginProducer? lift(RustBuffer buf) { return FfiConverterOptionalMoqOriginProducer.read(buf.asUint8List()).value; @@ -10493,6 +10816,53 @@ class FfiConverterOptionalMoqVideoHint { } } +class FfiConverterOptionalSequenceMoqRange { + static List? lift(RustBuffer buf) { + return FfiConverterOptionalSequenceMoqRange.read(buf.asUint8List()).value; + } + + static LiftRetVal?> read(Uint8List buf) { + if (ByteData.view(buf.buffer, buf.offsetInBytes).getInt8(0) == 0) { + return LiftRetVal(null, 1); + } + final result = FfiConverterSequenceMoqRange.read( + Uint8List.view(buf.buffer, buf.offsetInBytes + 1), + ); + return LiftRetVal?>(result.value, result.bytesRead + 1); + } + + static int allocationSize([List? value]) { + if (value == null) { + return 1; + } + return FfiConverterSequenceMoqRange.allocationSize(value) + 1; + } + + static RustBuffer lower(List? value) { + if (value == null) { + return toRustBuffer(Uint8List.fromList([0])); + } + final buf = Uint8List( + FfiConverterOptionalSequenceMoqRange.allocationSize(value), + ); + FfiConverterOptionalSequenceMoqRange.write(value, buf); + return toRustBuffer(buf); + } + + static int write(List? value, Uint8List buf) { + if (value == null) { + buf[0] = 0; + return 1; + } + buf[0] = 1; + return FfiConverterSequenceMoqRange.write( + value, + Uint8List.view(buf.buffer, buf.offsetInBytes + 1), + ) + + 1; + } +} + class FfiConverterOptionalSequenceString { static List? lift(RustBuffer buf) { return FfiConverterOptionalSequenceString.read(buf.asUint8List()).value; @@ -10720,6 +11090,49 @@ class FfiConverterOptionalUint8List { } } +class FfiConverterSequenceMoqRange { + static List lift(RustBuffer buf) { + return FfiConverterSequenceMoqRange.read(buf.asUint8List()).value; + } + + static LiftRetVal> read(Uint8List buf) { + List res = []; + final length = buf.buffer.asByteData(buf.offsetInBytes).getInt32(0); + int offset = buf.offsetInBytes + 4; + for (var i = 0; i < length; i++) { + final ret = FfiConverterMoqRange.read(Uint8List.view(buf.buffer, offset)); + offset += ret.bytesRead; + res.add(ret.value); + } + return LiftRetVal(res, offset - buf.offsetInBytes); + } + + static int write(List value, Uint8List buf) { + buf.buffer.asByteData(buf.offsetInBytes).setInt32(0, value.length); + int offset = buf.offsetInBytes + 4; + for (var i = 0; i < value.length; i++) { + offset += FfiConverterMoqRange.write( + value[i], + Uint8List.view(buf.buffer, offset), + ); + } + return offset - buf.offsetInBytes; + } + + static int allocationSize(List value) { + return value + .map((l) => FfiConverterMoqRange.allocationSize(l)) + .fold(0, (a, b) => a + b) + + 4; + } + + static RustBuffer lower(List value) { + final buf = Uint8List(allocationSize(value)); + write(value, buf); + return toRustBuffer(buf); + } +} + class FfiConverterSequenceString { static List lift(RustBuffer buf) { return FfiConverterSequenceString.read(buf.asUint8List()).value; diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 9e6d534668d..907a062db5e 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -198,6 +198,12 @@ Each subscription carries the knobs that decide behavior under congestion: | **Priority** (0..255) | Higher-priority tracks get bandwidth first. Audio above video, base layer above enhancement. | | **Order** | Which group to send first when several are pending. Newest first for live (the default), oldest first for catch-up. Needs moq-lite 07 or moq-transport; older moq-lite always sends newest first. | | **Max delay** | How long a non-latest group stays worth delivering before it is skipped. Zero means "live edge only", and raising it is also what asks for history. | +| **Ranges** | Which groups to ask for: from a group, or a number of groups back from the latest, optionally up to an end. The default asks for everything max delay allows. Max delay still caps every range. | + +Ranges are a request, not a filter. A subscriber reads from where its lowest +range starts, and may also get groups another subscriber of the same track +asked for. A relay merges its subscribers' ranges into one upstream request, +which today's wire versions carry as the single span covering them. Order only decides what goes first, never what is skipped, so catching up oldest first needs a max delay that covers the backlog. When a relay serves subscribers of diff --git a/go/wrapper/moq_test.go b/go/wrapper/moq_test.go index 30d6a37f5d1..db9136b6e49 100644 --- a/go/wrapper/moq_test.go +++ b/go/wrapper/moq_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "runtime" + "slices" "sync" "testing" "time" @@ -653,6 +654,58 @@ func TestTrackPublishConsume(t *testing.T) { } } +func TestAscendingSubscriptionReadsCachedBacklogOldestFirst(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + broadcast, err := moq.NewBroadcastProducer() + if err != nil { + t.Fatal(err) + } + track, err := broadcast.PublishTrack("backlog", timedInfo()) + if err != nil { + t.Fatal(err) + } + for i := range 3 { + group, err := track.AppendGroup() + if err != nil { + t.Fatal(err) + } + if err := group.WriteFrame(moq.Frame{Payload: []byte("x"), Timestamp: ts(time.Duration(i) * time.Millisecond)}); err != nil { + t.Fatal(err) + } + if err := group.Finish(); err != nil { + t.Fatal(err) + } + } + + order := moq.OrderAscending + consumer, err := track.Consume(&moq.Subscription{ + MaxDelay: 10 * time.Second, + Ranges: []moq.Range{{Start: moq.AnchorRelative{Groups: 2}}}, + Order: &order, + }) + if err != nil { + t.Fatal(err) + } + defer consumer.Cancel() + + var sequences []uint64 + for range 3 { + group, err := consumer.RecvGroup(ctx) + if err != nil { + t.Fatal(err) + } + if group == nil { + t.Fatal("expected a group") + } + sequences = append(sequences, group.Sequence()) + } + if !slices.Equal(sequences, []uint64{0, 1, 2}) { + t.Fatalf("sequences = %v, want [0 1 2]", sequences) + } +} + func TestReadFrameSkipsEmptyThenPopulatedGroups(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), testTimeout) defer cancel() diff --git a/go/wrapper/records.go b/go/wrapper/records.go index 27efff4da76..1cfb9e36040 100644 --- a/go/wrapper/records.go +++ b/go/wrapper/records.go @@ -75,22 +75,28 @@ func datagramFromFFI(d *ffi.MoqDatagram) *Datagram { } // Subscription holds subscriber-side delivery preferences: priority, the max delay -// a non-latest group may fall behind before it is skipped (zero skips at once), and -// an optional group range (GroupStart a floor, GroupEnd exclusive). +// a non-latest group may fall behind before it is skipped (zero skips at once, and it +// caps every range), the ranges of the track to deliver (nil for everything MaxDelay +// still allows, while an empty non-nil slice asks for nothing), and the order the +// publisher sends pending groups in (nil for newest first). type Subscription struct { - Priority uint8 - MaxDelay time.Duration - GroupStart *uint64 - GroupEnd *uint64 + Priority uint8 + MaxDelay time.Duration + Ranges []Range + Order *Order } func (s Subscription) ffi() (ffi.MoqSubscription, error) { maxDelay, err := micros("max delay", s.MaxDelay) + var ranges *[]ffi.MoqRange + if s.Ranges != nil { + ranges = &s.Ranges + } return ffi.MoqSubscription{ Priority: s.Priority, MaxDelayUs: maxDelay, - GroupStart: s.GroupStart, - GroupEnd: s.GroupEnd, + Ranges: ranges, + Order: s.Order, }, err } diff --git a/go/wrapper/types.go b/go/wrapper/types.go index 53394b7b579..f8377f4fa06 100644 --- a/go/wrapper/types.go +++ b/go/wrapper/types.go @@ -52,6 +52,18 @@ type ( // AnnounceEventRestart reports another publisher instance now serving a prefix: // drop what was resolved under it and request afresh. AnnounceEventRestart = ffi.MoqAnnounceEventRestart + // Range is a span of a track to deliver: from Start up to End, or onward when End is nil. + // A relative start is the first frame of its group, a relative end includes its whole + // group, and an absolute end is exclusive. + Range = ffi.MoqRange + // Anchor is where one end of a Range falls: AnchorAbsolute or AnchorRelative. + Anchor = ffi.MoqAnchor + // AnchorAbsolute is an exact group and frame. + AnchorAbsolute = ffi.MoqAnchorAbsolute + // AnchorRelative is the group Groups before the latest one, clamped to group 0: 0 is the latest. + AnchorRelative = ffi.MoqAnchorRelative + // Order is which of a subscription's pending groups the publisher sends first. + Order = ffi.MoqOrder // VideoDecoderOutput configures what DecodeVideo delivers: an optional resize, a max delay, and whether frames keep the decoder's surface (macOS only; refused elsewhere). VideoDecoderOutput = ffi.MoqVideoDecoderOutput // VideoSurface is a decoded frame's platform surface, from VideoDecodedFrame.Surface: VideoSurfacePixelBuffer on macOS and iOS. @@ -133,6 +145,14 @@ func NamedEncoder(name string) VideoEncoderKind { return VideoEncoderKindNamed{Name: name} } +// Order values: which pending group the publisher sends first. +const ( + // OrderAscending sends the oldest first, draining a backlog in sequence order. + OrderAscending = ffi.MoqOrderAscending + // OrderDescending sends the newest first, so congestion sheds the backlog rather than the live edge. + OrderDescending = ffi.MoqOrderDescending +) + // ConnectionStatus values: the lifecycle of a client session's connection. const ( // ConnectionStatusConnected means a session connected (the first connect, or a reconnect after a drop). diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt index b7bae56f747..72ce54084ec 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt @@ -107,8 +107,18 @@ typealias Datagram = uniffi.moq.MoqDatagram typealias Frame = uniffi.moq.MoqFrame /** A path-prefix route: the publisher epoch it serves (null when unknown), relay hop ids (oldest first), and static production and link cost (lower wins). */ typealias Route = uniffi.moq.MoqRoute -/** Tunes how a track subscription is delivered: priority, group ordering, and range. */ +/** Tunes how a track subscription is delivered: priority, latency budget, ranges, and group ordering. */ typealias Subscription = uniffi.moq.MoqSubscription +/** A span of a track to deliver: from `start` up to `end`, or onward when `end` is null. */ +typealias Range = uniffi.moq.MoqRange +/** Where one end of a [Range] falls: [AnchorAbsolute] or [AnchorRelative]. */ +typealias Anchor = uniffi.moq.MoqAnchor +/** An exact group and frame. */ +typealias AnchorAbsolute = uniffi.moq.MoqAnchor.Absolute +/** The group `groups` before the latest one, clamped to group 0: 0 is the latest. */ +typealias AnchorRelative = uniffi.moq.MoqAnchor.Relative +/** Which pending group the publisher sends first: `ASCENDING` or `DESCENDING`. */ +typealias Order = uniffi.moq.MoqOrder /** Options for fetching one past group by sequence. */ typealias FetchGroupOptions = uniffi.moq.MoqFetchGroupOptions /** Delivery settings for a raw track: priority, ordering, latency budget, and timescale. */ diff --git a/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt b/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt index 8d68faba273..379b756f6f4 100644 --- a/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt +++ b/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt @@ -236,6 +236,27 @@ class SmokeTest { } } + @Test + fun `an ascending subscription reads a cached backlog oldest first`() = runTest { + BroadcastProducer().use { broadcast -> + val track = broadcast.publishTrack("backlog", TrackInfo(timescale = 1_000_000uL)) + for (ms in 0uL until 3uL) { + val group = track.appendGroup() + group.writeFrame(Frame(payload = "x".encodeToByteArray(), timestampUs = ms * 1_000uL)) + group.finish() + } + val consumer = track.consume( + Subscription( + maxDelayUs = 10_000_000uL, + ranges = listOf(Range(start = AnchorRelative(groups = 2uL))), + order = Order.ASCENDING, + ), + ) + val sequences = List(3) { checkNotNull(consumer.recvGroup()).sequence() } + assertEquals(listOf(0uL, 1uL, 2uL), sequences) + } + } + @Test fun `readFrame skips empty then populated groups`() = runTest { BroadcastProducer().use { broadcast -> diff --git a/py/moq-rs/moq/__init__.py b/py/moq-rs/moq/__init__.py index d83c275c224..957e4d380a8 100644 --- a/py/moq-rs/moq/__init__.py +++ b/py/moq-rs/moq/__init__.py @@ -49,6 +49,7 @@ VideoConsumer, ) from .types import ( + Anchor, AudioCodec, AudioDecoderOutput, AudioEncoderInput, @@ -62,8 +63,10 @@ ErrorScope, FetchGroupOptions, Frame, + Order, ProtocolError, ProtocolKind, + Range, Route, Subscription, TrackInfo, @@ -78,6 +81,7 @@ ) __all__ = [ + "Anchor", "Announce", "AnnounceConsumer", "AnnouncedBroadcast", @@ -117,6 +121,8 @@ "OriginConsumer", "OriginDynamic", "OriginProducer", + "Order", + "Range", "Request", "Reservation", "Route", diff --git a/py/moq-rs/moq/_records.py b/py/moq-rs/moq/_records.py index af707a14787..6248fcb7d7c 100644 --- a/py/moq-rs/moq/_records.py +++ b/py/moq-rs/moq/_records.py @@ -16,6 +16,8 @@ MoqConnectionStats, MoqDatagram, MoqFrame, + MoqOrder, + MoqRange, MoqSubscription, MoqTrackInfo, ) @@ -91,21 +93,22 @@ class Subscription: """Subscriber-side delivery preferences, mirroring moq-net's ``track::Subscription``. ``max_delay`` is how far a non-latest group may fall behind before it is skipped; - zero skips at once. ``group_start`` is a floor and ``group_end`` an exclusive end, ``None`` for - no bound. + zero skips at once, and it caps every range. ``ranges`` are the spans of the track to + deliver, ``None`` for everything ``max_delay`` still allows. ``order`` is which pending + group the publisher sends first, ``None`` for newest first. """ priority: int = 0 max_delay: timedelta = timedelta(0) - group_start: int | None = None - group_end: int | None = None + ranges: Sequence[MoqRange] | None = None + order: MoqOrder | None = None def _ffi(self) -> MoqSubscription: return MoqSubscription( priority=self.priority, max_delay_us=_to_us(self.max_delay, "max_delay"), - group_start=self.group_start, - group_end=self.group_end, + ranges=None if self.ranges is None else list(self.ranges), + order=self.order, ) diff --git a/py/moq-rs/moq/types.py b/py/moq-rs/moq/types.py index 7ad85d765f3..ef2cfbdbd5f 100644 --- a/py/moq-rs/moq/types.py +++ b/py/moq-rs/moq/types.py @@ -3,6 +3,9 @@ Records with duration fields are owned in `_records` instead, so they read as `timedelta`. """ +from moq_ffi import ( + MoqAnchor as Anchor, +) from moq_ffi import ( MoqAudioCodec as AudioCodec, ) @@ -30,12 +33,18 @@ from moq_ffi import ( MoqFetchGroupOptions as FetchGroupOptions, ) +from moq_ffi import ( + MoqOrder as Order, +) from moq_ffi import ( MoqProtocolError as ProtocolError, ) from moq_ffi import ( MoqProtocolKind as ProtocolKind, ) +from moq_ffi import ( + MoqRange as Range, +) from moq_ffi import ( MoqRoute as Route, ) @@ -67,6 +76,7 @@ from ._records import Backoff, ConnectionStats, Datagram, Frame, Subscription, TrackInfo __all__ = [ + "Anchor", "AudioCodec", "AudioDecoderOutput", "AudioEncoderInput", @@ -80,8 +90,10 @@ "ErrorScope", "Frame", "FetchGroupOptions", + "Order", "ProtocolError", "ProtocolKind", + "Range", "Route", "Subscription", "TrackInfo", diff --git a/py/moq-rs/tests/test_local.py b/py/moq-rs/tests/test_local.py index 805084da9d2..1d30d516763 100644 --- a/py/moq-rs/tests/test_local.py +++ b/py/moq-rs/tests/test_local.py @@ -9,6 +9,9 @@ import moq_ffi import pytest +# Every group from the first, rather than only what the live edge still reaches. +FROM_GROUP_0 = [moq.Range(start=moq.Anchor.ABSOLUTE(group=0, frame=0))] + def create_announced(origin: moq.OriginProducer, path: str) -> moq.BroadcastProducer: broadcast = origin.create_broadcast(path) @@ -839,7 +842,7 @@ async def test_raw_multiple_frames(): async for announcement in routes(consumer.announced()): broadcast_consumer = await consumer.request_broadcast(announcement.prefix) - subscription = moq.Subscription(max_delay=timedelta(seconds=1), group_start=0) + subscription = moq.Subscription(max_delay=timedelta(seconds=1), ranges=FROM_GROUP_0) raw_consumer = await broadcast_consumer.subscribe_track("commands", subscription) messages = [ @@ -865,7 +868,7 @@ async def test_raw_producer_consume_direct(): """Consume a raw track directly from the producer, no origin/broadcast plumbing.""" broadcast = moq.BroadcastProducer() track = broadcast.publish_track("direct") - consumer = track.consume(moq.Subscription(max_delay=timedelta(seconds=1), group_start=0)) + consumer = track.consume(moq.Subscription(max_delay=timedelta(seconds=1), ranges=FROM_GROUP_0)) track.write_frame(b"hello") track.write_frame(b"world") @@ -880,6 +883,30 @@ async def test_raw_producer_consume_direct(): assert received == [b"hello", b"world"] +async def test_ascending_subscription_reads_a_cached_backlog_oldest_first(): + """An ascending subscription reaching back over a cached backlog reads it oldest first.""" + broadcast = moq.BroadcastProducer() + track = broadcast.publish_track("backlog", moq.TrackInfo(timescale=1_000_000)) + for ms in range(3): + group = track.append_group() + group.write_frame(b"x", timedelta(milliseconds=ms)) + group.finish() + + consumer = track.consume( + moq.Subscription( + max_delay=timedelta(seconds=10), + ranges=[moq.Range(start=moq.Anchor.RELATIVE(groups=2))], + order=moq.Order.ASCENDING, + ) + ) + sequences = [] + for _ in range(3): + group = await asyncio.wait_for(consumer.recv_group(), timeout=5.0) + assert group is not None + sequences.append(group.sequence) + assert sequences == [0, 1, 2] + + async def test_raw_group_producer_consume_direct(): """Consume a single group directly from the group producer.""" broadcast = moq.BroadcastProducer() @@ -923,7 +950,7 @@ async def test_raw_group_sequence(): async for announcement in routes(consumer.announced()): broadcast_consumer = await consumer.request_broadcast(announcement.prefix) raw_consumer = await broadcast_consumer.subscribe_track( - "seq", moq.Subscription(max_delay=timedelta(seconds=1), group_start=0) + "seq", moq.Subscription(max_delay=timedelta(seconds=1), ranges=FROM_GROUP_0) ) sent_sequences = [] @@ -955,7 +982,7 @@ async def test_default_iteration_is_sequence_order(): broadcast = create_announced(origin, "track/ordering") raw = broadcast.publish_track("ordering") - subscription = moq.Subscription(max_delay=timedelta(seconds=1), group_start=0) + subscription = moq.Subscription(max_delay=timedelta(seconds=1), ranges=FROM_GROUP_0) seq_consumer = raw.consume(subscription) arr_consumer = raw.consume(subscription) @@ -1103,7 +1130,7 @@ async def test_read_frame_one_per_group(): """read_frame() returns the first frame of each successive group.""" broadcast = moq.BroadcastProducer() track = broadcast.publish_track("status") - consumer = track.consume(moq.Subscription(max_delay=timedelta(seconds=1), group_start=0)) + consumer = track.consume(moq.Subscription(max_delay=timedelta(seconds=1), ranges=FROM_GROUP_0)) track.write_frame(b"ready") track.write_frame(b"running") @@ -1147,7 +1174,7 @@ async def test_read_frame_skips_remaining_frames_in_group(): """read_frame() only returns the first frame of a multi-frame group.""" broadcast = moq.BroadcastProducer() track = broadcast.publish_track("mixed") - consumer = track.consume(moq.Subscription(max_delay=timedelta(seconds=1), group_start=0)) + consumer = track.consume(moq.Subscription(max_delay=timedelta(seconds=1), ranges=FROM_GROUP_0)) group = track.append_group() group.write_frame(b"first") diff --git a/swift/Sources/Moq/Aliases.swift b/swift/Sources/Moq/Aliases.swift index bfe8018b5fc..9863c6ce313 100644 --- a/swift/Sources/Moq/Aliases.swift +++ b/swift/Sources/Moq/Aliases.swift @@ -59,8 +59,8 @@ public typealias Announce = MoqFFI.MoqAnnounce /// What an `AnnounceConsumer` yields: `.start`, `.update`, `.end`, or /// `.restart` (request the path again), each carrying an `Announce`. public typealias AnnounceEvent = MoqFFI.MoqAnnounceEvent -/// Per-subscription delivery preferences: priority, group ordering, latency -/// budget, and group range. +/// Per-subscription delivery preferences: priority, latency budget, the ranges +/// of the track to deliver, and group ordering. public typealias Subscription = MoqFFI.MoqSubscription /// Options for fetching one complete group by sequence. public typealias FetchGroupOptions = MoqFFI.MoqFetchGroupOptions @@ -106,3 +106,15 @@ public typealias ProtocolKind = MoqFFI.MoqProtocolKind /// A protocol failure: scope, verbatim wire code, kind, and a diagnostic message. public typealias ProtocolError = MoqFFI.MoqProtocolError + +// Nested rather than top-level: a module-level `Range` would shadow `Swift.Range` +// in every file that imports Moq. +extension Subscription { + /// A span of a track to deliver: from `start` up to `end`, or onward when `end` is `nil`. + public typealias Range = MoqFFI.MoqRange + /// Where one end of a `Range` falls: `.absolute(group:frame:)` or + /// `.relative(groups:)` back from the latest group. + public typealias Anchor = MoqFFI.MoqAnchor + /// Which pending group the publisher sends first: `.ascending` or `.descending`. + public typealias Order = MoqFFI.MoqOrder +} diff --git a/swift/Tests/MoqTests/SmokeTests.swift b/swift/Tests/MoqTests/SmokeTests.swift index c47d90a4161..166b30d314b 100644 --- a/swift/Tests/MoqTests/SmokeTests.swift +++ b/swift/Tests/MoqTests/SmokeTests.swift @@ -38,6 +38,27 @@ final class SmokeTests: XCTestCase { } } + func testAscendingSubscriptionReadsACachedBacklogOldestFirst() async throws { + let broadcast = try BroadcastProducer() + let track = try broadcast.publishTrack(name: "backlog", info: TrackInfo(timescale: 1_000_000)) + for ms: UInt64 in 0..<3 { + let group = try track.appendGroup() + try group.writeFrame(Data("x".utf8), timestampUs: ms * 1_000) + try group.finish() + } + let consumer = try track.consume(subscription: Subscription( + maxDelayUs: 10_000_000, + ranges: [Subscription.Range(start: .relative(groups: 2))], + order: .ascending + )) + var sequences: [UInt64] = [] + for _ in 0..<3 { + let group = try await consumer.recvGroup() + sequences.append(try XCTUnwrap(group).sequence) + } + XCTAssertEqual(sequences, [0, 1, 2]) + } + /// An error's description is the Rust error message. func testErrorDescriptionIsRustMessage() { XCTAssertEqual(MoqError.Closed.description, "closed") From 118a422228816b76ebc2fd6b7d36171575064084 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 08:52:39 -0700 Subject: [PATCH 07/20] chore(quest): finish model ranges, split dynamic range requests The model now carries ranges. Range requests on Dynamic, per-subscriber range filtering and drop accounting move to quest/m1/subscribe-ranges/dynamic.md. The lite-05 mixed relay case moves to the lite quest, and the draft-20+ widening gap is recorded on the moq-transport update quest. Co-Authored-By: Claude Opus 5.5 --- quest/m1/subscribe-ranges/README.md | 2 +- quest/m1/subscribe-ranges/dynamic.md | 42 ++++++++++++++++ quest/m1/subscribe-ranges/ietf-update.md | 6 ++- quest/m1/subscribe-ranges/ietf.md | 2 +- quest/m1/subscribe-ranges/lite.md | 11 +++- quest/m1/subscribe-ranges/model.md | 64 ------------------------ quest/m2/perf/demand-aggregate.md | 8 +-- 7 files changed, 63 insertions(+), 72 deletions(-) create mode 100644 quest/m1/subscribe-ranges/dynamic.md delete mode 100644 quest/m1/subscribe-ranges/model.md diff --git a/quest/m1/subscribe-ranges/README.md b/quest/m1/subscribe-ranges/README.md index ad6cefbdf7b..5468bebc8d5 100644 --- a/quest/m1/subscribe-ranges/README.md +++ b/quest/m1/subscribe-ranges/README.md @@ -73,7 +73,7 @@ every sequence delivered or dropped. ## Required -- [Model ranges](/quest/m1/subscribe-ranges/model.md) - `Subscription` carries ranges, and `Dynamic` fills misses by range +- [Dynamic range requests](/quest/m1/subscribe-ranges/dynamic.md) - `Dynamic` fills misses by range, and a subscriber reads only its own ranges - [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) - publishers name every group they won't deliver - [Lite-07 ranges](/quest/m1/subscribe-ranges/lite.md) - the lite-07 wire, Rust publisher and subscriber, and the draft - [moq-transport upstream updates](/quest/m1/subscribe-ranges/ietf-update.md) - the upstream request follows the aggregate priority; its order stays fixed, as every draft requires diff --git a/quest/m1/subscribe-ranges/dynamic.md b/quest/m1/subscribe-ranges/dynamic.md new file mode 100644 index 00000000000..4c709823b47 --- /dev/null +++ b/quest/m1/subscribe-ranges/dynamic.md @@ -0,0 +1,42 @@ +# [L] Dynamic range requests + +## Goal + +A publisher's `track::Dynamic` receives range requests for misses instead of +one request per group, and a track serves a subscription's ranges from one +cursor, in its order, without duplicating a group, naming every sequence in +range it won't deliver. `requested_group` and `fetch_group` are replaced, not +kept beside it. + +## Plan + +`track::Subscription` already carries aggregated `ranges`, but only as a wire +preference: locally a subscriber's ranges set where its reads start, and it +still reads every later group the aggregate brings in. Serving a range locally +means filtering a subscriber's reads to its own ranges, and a miss below the +cache becomes a range request on `Dynamic` rather than a group request. + +- Replace `requested_group` and `fetch_group` rather than keep both. Update + every caller (relay, moq-archive, hls, the ladder) in the same PR, plus the + published surfaces built on them, per the cross-package table: moq-ffi + (`fetch_group`, `requested_group`), moq-c (`poll_requested_group`), every + binding wrapper and its `doc/lib` page, and `moq fetch` through + `moq_relay::fetch_group` (added in the 2026-10-05 audit). +- Every sequence in range either arrives or is named as dropped, so a reader + never waits on a miss. Share the accounting with + [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) rather than adding a second one. +- Decided in the 2026-10-10 audit: `max_delay` caps every range, including + historical FETCH replacements, and publisher `max_age` always clamps it. + A FETCH with no reader delay uses publisher max-age alone, not the live + SUBSCRIBE zero-delay default. Preserve that distinction when translating + FETCH into a range request. Sharing a range never relaxes a caller's + budget or lets one expired caller cancel another still-valid reader. + [FETCH max-delay](/quest/m1/fetch-max-delay.md) owns the FETCH option and + enforcement; reuse its contract rather than restoring a historical exemption. +- Benchmark serving and filling over range count and span as separate axes, + beside `track_subscription_ranges`, which covers aggregation. + +## Related + +- [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) - publishers name every group they won't deliver +- [FETCH max-delay](/quest/m1/fetch-max-delay.md) - historical request budgets and per-caller enforcement diff --git a/quest/m1/subscribe-ranges/ietf-update.md b/quest/m1/subscribe-ranges/ietf-update.md index 09f6128134a..c2b68bcf4cf 100644 --- a/quest/m1/subscribe-ranges/ietf-update.md +++ b/quest/m1/subscribe-ranges/ietf-update.md @@ -44,7 +44,11 @@ What an update can carry, and this quest sends: - Forward stays 1. Pausing is not part of this. - The subscription filter is updatable from draft-15 on. Narrowing upstream ranges belongs to [moq-transport ranges](/quest/m1/subscribe-ranges/ietf.md), - which can reuse this update path. + which can reuse this update path. Widening is a bug today: on draft-20 and + later, a subscriber from a future group puts its absolute start in the + SUBSCRIBE filter, and a live subscriber merged in after it never gets the + current group. `rs/moq-net/tests/subscribe_ranges.rs` leaves draft-20+ out + for that reason; add it back once the update carries the widened filter. Implementation: the IETF subscriber watches the aggregate the way `lite/subscriber.rs` does (`buffer_update`), sending one update per change diff --git a/quest/m1/subscribe-ranges/ietf.md b/quest/m1/subscribe-ranges/ietf.md index 02337884a76..84e9a8f51e4 100644 --- a/quest/m1/subscribe-ranges/ietf.md +++ b/quest/m1/subscribe-ranges/ietf.md @@ -35,7 +35,7 @@ a live track's Largest Object, and test both cases. ## Required -- [Model ranges](/quest/m1/subscribe-ranges/model.md) - the range requests this answers +- [Dynamic range requests](/quest/m1/subscribe-ranges/dynamic.md) - the range requests this answers ## Related diff --git a/quest/m1/subscribe-ranges/lite.md b/quest/m1/subscribe-ranges/lite.md index 2906015a7a4..69ff71392a9 100644 --- a/quest/m1/subscribe-ranges/lite.md +++ b/quest/m1/subscribe-ranges/lite.md @@ -33,6 +33,15 @@ that is the Largest Object's group, so a newest group with no objects yet is not counted until its first object arrives; the relay reports what it has rather than waiting for the peer. +Test the lite-05 relay case #5000 left out, on mock time beside +`rs/moq-net/tests/late_lower_group.rs`: a live subscriber already reading +group 1, then one from `Absolute(0)`, and a fresh group 0 reaches only the +second. Today the merged request goes upstream as its covering span, which on +lite-05 has no floor and joins at the latest group, so group 0 never arrives; +it needs the explicit group 0 above. The merged-range relay test, +`rs/moq-net/tests/subscribe_ranges.rs`, extends to lite-07 once it carries the +list. + Update `drafts/draft-lcurley-moq-lite.md` in the same PR: the SUBSCRIBE and SUBSCRIBE_UPDATE fields, the Group Order section (the order is now a field), the removal of FETCH, and the lite-07 changelog. Fix the stale "offset by 1" @@ -44,5 +53,5 @@ FETCH, answered through the model's ranges. Run `just drafts check` and ## Required -- [Model ranges](/quest/m1/subscribe-ranges/model.md) - the model this serves +- [Dynamic range requests](/quest/m1/subscribe-ranges/dynamic.md) - the misses a relay fills upstream by range - [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) - lite-07's DROP for holes diff --git a/quest/m1/subscribe-ranges/model.md b/quest/m1/subscribe-ranges/model.md deleted file mode 100644 index f28dc4881a7..00000000000 --- a/quest/m1/subscribe-ranges/model.md +++ /dev/null @@ -1,64 +0,0 @@ -# [L] Model ranges - -## Goal - -The Rust model's `Subscription` carries a list of group ranges, aggregated -across subscribers. A track serves the union from one cursor without -duplicating a group, in the subscription's order, and reports every -sequence it won't deliver. A publisher's `track::Dynamic` receives range -requests for misses instead of one request per group. - -## Plan - -See the [line](/quest/m1/subscribe-ranges/README.md) for the decisions. -Guidance: - -- Ranges have `Absolute` or `Relative` ends (see the line). Aggregating - subscribers onto one upstream request is the union of their ranges; a - relative end resolves against the track's latest group when the publisher - serves it. Ranges are served in the existing `Subscription::order`, which - merges with descending winning. -- This absorbs the deleted lite-07 Live flag quest, so the model mirrors the - wire with no `live` field, and its tests move here, on mock time and both - in-process and through a relay: a subscriber from `Absolute(4)` merged with - a `Relative(0)..` subscriber on a quiet track whose newest group is 3 (the - live one still gets group 3); a buffering `Relative(k)..` merged with a - range above the live edge; a frame-precise start merged with live; an - untimed `Absolute(0)..` (it replays what - the wall clock has not made stale); and the lite-05 relay - case #5000 left out: a live subscriber already reading group 1, then one - from `Absolute(0)`, and a fresh group 0 reaches only the second. The closed - #5085 branch has reusable pieces (older-wire codec mappings, the JS fix - keeping groups below a floor buffered, mocked-time tests). -- Replace `requested_group` and `fetch_group` rather than keep both. Update - every caller (relay, moq-archive, hls, the ladder) in the same PR, plus the - published surfaces built on them, per the cross-package table: moq-ffi - (`fetch_group`, `requested_group`), moq-c (`poll_requested_group`), every - binding wrapper and its `doc/lib` page, and `moq fetch` through - `moq_relay::fetch_group` (added in the 2026-10-05 audit). -- Decided 2026-10-13 (#5400): the same reshape of `MoqSubscription` in moq-ffi - (`rs/moq-ffi/src/consumer.rs`) adds a `MoqOrder` enum (`Ascending`, - `Descending`) and `MoqSubscription.order`, defaulting to `Descending` and - mapped to `track::Subscription::order`, beside the ranges. It follows the - cross-package table like the rest: the `py/moq-rs`, `swift`, `kt`, - `dart/moq` and `go/wrapper/moq` wrappers, `cpp/moq`, C (inherited if the - [generated C bindings](/quest/m1/c/README.md) have landed, otherwise - `rs/moq-c/src/api.rs`, from which cbindgen generates `moq.h`), and their - `doc/lib` pages. `cpp/obs/src` and `doc/bin/obs.md` have no subscription - surface today, so they change only if OBS starts building a - `MoqSubscription`. Folded in here rather than a separate - quest, so the bindings break once. Test that an ascending subscription - receives a cached backlog oldest first in moq-ffi and each wrapper with tests. -- Decided in the 2026-10-10 audit: `max_delay` caps every range, including - historical FETCH replacements, and publisher `max_age` always clamps it. - A FETCH with no reader delay uses publisher max-age alone, not the live - SUBSCRIBE zero-delay default. Preserve that distinction when translating - FETCH into a range request. Sharing a range never relaxes a caller's - budget or lets one expired caller cancel another still-valid reader. - [FETCH max-delay](/quest/m1/fetch-max-delay.md) owns the FETCH option and - enforcement; reuse its contract rather than restoring a historical exemption. -- Benchmark range count and span as separate axes (AGENTS.md fan-out rule). - -## Related - -- [FETCH max-delay](/quest/m1/fetch-max-delay.md) - historical request budgets and per-caller enforcement diff --git a/quest/m2/perf/demand-aggregate.md b/quest/m2/perf/demand-aggregate.md index 543b9fe3f8f..524e0f0768b 100644 --- a/quest/m2/perf/demand-aggregate.md +++ b/quest/m2/perf/demand-aggregate.md @@ -15,9 +15,10 @@ readers, so `js/net/src/track.ts` keeps its fold. ## Plan - Shared state, no message passing (decided 2026-10-06). `Subscriptions` - becomes per-field counted ordered maps for `priority`, `max_delay`, `start`, - and `end`, with a count for the `None` arm (no floor, unbounded) that absorbs - `start` and `end`. Subscribe, update, and drop mutate it directly under the + becomes per-field counted ordered maps for `priority` and `max_delay`, a + count of descending subscribers for `order`, and a counted multiset of + `ranges` whose union is taken on read (`track_subscription_ranges` measures + that union). Subscribe, update, and drop mutate it directly under the lock: remove the subscriber's old contribution, add the new one. Each subscriber keeps its own last value for that. The per-subscriber `kio::Consumer` channels, the waiter re-arming, and the @@ -41,4 +42,3 @@ Public API: none expected. Wire: none. ## Related - [Front deadlines](/quest/m2/front-deadline-index.md) - the per-track half of a front's cost; this is the per-reader half -- [Subscribe ranges](/quest/m1/subscribe-ranges/model.md) - changes what the aggregate holds; whichever lands second adapts From 0de44c0f72f22703b7bd554f5718cdaea2ede87f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 08:57:55 -0700 Subject: [PATCH 08/20] style: format and convert the remaining start floors to with_groups Co-Authored-By: Claude Opus 5.5 --- rs/moq-c/src/api.rs | 14 +++++++----- rs/moq-c/src/test.rs | 5 ++++- rs/moq-ffi/src/test.rs | 2 +- rs/moq-hls/src/export/archive_tests.rs | 2 +- rs/moq-net/src/lite/subscriber.rs | 14 ++++++------ rs/moq-net/src/model/subscription.rs | 30 +++++++++++++++----------- rs/moq-net/src/model/track.rs | 6 +++++- rs/moq-net/tests/history_groups.rs | 8 ++----- rs/moq-net/tests/late_lower_group.rs | 8 +++++-- rs/moq-net/tests/loom.rs | 3 +-- rs/moq-stats/src/produce.rs | 2 +- rs/moq-tokio/tests/broadcast.rs | 2 +- 12 files changed, 57 insertions(+), 39 deletions(-) diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index 821b00c8bf7..e70d62fbb75 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -635,10 +635,12 @@ pub enum moq_order { fn parse_anchor(anchor: &moq_anchor) -> Result { Ok(match anchor.kind { - v if v == moq_anchor_kind::MOQ_ANCHOR_ABSOLUTE as u32 => moq_net::track::Anchor::Absolute(moq_net::track::Position { - group: anchor.group, - frame: anchor.frame, - }), + v if v == moq_anchor_kind::MOQ_ANCHOR_ABSOLUTE as u32 => { + moq_net::track::Anchor::Absolute(moq_net::track::Position { + group: anchor.group, + frame: anchor.frame, + }) + } v if v == moq_anchor_kind::MOQ_ANCHOR_RELATIVE as u32 => moq_net::track::Anchor::Relative(anchor.group), _ => return Err(Error::InvalidCode), }) @@ -646,7 +648,9 @@ fn parse_anchor(anchor: &moq_anchor) -> Result { /// # Safety /// - `subscription->ranges` must be NULL or point to `subscription->ranges_len` ranges. -pub(crate) unsafe fn parse_subscription(subscription: &moq_subscription) -> Result { +pub(crate) unsafe fn parse_subscription( + subscription: &moq_subscription, +) -> Result { let order = match subscription.order { v if v == moq_order::MOQ_ORDER_DESCENDING as u32 => moq_net::track::Order::Descending, v if v == moq_order::MOQ_ORDER_ASCENDING as u32 => moq_net::track::Order::Ascending, diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index 1c1537ae691..b5e88a3d834 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -1764,7 +1764,10 @@ fn raw_track_subscription_options_and_update() { assert_eq!(moq_consume_track_frame_free(frame_id), 0); let widened = [moq_range { - end: moq_anchor { group: 3, ..ranges[0].end }, + end: moq_anchor { + group: 3, + ..ranges[0].end + }, ..ranges[0] }]; let update = moq_subscription { diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index 5b7da429b52..711b69ce174 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -5,8 +5,8 @@ use super::session::{MoqClient, MoqClientConfig, MoqClientTls, MoqSession}; use crate::consumer::MoqBroadcastConsumer; use crate::consumer::MoqFetchGroupOptions; use crate::consumer::MoqGroupConsumer; -use crate::consumer::{MoqAnchor, MoqOrder, MoqRange, MoqSubscription}; use crate::consumer::MoqTrackConsumer; +use crate::consumer::{MoqAnchor, MoqOrder, MoqRange, MoqSubscription}; use crate::error::{MoqError, MoqProtocolError, MoqProtocolKind}; use crate::flate::{MoqFlateConfig, MoqFlateSnapshotProducer, MoqFlateStreamProducer}; use crate::json::{ diff --git a/rs/moq-hls/src/export/archive_tests.rs b/rs/moq-hls/src/export/archive_tests.rs index a72e8cb6e3f..a0954fa3aea 100644 --- a/rs/moq-hls/src/export/archive_tests.rs +++ b/rs/moq-hls/src/export/archive_tests.rs @@ -685,7 +685,7 @@ async fn history_after_expiry(finished: bool) { .pool .gc(now + Duration::from_secs(31), &kio::Waiter::noop()); let subscription = moq_net::track::Subscription::default() - .with_start(moq_net::track::Position::group(0)) + .with_groups(0..) .with_max_delay(Duration::from_millis((1_u64 << 62) - 1)); let mut retained = broadcast .consume() diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index aee63264717..896596edc67 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -5645,12 +5645,14 @@ impl ServeLoop { // parks, since a cap can be raised). Those streams can still // be in flight, so wait for the tail before finishing. let subscription = self.serving.subscription(); - let requested_end = subscription.as_ref().and_then(|sub| sub.end()).map(|end| match end - .frame - { - 0 => end.group, - _ => end.group.saturating_add(1), - }); + let requested_end = + subscription + .as_ref() + .and_then(|sub| sub.end()) + .map(|end| match end.frame { + 0 => end.group, + _ => end.group.saturating_add(1), + }); // The effective max delay is the stopgap grace: the wrong clock // (it bounds presentation-time drift), but it is how long the // subscriber was willing to wait for a late group anyway. diff --git a/rs/moq-net/src/model/subscription.rs b/rs/moq-net/src/model/subscription.rs index 3293c957b8b..7ad37954f0d 100644 --- a/rs/moq-net/src/model/subscription.rs +++ b/rs/moq-net/src/model/subscription.rs @@ -208,7 +208,9 @@ pub(super) fn union(ranges: impl IntoIterator) -> Vec { }) .max(); ranges.retain(|range| match range.start { - Anchor::Absolute(start) => open_absolute.is_none_or(|open| start < open || (start == open && range.end.is_none())), + Anchor::Absolute(start) => { + open_absolute.is_none_or(|open| start < open || (start == open && range.end.is_none())) + } Anchor::Relative(back) => open_relative.is_none_or(|open| back > open || (back == open && range.end.is_none())), }); ranges.sort_unstable(); @@ -356,10 +358,7 @@ impl Subscription { }, }) .collect(); - Self { - ranges, - ..self.clone() - } + Self { ranges, ..self.clone() } } /// The lowest group this subscription's ranges start at, resolving relative starts @@ -553,10 +552,7 @@ mod tests { } fn range(start: Anchor, end: impl Into>) -> Range { - Range { - start, - end: end.into(), - } + Range { start, end: end.into() } } fn ranges(subscriptions: &[Subscription]) -> Vec { @@ -687,7 +683,10 @@ mod tests { "a gap stays a gap" ); assert_eq!( - ranges(&[sub(&[Range::groups(2..5), Range::groups(9..)]), sub(&[Range::groups(4..)])]), + ranges(&[ + sub(&[Range::groups(2..5), Range::groups(9..)]), + sub(&[Range::groups(4..)]) + ]), vec![Range::groups(2..)] ); assert_eq!( @@ -731,7 +730,10 @@ mod tests { sub(range(absolute(5, 9), absolute(5, 10))), sub(range(absolute(5, 2), absolute(5, 3))) ]), - vec![range(absolute(5, 2), absolute(5, 3)), range(absolute(5, 9), absolute(5, 10))] + vec![ + range(absolute(5, 2), absolute(5, 3)), + range(absolute(5, 9), absolute(5, 10)) + ] ); assert_eq!( ranges(&[ @@ -750,7 +752,11 @@ mod tests { assert_eq!(combine(&[a.clone(), b.clone()]), combine(&[b.clone(), a.clone()])); let combined = combine(&[a.clone(), b]); - assert_eq!(combine(&[combined.clone().unwrap(), a]), combined, "a subset adds nothing"); + assert_eq!( + combine(&[combined.clone().unwrap(), a]), + combined, + "a subset adds nothing" + ); } #[test] diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index e95780794be..b4e403613ce 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -4022,7 +4022,11 @@ impl Cursor { // arrive, however many have arrived since. let latest = { let state = self.state.read(); - state.arrival.front().map(|(sequence, _)| *sequence).or(state.max_sequence) + state + .arrival + .front() + .map(|(sequence, _)| *sequence) + .or(state.max_sequence) }; if let Some(floor) = self.subscription.read().floor(latest) { self.min_sequence = self.min_sequence.max(floor); diff --git a/rs/moq-net/tests/history_groups.rs b/rs/moq-net/tests/history_groups.rs index 4668c147126..ce2923e195a 100644 --- a/rs/moq-net/tests/history_groups.rs +++ b/rs/moq-net/tests/history_groups.rs @@ -82,9 +82,7 @@ async fn round(version: &str, relay: bool, newest_first: bool) -> Vec<(u64, usiz .expect("broadcast resolves"); let reader = moq_net_sim::spawn(async move { - let subscription = Subscription::default() - .with_max_delay(FOREVER) - .with_groups(0..); + let subscription = Subscription::default().with_max_delay(FOREVER).with_groups(0..); let mut sub = remote .track("history") .unwrap() @@ -225,9 +223,7 @@ async fn a_late_subscriber_receives_the_relays_cached_history() { let consumer = first.consume(); consumer.routed("bcast").await.unwrap(); let remote = consumer.request_broadcast("bcast", None).await.unwrap(); - let subscription = Subscription::default() - .with_max_delay(FOREVER) - .with_groups(0..); + let subscription = Subscription::default().with_max_delay(FOREVER).with_groups(0..); let first_subscription = subscription.clone(); let first_reader = moq_net_sim::spawn(async move { let mut sub = remote diff --git a/rs/moq-net/tests/late_lower_group.rs b/rs/moq-net/tests/late_lower_group.rs index 9406bfadfe4..165b42264b7 100644 --- a/rs/moq-net/tests/late_lower_group.rs +++ b/rs/moq-net/tests/late_lower_group.rs @@ -120,7 +120,9 @@ async fn session(version: &str, relay: bool, start: Option) -> Vec<(u6 .expect("resolve timeout") .expect("broadcast resolves"); - let subscription = Subscription::default().with_max_delay(BUDGET).with_ranges([floored(start)]); + let subscription = Subscription::default() + .with_max_delay(BUDGET) + .with_ranges([floored(start)]); let reader = moq_net_sim::spawn(async move { let mut sub = remote .track("late") @@ -162,7 +164,9 @@ async fn in_process(start: Option) -> Vec<(u64, usize)> { .with_max_age(FOREVER), ) .unwrap(); - let subscription = Subscription::default().with_max_delay(BUDGET).with_ranges([floored(start)]); + let subscription = Subscription::default() + .with_max_delay(BUDGET) + .with_ranges([floored(start)]); let mut sub = track.subscribe(subscription); let reader = moq_net_sim::spawn(async move { read_all(&mut sub).await }); moq_net_sim::sleep(Duration::from_millis(10)).await; diff --git a/rs/moq-net/tests/loom.rs b/rs/moq-net/tests/loom.rs index 71c3f7dca81..56f3b272907 100644 --- a/rs/moq-net/tests/loom.rs +++ b/rs/moq-net/tests/loom.rs @@ -513,8 +513,7 @@ fn an_aborted_parked_group_releases_a_finished_track() { let track = broadcast.create_track("parked", timed()).unwrap(); let group = track.append_group().unwrap(); group.clone().finish().unwrap(); - let mut subscriber = - track.subscribe(moq_net::track::Subscription::default().with_groups(0..)); + let mut subscriber = track.subscribe(moq_net::track::Subscription::default().with_groups(0..)); subscriber.set_groups(..0); track.finish().unwrap(); let aborting = thread::spawn(move || group.abort(Error::Old).unwrap()); diff --git a/rs/moq-stats/src/produce.rs b/rs/moq-stats/src/produce.rs index ad348957a2c..291ad952287 100644 --- a/rs/moq-stats/src/produce.rs +++ b/rs/moq-stats/src/produce.rs @@ -1379,7 +1379,7 @@ mod tests { let subscribing = consumer .track(name) .unwrap() - .subscribe(track::Subscription::default().with_start(track::Position::group(floor))); + .subscribe(track::Subscription::default().with_groups(floor..)); family.adopt_pair( &broadcast, &mut requested, diff --git a/rs/moq-tokio/tests/broadcast.rs b/rs/moq-tokio/tests/broadcast.rs index bf01de50450..2ff5876ada8 100644 --- a/rs/moq-tokio/tests/broadcast.rs +++ b/rs/moq-tokio/tests/broadcast.rs @@ -1117,7 +1117,7 @@ async fn broadcast_route_migration() { // Resolve and subscribe: the cheaper route (A) serves the track. let subscription = moq_net::track::Subscription::default() - .with_start(moq_net::track::Position::group(1)) + .with_groups(1..) .with_max_delay(Duration::from_secs(10)); let broadcast = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test", None)) .await From 3c47a5f01df69d719fc3b24031f393dcf1d7c391 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 09:05:45 -0700 Subject: [PATCH 09/20] fix(net): gate the wire-bound helpers on the async feature Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/subscription.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rs/moq-net/src/model/subscription.rs b/rs/moq-net/src/model/subscription.rs index 7ad37954f0d..3f6b78440af 100644 --- a/rs/moq-net/src/model/subscription.rs +++ b/rs/moq-net/src/model/subscription.rs @@ -303,6 +303,7 @@ impl Subscription { /// the latest group. An empty list is the empty range, `Position::group(0)`. /// /// The form a wire without ranges carries, alongside [`Self::start`]. + #[cfg(any(feature = "async", test))] pub(crate) fn end(&self) -> Option { let mut end = Some(Position::default()); for range in &self.ranges { @@ -316,6 +317,7 @@ impl Subscription { /// Replace the ranges with one from `start` (or [`Range::oldest`]'s) to `end`: the form /// a wire without ranges carries. + #[cfg(any(feature = "async", test))] pub(crate) fn set_bounds(&mut self, start: Option, end: Option) { self.ranges = vec![Range { start: start.map_or(Range::oldest().start, Anchor::Absolute), @@ -324,12 +326,14 @@ impl Subscription { } /// Replace the start of [`Self::set_bounds`]'s one range, keeping its end. + #[cfg(feature = "async")] pub(crate) fn with_start(mut self, start: impl Into>) -> Self { self.set_bounds(start.into(), self.end()); self } /// Replace the end of [`Self::set_bounds`]'s one range, keeping its start. + #[cfg(any(feature = "async", test))] pub(crate) fn with_end(mut self, end: impl Into>) -> Self { self.set_bounds(self.start(), end.into()); self From 35a6e676e5f8f7dd9c6cb3a71099f56fdc318605 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 09:11:20 -0700 Subject: [PATCH 10/20] fix(net): expect result_large_err only where the producer is large A Subscription holding a Vec of ranges keeps the producer under the lint's threshold on 32-bit targets. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index b4e403613ce..2050c876d96 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -2277,9 +2277,13 @@ impl Producer { /// the track is closed, including when it was already closed; existing handles /// may still observe its final state. `Err(producer)` leaves the track unchanged /// so the caller can continue serving and wait for the next unused wake. - #[expect( - clippy::result_large_err, - reason = "return the owned producer without allocating on an idle check" + // Only 64-bit targets cross the lint's size threshold. + #[cfg_attr( + target_pointer_width = "64", + expect( + clippy::result_large_err, + reason = "return the owned producer without allocating on an idle check" + ) )] pub fn abort_unused(self, err: Error) -> std::result::Result<(), Self> { match self.state.write_unused() { From d32e1811e16b1c88f448d91b5a5e63fbf2be5fc7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 09:33:59 -0700 Subject: [PATCH 11/20] test(py): cast anchor variants for pyright Co-Authored-By: Claude Opus 5.5 --- py/moq-rs/tests/test_local.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py/moq-rs/tests/test_local.py b/py/moq-rs/tests/test_local.py index 1d30d516763..eac67560d6a 100644 --- a/py/moq-rs/tests/test_local.py +++ b/py/moq-rs/tests/test_local.py @@ -10,7 +10,7 @@ import pytest # Every group from the first, rather than only what the live edge still reaches. -FROM_GROUP_0 = [moq.Range(start=moq.Anchor.ABSOLUTE(group=0, frame=0))] +FROM_GROUP_0 = [moq.Range(start=cast(moq.Anchor, moq.Anchor.ABSOLUTE(group=0, frame=0)))] def create_announced(origin: moq.OriginProducer, path: str) -> moq.BroadcastProducer: @@ -895,7 +895,7 @@ async def test_ascending_subscription_reads_a_cached_backlog_oldest_first(): consumer = track.consume( moq.Subscription( max_delay=timedelta(seconds=10), - ranges=[moq.Range(start=moq.Anchor.RELATIVE(groups=2))], + ranges=[moq.Range(start=cast(moq.Anchor, moq.Anchor.RELATIVE(groups=2)))], order=moq.Order.ASCENDING, ) ) From f717aac06d000c48b75186acc022c2c702fc593d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 22:24:47 -0700 Subject: [PATCH 12/20] docs(net): say what a relative anchor sends upstream today Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/subscription.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/subscription.rs b/rs/moq-net/src/model/subscription.rs index e7aaedc3de2..655ee9b2ed7 100644 --- a/rs/moq-net/src/model/subscription.rs +++ b/rs/moq-net/src/model/subscription.rs @@ -104,7 +104,11 @@ pub enum Order { /// /// A relative anchor resolves against the latest group of the hop that serves it, once, /// when it starts serving. Before any group exists it resolves at the first one, so -/// [`Range::live`] starts there. A relay that cannot resolve it forwards it upstream as is. +/// [`Range::live`] starts there. +/// +/// No published wire carries one yet: upstream, a relative start is no floor, the same +/// request as [`Range::oldest`], and a relative end is no end. Behind a moq-transport upstream, +/// which joins without a floor at the live group, `Relative(n)..` starts there. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Anchor { /// This exact position. From ef5ef409e121eab11bbc243889411be43bdf9642 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 08:01:26 -0700 Subject: [PATCH 13/20] perf(net): keep the range union incremental Counts absolute ranges as signed boundaries in a treap that keeps each subtree's sum and lowest running sum, so a subscriber's range change costs O(log n) whatever the audience holds. Relative and mixed ranges are counted by value. The union is cached until a change moves it, and the reference fold is test-only. track_range_demand sweeps readers over distinct ranges. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/benches/track.rs | 49 ++- rs/moq-net/src/model/subscription.rs | 87 ++--- rs/moq-net/src/model/track/demand.rs | 93 +++-- rs/moq-net/src/model/track/demand/ranges.rs | 381 ++++++++++++++++++++ 4 files changed, 524 insertions(+), 86 deletions(-) create mode 100644 rs/moq-net/src/model/track/demand/ranges.rs diff --git a/rs/moq-net/benches/track.rs b/rs/moq-net/benches/track.rs index 81f923c0bd0..d59b86c436f 100644 --- a/rs/moq-net/benches/track.rs +++ b/rs/moq-net/benches/track.rs @@ -44,10 +44,13 @@ //! `track_fetch_expiry` polls budgeted fetched readers over independent fanout and //! retained-history axes, so expiry work that scans either table appears as a slope. //! -//! `track_subscription_ranges` aggregates subscribers that each ask for a disjoint range, -//! swept over how many ranges and how many groups each spans: the union must cost the -//! same per range whatever it spans, and fold every subscriber's ranges at once rather -//! than re-merging the aggregate per subscriber, which would show up as a slope over them. +//! `track_range_demand` moves one reader between ranges nobody else holds, over readers +//! holding a given number of distinct overlapping ranges: both axes must stay flat, since +//! a change touches only the moved range, never the rest of the audience's. +//! +//! `track_subscription_ranges` reads the aggregate of subscribers that each ask for a +//! disjoint range, swept over how many ranges and how many groups each spans: reading the +//! union must cost the same per range whatever it spans. //! //! Run with `cargo bench -p moq-net --bench track`. @@ -91,6 +94,9 @@ const CHURN: [usize; 3] = [1, 8, 64]; /// Viewers that joined and left before the churn, leaving the list room for that many. const PEAK: [usize; 3] = [0, 1024, 16384]; +/// Distinct ranges the steady readers hold between them, capped at the readers. +const DISTINCT: [usize; 4] = [1, 100, 1000, 10_000]; + /// Disjoint ranges in one aggregate, one per subscriber. const RANGES: [u64; 4] = [1, 16, 256, 4096]; @@ -671,6 +677,40 @@ fn bench_subscriber_preferences(c: &mut Criterion) { group.finish(); } +/// One reader moving between ranges of its own, over readers holding `distinct` ranges. +fn bench_range_demand(c: &mut Criterion) { + // Far past every held range, so each move adds one run and removes another. + let away = |offset: u64| { + let group = (1 << 40) + offset; + track::Subscription::default().with_groups(group..=group) + }; + let mut group = c.benchmark_group("track_range_demand"); + for steady in STEADY { + for distinct in DISTINCT.into_iter().filter(|&distinct| distinct <= steady) { + group.throughput(Throughput::Elements(2)); + group.bench_function(BenchmarkId::new(format!("{steady}r"), distinct), |b| { + let mut setup = Churn::new(steady, 0); + for (i, subscriber) in setup._steady.iter_mut().enumerate() { + // Overlapping, so the held ranges union into one run. + let start = (i % distinct) as u64 * 10; + subscriber + .update(track::Subscription::default().with_groups(start..start + 1000)) + .unwrap(); + } + let mut viewer = setup.track.subscribe(away(0)); + assert!(setup.track.poll_subscription_changed(&setup.waiter).is_ready()); + b.iter(|| { + viewer.update(away(2)).unwrap(); + assert!(black_box(setup.track.poll_subscription_changed(&setup.waiter)).is_ready()); + viewer.update(away(0)).unwrap(); + assert!(black_box(setup.track.poll_subscription_changed(&setup.waiter)).is_ready()); + }); + }); + } + } + group.finish(); +} + fn bench_subscription_ranges(c: &mut Criterion) { let mut group = c.benchmark_group("track_subscription_ranges"); for span in SPAN { @@ -1095,6 +1135,7 @@ criterion_group!( bench_subscriber_churn, bench_subscriber_churn_after_peak, bench_subscriber_join, + bench_range_demand, bench_subscription_ranges, bench_subscriber_preferences, bench_datagram_subscriber_join, diff --git a/rs/moq-net/src/model/subscription.rs b/rs/moq-net/src/model/subscription.rs index b63f5bb5808..f90c22f9f64 100644 --- a/rs/moq-net/src/model/subscription.rs +++ b/rs/moq-net/src/model/subscription.rs @@ -217,7 +217,7 @@ impl Range { } /// Whether this range certainly delivers nothing, wherever the latest group lands. - fn is_empty(&self) -> bool { + pub(super) fn is_empty(&self) -> bool { match (self.start, self.end) { (Anchor::Absolute(start), Some(Anchor::Absolute(end))) => end <= start, _ => false, @@ -225,63 +225,66 @@ impl Range { } } -/// The union of `ranges` in a canonical order, without empty or redundant ranges. +/// The union of `ranges` in a canonical order, without empty or redundant ranges: the +/// reference for the track's incremental union. /// -/// Exact only where the latest group cannot matter, which is enough for an aggregate to -/// compare equal across subscribers asking for the same thing in different words. +/// Absolute spans merge where they overlap or touch. The rest can't be placed without the +/// latest group, so only an open range drops what it certainly covers: the open absolute +/// span drops ranges starting inside it, and the farthest open relative range drops +/// relative ranges starting no farther back. [`Range::oldest`] covers everything. +#[cfg(test)] pub(super) fn union(ranges: impl IntoIterator) -> Vec { - let mut ranges: Vec = ranges.into_iter().filter(|range| !range.is_empty()).collect(); + let ranges: Vec = ranges.into_iter().filter(|range| !range.is_empty()).collect(); if ranges.contains(&Range::oldest()) { return vec![Range::oldest()]; } - // An open range covers every range starting at or after it, whatever its end. - let open_absolute = ranges + let mut spans: Vec<(Position, Option)> = ranges .iter() .filter_map(|range| match (range.start, range.end) { - (Anchor::Absolute(start), None) => Some(start), + (Anchor::Absolute(start), None) => Some((start, None)), + (Anchor::Absolute(start), Some(Anchor::Absolute(end))) => Some((start, Some(end))), _ => None, }) - .min(); - let open_relative = ranges + .collect(); + spans.sort_unstable(); + let mut runs: Vec<(Position, Option)> = Vec::new(); + for (start, end) in spans { + match runs.last_mut() { + Some(last) if last.1.is_none_or(|last| start <= last) => { + last.1 = last.1.zip(end).map(|(last, end)| last.max(end)); + } + _ => runs.push((start, end)), + } + } + let tail = runs.last().filter(|run| run.1.is_none()).map(|run| run.0); + let farthest = ranges .iter() .filter_map(|range| match (range.start, range.end) { (Anchor::Relative(back), None) => Some(back), _ => None, }) .max(); - ranges.retain(|range| match range.start { - Anchor::Absolute(start) => { - open_absolute.is_none_or(|open| start < open || (start == open && range.end.is_none())) - } - Anchor::Relative(back) => open_relative.is_none_or(|open| back > open || (back == open && range.end.is_none())), - }); - ranges.sort_unstable(); - ranges.dedup(); - - // Merge overlapping or adjacent absolute ranges. Sorted by start, so one pass suffices. - let mut merged: Vec = Vec::with_capacity(ranges.len()); - for range in ranges { - if let Some(last) = merged.last_mut() - && let Anchor::Absolute(start) = range.start - && let Some(Anchor::Absolute(end)) = last.end - && start <= end - { - match range.end { - Some(Anchor::Absolute(next)) => { - last.end = Some(Anchor::Absolute(end.max(next))); - continue; - } - None => { - last.end = None; - continue; - } - Some(Anchor::Relative(_)) => {} - } - } - merged.push(range); - } - merged + + let mut union: Vec = runs + .into_iter() + .map(|(start, end)| Range { + start: Anchor::Absolute(start), + end: end.map(Anchor::Absolute), + }) + .collect(); + union.extend(ranges.into_iter().filter(|range| match (range.start, range.end) { + (Anchor::Absolute(_), None | Some(Anchor::Absolute(_))) | (Anchor::Relative(_), None) => false, + (Anchor::Absolute(start), _) => tail.is_none_or(|tail| start < tail), + (Anchor::Relative(back), _) => farthest.is_none_or(|farthest| back > farthest), + })); + union.extend(farthest.map(|back| Range { + start: Anchor::Relative(back), + end: None, + })); + union.sort_unstable(); + union.dedup(); + union } impl Default for Subscription { diff --git a/rs/moq-net/src/model/track/demand.rs b/rs/moq-net/src/model/track/demand.rs index cb8fa9ec32d..e3a81570302 100644 --- a/rs/moq-net/src/model/track/demand.rs +++ b/rs/moq-net/src/model/track/demand.rs @@ -1,36 +1,50 @@ //! Incremental demand, separate from the track's cached content. use std::{collections::BTreeMap, sync::Arc, task::Poll, time::Duration}; -use super::super::subscription::union; -use super::{NewGroup, Order, Range, Subscription, TrackState, clamp_combined}; +use super::{NewGroup, Order, Subscription, TrackState, clamp_combined}; + +mod ranges; +use ranges::Ranges; #[derive(Default)] pub(super) struct Subscriptions { count: usize, priority: BTreeMap, max_delay: BTreeMap, - ranges: BTreeMap, - /// The union of `ranges`' keys, redone only when a range first appears or last leaves, - /// so readers sharing a range never pay for it. - union: Vec, + ranges: Ranges, descending: usize, /// Only the subscribers holding a request. new_group: BTreeMap, waiters: kio::WaiterList, } +/// Everything the aggregate holds besides its ranges. +type Scalars = (u8, Duration, Order, Option); + impl Subscriptions { pub(super) fn combined(&self) -> Option { - (self.count != 0).then(|| Subscription { - priority: *self.priority.last_key_value().unwrap().0, - max_delay: *self.max_delay.last_key_value().unwrap().0, - ranges: self.union.clone(), - order: if self.descending == 0 { - Order::Ascending - } else { - Order::Descending - }, - new_group: self.new_group.last_key_value().map(|(new_group, _)| *new_group), + let (priority, max_delay, order, new_group) = self.scalars()?; + Some(Subscription { + priority, + max_delay, + ranges: self.ranges.union(), + order, + new_group, + }) + } + + fn scalars(&self) -> Option { + (self.count != 0).then(|| { + ( + *self.priority.last_key_value().unwrap().0, + *self.max_delay.last_key_value().unwrap().0, + if self.descending == 0 { + Order::Ascending + } else { + Order::Descending + }, + self.new_group.last_key_value().map(|(new_group, _)| *new_group), + ) }) } @@ -52,27 +66,19 @@ impl Subscriptions { } } - /// Mutate counts under one lock, taking wakes only when an extreme changes. + /// Mutate counts under one lock, taking wakes only when the aggregate changes. fn replace(&mut self, old: Option<&Subscription>, new: Option<&Subscription>) -> kio::WaiterList { - let prev = self.combined(); + let prev = self.scalars(); + // New ranges count before old ones leave, so a range both hold never reads as gone. let mut reshaped = false; - if let Some(old) = old { - remove(&mut self.priority, old.priority); - remove(&mut self.max_delay, old.max_delay); - for range in &old.ranges { - reshaped |= remove(&mut self.ranges, *range); - } - self.descending -= usize::from(old.order == Order::Descending); - if let Some(new_group) = old.new_group { - remove(&mut self.new_group, new_group); - } - self.count -= 1; - } + let moved = old.map(|old| &old.ranges) != new.map(|new| &new.ranges); if let Some(new) = new { add(&mut self.priority, new.priority); add(&mut self.max_delay, new.max_delay); - for range in &new.ranges { - reshaped |= add(&mut self.ranges, *range); + if moved { + for range in &new.ranges { + reshaped |= self.ranges.add(*range); + } } self.descending += usize::from(new.order == Order::Descending); if let Some(new_group) = new.new_group { @@ -80,14 +86,21 @@ impl Subscriptions { } self.count += 1; } - if reshaped { - // The default range covers every other, so an audience holding it skips the walk. - self.union = match self.ranges.contains_key(&Range::oldest()) { - true => vec![Range::oldest()], - false => union(self.ranges.keys().copied()), - }; + if let Some(old) = old { + remove(&mut self.priority, old.priority); + remove(&mut self.max_delay, old.max_delay); + if moved { + for range in &old.ranges { + reshaped |= self.ranges.remove(*range); + } + } + self.descending -= usize::from(old.order == Order::Descending); + if let Some(new_group) = old.new_group { + remove(&mut self.new_group, new_group); + } + self.count -= 1; } - if self.combined() != prev { + if reshaped || self.scalars() != prev { self.waiters.take() } else { Default::default() @@ -222,7 +235,7 @@ impl From> for Preferences { #[cfg(test)] mod tests { use super::super::super::subscription::Combined; - use super::super::{Anchor, Position}; + use super::super::{Anchor, Position, Range}; use super::*; use rand::{RngExt, SeedableRng}; use std::sync::atomic::{AtomicUsize, Ordering}; diff --git a/rs/moq-net/src/model/track/demand/ranges.rs b/rs/moq-net/src/model/track/demand/ranges.rs new file mode 100644 index 00000000000..120723920d5 --- /dev/null +++ b/rs/moq-net/src/model/track/demand/ranges.rs @@ -0,0 +1,381 @@ +//! The union of every subscriber's ranges, kept current per change rather than refolded. +//! +//! Absolute ranges become signed boundaries over positions, +1 where one starts and -1 +//! where it ends, so a position is covered while the running sum stays above zero. A treap +//! keeps each subtree's sum and lowest running sum, so a change touches two boundaries and +//! asks one range-minimum question, whatever else the audience holds. Ranges anchored to +//! the latest group can't be placed on that line and are counted by value instead. + +use std::cell::OnceCell; +use std::collections::BTreeMap; +use std::hash::{BuildHasher, RandomState}; + +use super::super::{Anchor, Position, Range}; +use super::{add, remove}; + +/// Every subscriber's ranges, counted so that one change costs O(log n). +#[derive(Default)] +pub(super) struct Ranges { + /// Holders of [`Range::oldest`], which covers every other range. + oldest: usize, + absolute: Coverage, + /// Open relative ranges by how far back they start: only the farthest matters. + relative: BTreeMap, + /// Ranges with one relative end, or a bounded relative span, kept as they are. + mixed: BTreeMap, + /// The last union read, until a change moves it. + union: OnceCell>, +} + +/// How [`Ranges`] counts one range. +enum Shape { + Empty, + Oldest, + Absolute(Position, Option), + Relative(u64), + Mixed, +} + +impl Shape { + fn of(range: &Range) -> Self { + match (range.start, range.end) { + _ if range.is_empty() => Self::Empty, + (Anchor::Relative(u64::MAX), None) => Self::Oldest, + (Anchor::Absolute(start), None) => Self::Absolute(start, None), + (Anchor::Absolute(start), Some(Anchor::Absolute(end))) => Self::Absolute(start, Some(end)), + (Anchor::Relative(back), None) => Self::Relative(back), + _ => Self::Mixed, + } + } +} + +impl Ranges { + /// Count `range`, returning whether the union changed. + pub(super) fn add(&mut self, range: Range) -> bool { + let changed = match Shape::of(&range) { + Shape::Empty => false, + Shape::Oldest => { + self.oldest += 1; + self.oldest == 1 + } + Shape::Absolute(start, end) => { + let uncovered = self.absolute.min(start, end) == 0; + self.absolute.add(start, end, 1); + uncovered && self.oldest == 0 + } + Shape::Relative(back) => { + let farthest = self.farthest(); + add(&mut self.relative, back); + self.oldest == 0 && self.farthest() != farthest + } + Shape::Mixed => add(&mut self.mixed, range) && self.oldest == 0 && !self.covered(&range), + }; + self.moved(changed) + } + + /// Uncount `range`, returning whether the union changed. + pub(super) fn remove(&mut self, range: Range) -> bool { + let changed = match Shape::of(&range) { + Shape::Empty => false, + Shape::Oldest => { + self.oldest -= 1; + self.oldest == 0 + } + Shape::Absolute(start, end) => { + self.absolute.add(start, end, -1); + self.oldest == 0 && self.absolute.min(start, end) == 0 + } + Shape::Relative(back) => { + let farthest = self.farthest(); + remove(&mut self.relative, back); + self.oldest == 0 && self.farthest() != farthest + } + Shape::Mixed => remove(&mut self.mixed, range) && self.oldest == 0 && !self.covered(&range), + }; + self.moved(changed) + } + + /// Drop the cached union when `changed`, passing it through. + fn moved(&mut self, changed: bool) -> bool { + if changed { + self.union.take(); + } + changed + } + + /// The union in [`Range`] order, costing its own length rather than the audience's. + pub(super) fn union(&self) -> Vec { + self.union.get_or_init(|| self.fold()).clone() + } + + fn fold(&self) -> Vec { + if self.oldest > 0 { + return vec![Range::oldest()]; + } + let mut union: Vec = self + .absolute + .runs() + .into_iter() + .map(|(start, end)| Range { + start: Anchor::Absolute(start), + end: end.map(Anchor::Absolute), + }) + .collect(); + union.extend(self.mixed.keys().filter(|range| !self.covered(range))); + union.extend(self.farthest().map(|back| Range { + start: Anchor::Relative(back), + end: None, + })); + union.sort_unstable(); + union + } + + fn farthest(&self) -> Option { + self.relative.last_key_value().map(|(back, _)| *back) + } + + /// Whether an open range already delivers everything a mixed `range` could. + fn covered(&self, range: &Range) -> bool { + match range.start { + Anchor::Absolute(start) => self.absolute.tail().is_some_and(|tail| tail <= start), + Anchor::Relative(back) => self.farthest().is_some_and(|farthest| back <= farthest), + } + } +} + +/// Signed boundary counts over positions: the coverage of the absolute ranges. +#[derive(Default)] +struct Coverage { + root: Link, + priorities: RandomState, +} + +type Link = Option>; + +struct Node { + key: Position, + /// Ranges starting here, minus ranges ending here. Never zero. + delta: i64, + priority: u64, + /// The sum of every delta in this subtree. + sum: i64, + /// The lowest running sum at any key of this subtree, counted from its first key. + low: i64, + left: Link, + right: Link, +} + +impl Coverage { + /// Count `delta` more ranges over `start..end`, open when `end` is `None`. + fn add(&mut self, start: Position, end: Option, delta: i64) { + self.shift(start, delta); + if let Some(end) = end { + self.shift(end, -delta); + } + } + + fn shift(&mut self, key: Position, delta: i64) { + let (below, rest) = split(self.root.take(), &key, false); + let (at, above) = split(rest, &key, true); + let delta = at.map_or(0, |node| node.delta) + delta; + let at = (delta != 0).then(|| Node::new(key, delta, self.priorities.hash_one(key))); + self.root = merge(merge(below, at), above); + } + + /// The fewest ranges covering any position in `start..end`. + fn min(&mut self, start: Position, end: Option) -> i64 { + let (through, rest) = split(self.root.take(), &start, true); + let (inside, above) = match end { + Some(end) => split(rest, &end, false), + None => (rest, None), + }; + let at = sum(&through); + let min = at.min(at.saturating_add(low(&inside))); + self.root = merge(through, merge(inside, above)); + min + } + + /// The covered spans in order, the last one open while an open range remains. + fn runs(&self) -> Vec<(Position, Option)> { + let mut runs = Vec::new(); + walk(&self.root, 0, &mut runs); + runs + } + + /// Where the open span starts, if an open range remains. + fn tail(&self) -> Option { + if sum(&self.root) <= 0 { + return None; + } + match last_zero(&self.root, 0) { + Some(zero) => successor(&self.root, &zero), + None => first(&self.root), + } + } +} + +impl Node { + fn new(key: Position, delta: i64, priority: u64) -> Box { + Box::new(Self { + key, + delta, + priority, + sum: delta, + low: delta, + left: None, + right: None, + }) + } + + fn update(&mut self) { + let at = sum(&self.left) + self.delta; + self.sum = at + sum(&self.right); + self.low = low(&self.left).min(at).min(at.saturating_add(low(&self.right))); + } +} + +fn sum(link: &Link) -> i64 { + link.as_ref().map_or(0, |node| node.sum) +} + +fn low(link: &Link) -> i64 { + link.as_ref().map_or(i64::MAX, |node| node.low) +} + +/// Split into the keys below `key` (through it when `inclusive`) and the rest. +fn split(link: Link, key: &Position, inclusive: bool) -> (Link, Link) { + let Some(mut node) = link else { + return (None, None); + }; + if node.key < *key || (inclusive && node.key == *key) { + let (inner, rest) = split(node.right.take(), key, inclusive); + node.right = inner; + node.update(); + (Some(node), rest) + } else { + let (rest, inner) = split(node.left.take(), key, inclusive); + node.left = inner; + node.update(); + (rest, Some(node)) + } +} + +/// Join two treaps, every key of `left` below every key of `right`. +fn merge(left: Link, right: Link) -> Link { + match (left, right) { + (None, link) | (link, None) => link, + (Some(mut left), Some(mut right)) => { + if left.priority > right.priority { + left.right = merge(left.right.take(), Some(right)); + left.update(); + Some(left) + } else { + right.left = merge(Some(left), right.left.take()); + right.update(); + Some(right) + } + } + } +} + +fn first(link: &Link) -> Option { + let mut node = link.as_ref()?; + while let Some(left) = &node.left { + node = left; + } + Some(node.key) +} + +/// The first key above `after`. +fn successor(mut link: &Link, after: &Position) -> Option { + let mut found = None; + while let Some(node) = link { + if node.key > *after { + found = Some(node.key); + link = &node.left; + } else { + link = &node.right; + } + } + found +} + +/// Append the covered spans under `link`, entered with `offset` ranges open. +/// +/// A subtree entered covered whose running sum never reaches zero holds no boundary of the +/// union, so it is skipped whole: the walk visits O(log n) nodes per span it returns. +fn walk(link: &Link, offset: i64, runs: &mut Vec<(Position, Option)>) { + let Some(node) = link else { + return; + }; + if offset > 0 && offset.saturating_add(node.low) > 0 { + return; + } + walk(&node.left, offset, runs); + let before = offset + sum(&node.left); + let at = before + node.delta; + if before == 0 { + runs.push((node.key, None)); + } else if at == 0 { + runs.last_mut().expect("an open run").1 = Some(node.key); + } + walk(&node.right, at, runs); +} + +/// The last key where the running sum, starting at `offset`, is zero. +fn last_zero(link: &Link, offset: i64) -> Option { + let node = link.as_ref()?; + if offset.saturating_add(node.low) > 0 { + return None; + } + let at = offset + sum(&node.left) + node.delta; + if let Some(key) = last_zero(&node.right, at) { + return Some(key); + } + if at <= 0 { + return Some(node.key); + } + last_zero(&node.left, offset) +} + +#[cfg(test)] +mod tests { + use super::super::super::super::subscription::union; + use super::*; + use rand::{RngExt, SeedableRng}; + + /// Each change reports exactly whether the union moved, and the union matches a full fold + /// of what is held. + #[test] + fn incremental_union_matches_a_full_fold() { + let mut rng = rand::rngs::StdRng::seed_from_u64(0x72616e676573); + let anchor = |rng: &mut rand::rngs::StdRng| match rng.random_range(0..6) { + 0 => Anchor::Relative(rng.random_range(0..4)), + 1 => Anchor::Relative(u64::MAX), + _ => Anchor::Absolute(Position { + group: rng.random_range(0..12), + frame: rng.random_range(0..3), + }), + }; + let mut ranges = Ranges::default(); + let mut held: Vec = Vec::new(); + for step in 0..50_000 { + let grow = if held.len() < 24 { 0.65 } else { 0.35 }; + let before = union(held.iter().copied()); + let changed = if held.is_empty() || rng.random_bool(grow) { + let range = Range { + start: anchor(&mut rng), + end: rng.random_bool(0.6).then(|| anchor(&mut rng)), + }; + held.push(range); + ranges.add(range) + } else { + let range = held.swap_remove(rng.random_range(0..held.len())); + ranges.remove(range) + }; + let after = union(held.iter().copied()); + assert_eq!(ranges.union(), after, "step {step}"); + assert_eq!(changed, before != after, "step {step}"); + } + } +} From d7aae3fec5885619b0b97d7c9196e3bf6811130b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 08:14:51 -0700 Subject: [PATCH 14/20] fix(net): expect result_large_err on every target again The producer crosses the lint's threshold on 32-bit targets too since the NEW_GROUP request joined the subscription. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index d064c7bd266..ed7f7a5d4e6 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -2413,13 +2413,9 @@ impl Producer { /// the track is closed, including when it was already closed; existing handles /// may still observe its final state. `Err(producer)` leaves the track unchanged /// so the caller can continue serving and wait for the next unused wake. - // Only 64-bit targets cross the lint's size threshold. - #[cfg_attr( - target_pointer_width = "64", - expect( - clippy::result_large_err, - reason = "return the owned producer without allocating on an idle check" - ) + #[expect( + clippy::result_large_err, + reason = "return the owned producer without allocating on an idle check" )] pub fn abort_unused(self, err: Error) -> std::result::Result<(), Self> { match self.state.write_unused() { From 374e453bcf5431a296a1fb13ea8c06b71e30c114 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 08:42:50 -0700 Subject: [PATCH 15/20] fix(net): count a relative start back from the first group, even once evicted A relative start taken on an empty track resolved against the oldest cached group, so evicting the first one before the subscriber read moved its floor up. The track now keeps the first sequence it was given. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 42 +++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index ed7f7a5d4e6..c0c38b7e431 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -291,6 +291,10 @@ pub(crate) struct TrackState { // datagrams, so it can run ahead of any cached group. max_sequence: Option, + // The first sequence appended, kept after it is evicted: where a relative start taken + // on an empty track counts back from. + first_sequence: Option, + // The name's sequence namespace within the broadcast, which can run past this track's // `max_sequence` when an earlier producer of the name wrote further. sequence: Sequence, @@ -1278,6 +1282,7 @@ impl TrackState { /// Record a written sequence, here and in the name's shared namespace. fn advance(&mut self, sequence: u64) { self.max_sequence = Some(self.max_sequence.map_or(sequence, |max| max.max(sequence))); + self.first_sequence.get_or_insert(sequence); self.sequence.advance(sequence); } @@ -4386,15 +4391,8 @@ impl Cursor { fn resolve_floor(&mut self) -> bool { if self.floor_pending { // The track was empty at subscribe, so the latest group then was the first to - // arrive, however many have arrived since. - let latest = { - let state = self.state.read(); - state - .arrival - .front() - .map(|(sequence, _)| *sequence) - .or(state.max_sequence) - }; + // arrive, however many have arrived or left since. + let latest = self.state.read().first_sequence; if let Some(floor) = self.subscription.read().floor(latest) { self.min_sequence = self.min_sequence.max(floor); self.floor_pending = false; @@ -12416,6 +12414,32 @@ mod test { assert_eq!(drain(&mut live), vec![0, 1]); } + /// A relative start on an empty track counts back from the first group to arrive, even + /// once that group is evicted before the subscriber reads. + #[test] + #[cfg(feature = "async")] + fn a_relative_start_on_an_empty_track_survives_its_first_group_leaving() { + let producer = untimed_producer(); + let buffering = Range { + start: Anchor::Relative(2), + end: None, + }; + let mut behind = producer.subscribe(replay().with_ranges([buffering])); + let write = |sequence| { + let mut group = producer.create_group(group::Info { sequence }).unwrap(); + group.write_frame(None, &b"x"[..]).unwrap(); + group.finish().unwrap(); + }; + write(10); + elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); + write(11); + assert!(!producer.state.read().lookup.contains_key(&10), "group 10 evicted"); + write(8); + let mut delivered = drain(&mut behind); + delivered.sort_unstable(); + assert_eq!(delivered, vec![8, 11]); + } + /// A relative start buffers that many groups behind the live edge, whatever a range /// above the edge merged beside it asks for. #[test] From 7d44715ca7116adcc836b5c082aba614c94b3a64 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 08:56:36 -0700 Subject: [PATCH 16/20] fix(net): resolve relative starts against groups, not datagrams Datagrams share the sequence space, so a datagram past the newest group moved a live subscriber's floor beyond the group it should start on. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index c0c38b7e431..07f1db1872b 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -291,9 +291,9 @@ pub(crate) struct TrackState { // datagrams, so it can run ahead of any cached group. max_sequence: Option, - // The first sequence appended, kept after it is evicted: where a relative start taken - // on an empty track counts back from. - first_sequence: Option, + // The first group cached, kept after it is evicted: where a relative start taken on a + // track without groups counts back from. + first_group: Option, // The name's sequence namespace within the broadcast, which can run past this track's // `max_sequence` when an earlier producer of the name wrote further. @@ -1282,7 +1282,6 @@ impl TrackState { /// Record a written sequence, here and in the name's shared namespace. fn advance(&mut self, sequence: u64) { self.max_sequence = Some(self.max_sequence.map_or(sequence, |max| max.max(sequence))); - self.first_sequence.get_or_insert(sequence); self.sequence.advance(sequence); } @@ -1345,6 +1344,7 @@ impl TrackState { self.evict.push_back((latest, prev.stamp)); } self.latest_group = Some(sequence); + self.first_group.get_or_insert(sequence); } if !self.protects(sequence) { group.cache_demote(); @@ -4319,7 +4319,8 @@ impl Cursor { let (floor, datagram_index) = { let preferences = subscription.read(); let state = state.read(); - let floor = preferences.floor(state.max_sequence); + // Relative starts count groups, so datagrams sharing the sequences don't move them. + let floor = preferences.floor(state.latest_group); // Arrival order need not be timestamp order. Only skip the initial stale prefix; // subsequent delivery stays in arrival order, including late datagrams. let newest = state.datagrams.iter().filter_map(|datagram| datagram.timestamp).max(); @@ -4390,9 +4391,9 @@ impl Cursor { /// count a relative start back from. Returns whether the floor is settled. fn resolve_floor(&mut self) -> bool { if self.floor_pending { - // The track was empty at subscribe, so the latest group then was the first to + // The track had no group at subscribe, so the latest group then was the first to // arrive, however many have arrived or left since. - let latest = self.state.read().first_sequence; + let latest = self.state.read().first_group; if let Some(floor) = self.subscription.read().floor(latest) { self.min_sequence = self.min_sequence.max(floor); self.floor_pending = false; @@ -12414,6 +12415,22 @@ mod test { assert_eq!(drain(&mut live), vec![0, 1]); } + /// A live range starts at the newest group even when a datagram took a later sequence. + #[test] + #[cfg(feature = "async")] + fn a_live_range_ignores_a_datagram_past_the_newest_group() { + let mut producer = track_producer("test", Info::timed()); + append_at(&mut producer, 0); + assert_eq!( + producer + .append_datagram(Timestamp::from_millis(1).unwrap(), &b"x"[..]) + .unwrap(), + 1 + ); + let mut live = producer.subscribe(replay().with_ranges([Range::live()])); + assert_eq!(drain(&mut live), vec![0]); + } + /// A relative start on an empty track counts back from the first group to arrive, even /// once that group is evicted before the subscriber reads. #[test] From c189a163ae7f160845df329d195846930bd17230 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 09:13:43 -0700 Subject: [PATCH 17/20] fix(net): skip empty ranges when resolving a subscriber's floor The union already drops them, so an empty span below a real one no longer starts the cursor early. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/subscription.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/model/subscription.rs b/rs/moq-net/src/model/subscription.rs index f90c22f9f64..1ef19acdc8d 100644 --- a/rs/moq-net/src/model/subscription.rs +++ b/rs/moq-net/src/model/subscription.rs @@ -415,10 +415,10 @@ impl Subscription { /// The lowest group this subscription's ranges start at, resolving relative starts /// against `latest`. `None` while one waits on a latest group that does not exist yet. /// - /// No ranges at all floor nowhere: group 0. + /// Empty ranges ask for nothing, and no ranges at all floor nowhere: group 0. pub(crate) fn floor(&self, latest: Option) -> Option { let mut floor = None; - for range in &self.ranges { + for range in self.ranges.iter().filter(|range| !range.is_empty()) { let start = range.resolve_start(latest)?.group; floor = Some(floor.map_or(start, |floor: u64| floor.min(start))); } @@ -673,8 +673,10 @@ mod tests { assert_eq!(mixed.floor(Some(9)), Some(4)); assert_eq!(mixed.floor(None), None); - // Nothing asked for floors nowhere. + // Nothing asked for floors nowhere, and an empty range asks for nothing. assert_eq!(Subscription::default().with_ranges([]).floor(None), Some(0)); + let empty = Subscription::default().with_ranges([Range::groups(5..5), Range::groups(10..)]); + assert_eq!(empty.floor(None), Some(10)); } /// The bounds a wire without ranges carries: the lowest absolute start and the highest From 405e94bf85b3cb9ca9979267f9313aa49a3649fe Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 09:34:42 -0700 Subject: [PATCH 18/20] fix(net): count relative starts from produced groups, not fetched backfill A fetch caching a group past the live edge advanced latest_group, so a live subscriber skipped the produced groups below it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 44 ++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 07f1db1872b..7c30715a431 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -291,9 +291,11 @@ pub(crate) struct TrackState { // datagrams, so it can run ahead of any cached group. max_sequence: Option, - // The first group cached, kept after it is evicted: where a relative start taken on a - // track without groups counts back from. - first_group: Option, + // The first and newest groups the publisher produced, which fetched backfill never + // moves: where relative starts count back from. The first is kept after it is evicted, + // for a relative start taken before any group was produced. + first_produced: Option, + newest_produced: Option, // The name's sequence namespace within the broadcast, which can run past this track's // `max_sequence` when an earlier producer of the name wrote further. @@ -1344,7 +1346,10 @@ impl TrackState { self.evict.push_back((latest, prev.stamp)); } self.latest_group = Some(sequence); - self.first_group.get_or_insert(sequence); + } + if visible { + self.first_produced.get_or_insert(sequence); + self.newest_produced = self.newest_produced.max(Some(sequence)); } if !self.protects(sequence) { group.cache_demote(); @@ -4319,8 +4324,7 @@ impl Cursor { let (floor, datagram_index) = { let preferences = subscription.read(); let state = state.read(); - // Relative starts count groups, so datagrams sharing the sequences don't move them. - let floor = preferences.floor(state.latest_group); + let floor = preferences.floor(state.newest_produced); // Arrival order need not be timestamp order. Only skip the initial stale prefix; // subsequent delivery stays in arrival order, including late datagrams. let newest = state.datagrams.iter().filter_map(|datagram| datagram.timestamp).max(); @@ -4391,9 +4395,9 @@ impl Cursor { /// count a relative start back from. Returns whether the floor is settled. fn resolve_floor(&mut self) -> bool { if self.floor_pending { - // The track had no group at subscribe, so the latest group then was the first to - // arrive, however many have arrived or left since. - let latest = self.state.read().first_group; + // The track had no produced group at subscribe, so the newest one then was the + // first to arrive, however many have arrived or left since. + let latest = self.state.read().first_produced; if let Some(floor) = self.subscription.read().floor(latest) { self.min_sequence = self.min_sequence.max(floor); self.floor_pending = false; @@ -12431,6 +12435,28 @@ mod test { assert_eq!(drain(&mut live), vec![0]); } + /// Fetched backfill past the newest produced group moves no relative start, whether the + /// track had a produced group at subscribe or not. + #[moq_clock::test] + #[cfg(feature = "async")] + async fn a_live_range_ignores_a_fetched_group_past_the_newest_group() { + let producer = track_producer("test", Info::timed()); + let dynamic = producer.dynamic(); + let consumer = producer.consume(); + let mut early = producer.subscribe(replay().with_ranges([Range::live()])); + let _fetching = consumer.fetch_group(5, None); + let req = dynamic.requested_group().now_or_never().unwrap().unwrap(); + req.accept(None).unwrap().finish().unwrap(); + producer + .create_group(group::Info { sequence: 0 }) + .unwrap() + .finish() + .unwrap(); + let mut late = producer.subscribe(replay().with_ranges([Range::live()])); + assert_eq!(drain(&mut early), vec![0]); + assert_eq!(drain(&mut late), vec![0]); + } + /// A relative start on an empty track counts back from the first group to arrive, even /// once that group is evicted before the subscriber reads. #[test] From f102ff4853d58a04abc1188e344f5221a955c63e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 09:56:17 -0700 Subject: [PATCH 19/20] test(net): pin live ranges delivering datagrams on a track without groups Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 7c30715a431..29589eb1223 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -12435,6 +12435,19 @@ mod test { assert_eq!(drain(&mut live), vec![0]); } + /// A live range on a track without groups delivers datagrams as they arrive, rather than + /// holding them for a group a datagram-only track never produces. + #[test] + #[cfg(feature = "async")] + fn a_live_range_on_a_track_without_groups_delivers_datagrams() { + let mut producer = track_producer("test", Info::timed()); + let mut live = producer.subscribe(replay().with_ranges([Range::live()])); + producer + .append_datagram(Timestamp::from_millis(0).unwrap(), &b"x"[..]) + .unwrap(); + assert_eq!(recv_datagram(&mut live).sequence, 0); + } + /// Fetched backfill past the newest produced group moves no relative start, whether the /// track had a produced group at subscribe or not. #[moq_clock::test] From 5f662fdb72857cf86e096d11aac72cab50d2931f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 10:23:51 -0700 Subject: [PATCH 20/20] quest: hand relative-start re-resolution outside the model to dynamic.md Co-Authored-By: Claude Opus 5.5 --- quest/m1/subscribe-ranges/dynamic.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/quest/m1/subscribe-ranges/dynamic.md b/quest/m1/subscribe-ranges/dynamic.md index 2c4f8d004d8..90d0f2aba08 100644 --- a/quest/m1/subscribe-ranges/dynamic.md +++ b/quest/m1/subscribe-ranges/dynamic.md @@ -33,6 +33,15 @@ cache becomes a range request on `Dynamic` rather than a group request. budget or lets one expired caller cancel another still-valid reader. Reuse FETCH's optional per-reader budget and publisher clamp rather than restoring a historical exemption. +- Move every relative-start resolution into the model, which counts back + from produced groups only. Today two consumers re-resolve relative starts + on their own. moq-c's `apply_raw_subscription` (`rs/moq-c/src/consume.rs`) + counts back from `Ordered::latest()`, the shared group and datagram + sequence, so a datagram past the newest group skews the start. It also + re-resolves on every update, because the model doesn't. moq-mux's + `container::Consumer::new` treats a relative start as an unfloored join + and adopts the first arrival, which under newest-first delivery can skip + the older groups the range asked for. - Benchmark serving and filling over range count and span as separate axes, beside `track_subscription_ranges`, which covers aggregation.