diff --git a/Cargo.lock b/Cargo.lock index ee815167cb4..427d528baa0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4569,10 +4569,8 @@ dependencies = [ "bytes", "criterion", "futures", - "hang", "kio 0.6.2", "moq-clock", - "moq-json", "moq-net", "object_store", "percent-encoding", @@ -4925,6 +4923,7 @@ dependencies = [ name = "moq-hls" version = "0.5.11" dependencies = [ + "async-trait", "axum", "bytes", "futures", @@ -4932,6 +4931,7 @@ dependencies = [ "humantime", "kio 0.6.2", "m3u8-rs", + "moq-archive", "moq-clock", "moq-json", "moq-mux", diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 961ce45d632..c060e64db20 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -307,8 +307,8 @@ equivalents. # Record every broadcast announced under `event` moq --connect https://relay.example.com/anon --broadcast event export archive s3://recordings -# Replay a timeline recording under another name -moq --connect https://relay.example.com/anon --broadcast event-replay.hang import archive s3://recordings/event +# Replay one recorded broadcast under another name +moq --connect https://relay.example.com/anon --broadcast event-replay.hang import archive s3://recordings/event/@ ``` `export archive` records every broadcast announced at or under `--broadcast` @@ -325,10 +325,15 @@ or stitches into an older recording. Any recording that fails ends the export. `--retention 1h` keeps only the last hour, a DVR, deleting older objects `--retention-grace` (default 30 s) after they leave the window. -`import archive` republishes a timeline recording made by an older `moq` as -live tracks, fetching spans on request. It ends where the recording ends; -`--follow 2s` keeps polling a recording still being made. It does not yet read -the per-track layout that `export archive` writes. +`import archive` republishes one recorded broadcast, the `/@` +prefix `export archive` wrote, as an ordinary broadcast under a new epoch: every +track, the catalog included, with its original groups, frames, timescale, and +priority. Each track is live from its newest stored group and serves older +groups from the store on request, with no max age, so the whole recording stays +reachable. A recording still being made is followed, checked every `--refresh` +(default 1 s), until `export archive` marks it complete; one whose export +stopped early stays live. It refuses a prefix with no recording, or with keys +`export archive` does not write. Store URLs are `file:///absolute/path`, `s3://bucket/prefix`, `gs://bucket/prefix`, or `az://container/prefix`, with credentials from the diff --git a/doc/bin/hls.md b/doc/bin/hls.md index e6abe0fad4b..1d062b7ea02 100644 --- a/doc/bin/hls.md +++ b/doc/bin/hls.md @@ -50,8 +50,17 @@ What that means for you: - **URLs name the publisher instance.** Every URL the multivariant playlist and DASH manifest list sits under `@`, the publisher's epoch, so every edge and every reload derive the same URLs and a CDN in front of export caches them. A restarted publisher gets URLs of its own, whose numbering starts over. The multivariant playlist and manifest switch to it at once, while a player already on the old instance keeps playing until that publisher leaves; the player then has to reload the multivariant playlist. An instance without an epoch (one that reached export over moq-lite-06 or older) gets an id each export mints, so only that export serves its URLs. - **Only a clean finish ends a playlist.** `EXT-X-ENDLIST` and a `static` DASH manifest mean the timeline finished. If the segment-boundary rendition's timeline fails instead, such as a publisher dropping its session, every media playlist and the DASH manifest answer HTTP 500 with `Cache-Control: no-store`, so a truncated recording never looks complete. A rendition whose own timeline fails answers the error alone and leaves the multivariant playlist and DASH manifest, so players switch to a sibling. -A [`moq-archive`](https://docs.rs/moq-archive) recording replayed through its -`Reader` is served the same way, with no second stored copy, and lists the +A [`moq-archive`](https://docs.rs/moq-archive) recording is served the same +way, with no second stored copy: record a broadcast, republish it, and export +the republished broadcast. + +```bash +moq --connect https://relay.example.com/anon --broadcast event export archive s3://recordings +moq --connect https://relay.example.com/anon --broadcast event-replay.hang import archive s3://recordings/event/@ +moq --connect https://relay.example.com/anon --broadcast event-replay.hang export hls --listen '[::]:8089' +``` + +The archived catalog and timelines replay as recorded, so the export lists the same window as a live broadcast. `--listen-tls-cert`/`--listen-tls-key` or `--listen-tls-generate` serve HTTPS, diff --git a/quest/m1/api-audit/README.md b/quest/m1/api-audit/README.md index 1c411d65c7b..ffbbfdc7f0b 100644 --- a/quest/m1/api-audit/README.md +++ b/quest/m1/api-audit/README.md @@ -41,11 +41,9 @@ Rules every child follows: - Each child lists its public API and wire impact in its PR, and its Impact bullet says how to migrate. -Already gated elsewhere, so not children: the archive `Producer` and `Reader` -disagree on layout while both claim version 2 -([archive consumer](/quest/m1/archive/consumer.md)), and the binary-to-flate -fold is followed by flate's removal ([compress migrate](/quest/m1/compress/migrate.md)). -Both are in m1, so the cut waits for them. hang's +Already gated elsewhere, so not a child: the binary-to-flate fold is followed +by flate's removal ([compress migrate](/quest/m1/compress/migrate.md)), in m1, +so the cut waits for it. hang's `Archive::replay_epoch` goes with the root `archive` section in [catalog HLS](/quest/m1/archive/catalog-hls.md). diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index 203eec70ed2..5f64b877d27 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -84,7 +84,6 @@ from a host. Each child fixes the stale docs it touches. ## Required -- [Archive consumer](/quest/m1/archive/consumer.md) - `archive::Consumer` serves a stored prefix as a broadcast from its key listing, and `moq export/import archive` use the new pair - [Catalog HLS field](/quest/m1/archive/catalog-hls.md) - the root `archive` section is gone; an opt-in per-rendition `hls` field marks each aligned HLS segment's first group and pts and declares its nominal duration - [Fixed HLS target duration](/quest/m1/archive/hls-target.md) - one `EXT-X-TARGETDURATION` for the run, from the declared `hls` duration; an overrun is listed with a warning diff --git a/quest/m1/archive/catalog-hls.md b/quest/m1/archive/catalog-hls.md index 5826909d00a..a29d56fe9c3 100644 --- a/quest/m1/archive/catalog-hls.md +++ b/quest/m1/archive/catalog-hls.md @@ -88,7 +88,3 @@ segment is refused, and that a rendition without `hls` gets no playlist. Public API: `hang::catalog::Archive` is deleted, the rendition types gain an `hls` field in Rust and JS, and `moq-mux` gains the opt-in. Wire: the hang catalog format. - -## Required - -- [Archive consumer](/quest/m1/archive/consumer.md) - the old writer and reader, which read the root `archive` section, are gone first diff --git a/quest/m1/archive/consumer.md b/quest/m1/archive/consumer.md deleted file mode 100644 index 7c1e4c30e49..00000000000 --- a/quest/m1/archive/consumer.md +++ /dev/null @@ -1,85 +0,0 @@ -# [M] Archive consumer - -## Goal - -`archive::Consumer` serves one broadcast's stored key prefix as an ordinary -broadcast: every stored track, the archived catalog included, with its -original groups, frames, timescale, and priority. A host publishes it at a path of its choosing; -moq-relay never uses it. `moq export archive` already records through -`archive::Producer`; `moq import archive` serves through -`archive::Consumer`. It replaces `moq_archive::Reader`, and with it goes -`.info`'s tolerance of the `epoch` an older `Writer` stored. - -## Plan - -Decided by the maintainer on 2026-10-17 (see the -[epic](/quest/m1/archive/README.md)): - -- One cached LIST of the prefix is the index: it names the tracks and maps - any group or pts to an object. Each track's `.info` supplies its timescale - and priority before any media is read. A growing archive refreshes each - track with its own cursor, a LIST of that track's key prefix StartAfter its - newest known key, since one prefix-wide cursor skips a new key under a - lexically earlier track. A delimiter LIST of the broadcast prefix finds - tracks added since. Every object is written once, an open group continuing - in the next object (decided 2026-10-21). -- Each served track advertises no Publisher Max Age limit (TRACK_INFO's - encoded `0`), not the source's: the store's retention bounds what it holds, - and DVR pruning is the publisher dropping a group early, which TRACK_INFO - permits. Advertising the source's bound would expire every archived group - older than it relative to the newest. -- Serve through `broadcast::Dynamic` and `track::Dynamic`. Create every listed - track up front, so TRACKS on the served broadcast lists them all. -- The archived catalog is an ordinary track, so a reader sees its newest - stored group, not a union of every rendition recorded. A rendition removed - before the archive ends is therefore not listed at the end; the replay-catalog - quest's union and `enabled: false` handling are dropped with it. -- The served broadcast takes its epoch from the caller or mints one; it never - claims the source's epoch, since it is served at another path. -- Each stored sequence replays in its recorded form: a group as a group, a - datagram as a datagram (`track::Producer::insert_datagram`), so the replay - keeps the source's lossy, uncached datagram contract. -- A track's `end`, reached by its `StartAfter` refresh since it sorts after - every object key, ends it as recorded. A clean end mirrors - `moq_archive::Reader::finish` today: refreshing stops, the track is - finished at the recorded end (`track::Producer::finish_at`), and - committed groups stay servable. An error end serves the stored groups and - then aborts the track with the recorded code (`StreamError::from_code`), - failing loud as the source did rather than reading as live. The - broadcast's `.end`, seen by the delimiter LIST, stops the search for - added tracks. The producer writes every marker only once the broadcast - has ended, so a live consumer finishes tracks only then; until then an - ended track just stops growing, and a re-added one grows again. Without markers the - archive is live and followed through refresh, which is how a producer - that crashed or was stopped reads. This replaces the caller-driven - `Reader::finish`. - -Guidance: - -- An absent group below the newest stored one is a gap and reports as dropped. - A group below the oldest stored key was pruned or never stored and reports - the same way. -- Keep a byte-bounded object LRU so neighbouring groups come from one GET. -- A `track::Fill` names group positions only, so a `Time(pts)` end the cache - can't resolve never reaches the handler. Resolving one from the key listing - (the object, then the group containing pts from its group timestamps) needs - a fill that carries the anchor. -- Cover HLS export over a consumer broadcast with its archived catalog: - playlists read only the timelines, and a segment GETs only its own - rendition's objects. The Reader-based `moq-hls` archive tests went with - `moq-archive`'s public segment API. -- Update `doc/bin/cli.md` and `doc/bin/hls.md` inline with an export, import, - and HLS example. - -Test with the in-memory store: a round trip preserving structure and payloads -byte for byte, a gap, a pruned start, a growing archive followed through -refresh (including two tracks where a new key sorts before the other track's -cursor, and a track added mid-recording), history older than the source's max -age still served, an untimed track, an interleaved group and datagram track -replayed in its original forms, a completed archive whose replay ends at -each track's recorded end (including mid-group), an errored track -replaying its stored groups and then the recorded error, a track removed -and re-added under its name replaying both runs in sequence, a growing archive -with no markers staying open, and an empty or unrecognized prefix refused loudly. - -Public API: `archive::Consumer` replaces `Reader`. Wire: none. diff --git a/quest/m1/compress/migrate.md b/quest/m1/compress/migrate.md index 613a0301088..7b42ff2be60 100644 --- a/quest/m1/compress/migrate.md +++ b/quest/m1/compress/migrate.md @@ -16,9 +16,8 @@ See the [epic](/quest/m1/compress/README.md) for the decisions. Decided 2026-10-20 (maintainer): the timeline is `.hls` with `compress: true`, never `.hls.z`. [Catalog HLS field](/quest/m1/archive/catalog-hls.md) -renames it and publishes it uncompressed, and the -[archive consumer](/quest/m1/archive/consumer.md) replaces the CLI's archive -reader, so this quest lands after both and only sets the flag. +renames it and publishes it uncompressed, and the archive consumer replaced +the CLI's archive reader, so this quest lands after it and only sets the flag. - Delete the `.z` names outright, with no transition release: hangz is opt-in and `main` takes breaking changes. The Impact bullet says how to migrate. @@ -61,4 +60,3 @@ Public API: removals listed above. Wire: the hang catalog loses ## Required - [Catalog HLS field](/quest/m1/archive/catalog-hls.md) - renames the timeline to `.hls`, which this compresses -- [Archive consumer](/quest/m1/archive/consumer.md) - replaces the CLI archive reader that reads `.z` timelines diff --git a/quest/m1/release-audit/README.md b/quest/m1/release-audit/README.md index 028f4ec2d02..fbd79026a82 100644 --- a/quest/m1/release-audit/README.md +++ b/quest/m1/release-audit/README.md @@ -98,7 +98,6 @@ child records whether its behavior is new on main or inherited from release. - [JS subscription ranges](/quest/m1/subscribe-ranges/js.md) - range lists and frame offsets build on the bounds fixed here - [OBS video source](/quest/m2/obs-moq-video/source.md) - the decoder replacement keeps the timing and EOF behavior fixed here - [Compressed tracks](/quest/m1/compress/README.md) - application migration has an existing owner -- [Archive consumer](/quest/m1/archive/consumer.md) - current producer and CLI reader still need the new-format roundtrip - [Measured release performance](/quest/m1/release-perf/README.md) - existing paired benchmark regressions and their confirmation runs - [Transport performance audit](/quest/m2/audit-perf.md) - congestion-control coverage remains outstanding - [Binding timestamps](/quest/m1/binding-timestamps.md) - exact timestamps across language APIs diff --git a/quest/m2/README.md b/quest/m2/README.md index 40037322742..b72f8e9b5cf 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -181,7 +181,9 @@ lite-07; bitrate enforcement is not part of this release. - [QUIC on the sim](/quest/m2/quic-sim.md) - MoQ sessions and the relay run over `moq-quic` on the seeded sim, reusing its `Pair` and `Link` - [Seeded sim](/quest/m2/sim-seed.md) - sim tests draw every jitter from one seeded generator and replay bit-exact +- [Archive time ranges](/quest/m2/archive-time-ranges.md) - a `Time(pts)` range on an archive consumer broadcast resolves from the archive's keys, however old - [DVR rewind](/quest/m2/archive-dvr.md) - follow-on capability after the release contract is stable +- [An errored track end keeps its history](/quest/m2/errored-end.md) - a replayed archive ends with its source's error after serving every stored group, instead of a clean finish - [JS HLS field](/quest/m2/js-hls.md) - follow-on capability after the release contract is stable diff --git a/quest/m2/archive-dvr.md b/quest/m2/archive-dvr.md index 27c5fa271fe..b1f29bd5dd6 100644 --- a/quest/m2/archive-dvr.md +++ b/quest/m2/archive-dvr.md @@ -31,4 +31,4 @@ over missing groups, and return to live. ## Required - [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) - the `Time(pts)` range a seek sends -- [Archive consumer](/quest/m1/archive/consumer.md) - serves a stored broadcast and resolves time ranges from its keys +- [Archive time ranges](/quest/m2/archive-time-ranges.md) - the archive consumer resolves time ranges from its keys diff --git a/quest/m2/archive-time-ranges.md b/quest/m2/archive-time-ranges.md new file mode 100644 index 00000000000..88096dfdb80 --- /dev/null +++ b/quest/m2/archive-time-ranges.md @@ -0,0 +1,35 @@ +# [M] Archive time ranges + +## Goal + +A `Time(pts)` range on an `archive::Consumer` broadcast resolves from the +archive's key listing, not only from the groups the consumer's cache holds: +a start reaches the stored group containing pts, and an end stops just after +it, however old. + +## Plan + +Decided 2026-10-23 (maintainer, on +[moq-dev/moq#6262](https://github.com/moq-dev/moq/pull/6262)): this stays in +m2, required by [DVR rewind](/quest/m2/archive-dvr.md), rather than blocking +the archive consumer, which ships without time anchors. + +The consumer creates each track live from its newest stored group and serves +older groups through `track::Fill`, so its cache holds almost nothing a time +anchor can resolve against: today a `Time(pts)` start older than the newest +group resolves to the oldest cached group instead. A `Fill` names group +positions only, so an anchor the cache can't resolve never reaches the +handler. [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171) deferred carrying the anchor to the consumer quest, which +landed without it. + +- `moq-net` passes a time anchor the cache can't resolve to the + `track::Dynamic` handler, unresolved, in its fill. +- The consumer resolves it from its index: every object key carries its first + frame's pts, so the object holding pts is one lookup, and the group within + it comes from the stored group timestamps. + +Test with the in-memory store: a start and an end older than the newest +group, an anchor before the oldest stored object, an anchor in a gap, and an +untimed track refusing it. + +Public API: `track::Fill` gains the anchor. Wire: none. diff --git a/quest/m2/errored-end.md b/quest/m2/errored-end.md new file mode 100644 index 00000000000..3f31c7c4c0b --- /dev/null +++ b/quest/m2/errored-end.md @@ -0,0 +1,41 @@ +# [M] An errored track end keeps its history + +## Goal + +A track can end with an error and still serve the groups it holds or can fill: +a reader arriving later gets that history, then the error, instead of either +the error alone or a clean finish. `archive::Consumer` replays a stored +`{"error":code}` end this way, so a replayed recording ends the way its source +did. + +## Plan + +From [moq-dev/moq#6262](https://github.com/moq-dev/moq/pull/6262), decided by +the maintainer 2026-10-23: the consumer replays an errored end as a clean +finish after its newest stored group, plus a warn log, because +`track::Producer::abort` refuses every later reader the stored history. The +alternatives were an abort (loses history) and an abort only after the last +group (still refuses later readers). This quest replaces the clean finish. + +- `moq-net`: an end that records an error alongside the final position. + Readers below the end read groups and fills as after `finish_at`; a reader + reaching the end gets the error instead of `None`. Unlike `abort`, the cache, + fills, and the listing stay as after a finish. Pick the API shape in the PR + with options and a recommendation, such as `finish_at` taking an end reason, + and no `finish_with_x` variant. +- Carry it across a hop if a relay must forward it: moq-lite resets the + subscription with the code today, which a later subscriber never sees. Any + wire change goes in the next unpublished lite version and its draft. +- `archive::Consumer` maps `End::Error(code)` to the new end and drops its + warn log. + +Test a consumer replaying an errored track: a subscriber from the oldest group +reads every stored group and then the error; a later fetch still reads stored +groups. + +Public API: a `track::Producer` end carrying an error, and what a reader +observes at it. Wire: possibly a lite track end carrying an error code. + +## Related + +- [DVR rewind](/quest/m2/archive-dvr.md) - seeks back through an archive whose source may have ended with an error diff --git a/quest/m2/performance-survey/README.md b/quest/m2/performance-survey/README.md index 495584f9f8e..ec9126c6d8a 100644 --- a/quest/m2/performance-survey/README.md +++ b/quest/m2/performance-survey/README.md @@ -167,9 +167,8 @@ No duplicate quest or workflow edit belongs here. `Unlimited`; their numbers cannot establish controller performance. Expand reassembly flight/reordering coverage there before proposing another index. [BBR study](/quest/m2/quic-bbr-study.md) owns controller behavior. -- Archive reader internals are being replaced by the - [archive consumer](/quest/m1/archive/consumer.md). Do not optimize the - outgoing timeline reader. Benchmark the landed replacement when revisiting it. +- The archive consumer replaced the timeline reader; benchmark it when + revisiting archive reads. ## Required diff --git a/quest/m3/archive-paced-replay.md b/quest/m3/archive-paced-replay.md index b5a7f74bb50..d43f86e636b 100644 --- a/quest/m3/archive-paced-replay.md +++ b/quest/m3/archive-paced-replay.md @@ -10,7 +10,7 @@ viewer who joins late joins mid-replay. ## Plan -Retargeted 2026-10-17 onto [Archive consumer](/quest/m1/archive/consumer.md), +Retargeted 2026-10-17 onto the archive consumer, whose broadcast already includes the archived catalog as an ordinary track. - Decided 2026-10-08: the catalog paces on the same clock as the media, each @@ -36,7 +36,3 @@ whose broadcast already includes the archived catalog as an ordinary track. current group, and that two tracks stay aligned. - Deferred in the 2026-09-30 audit and moved to m3 in the 2026-10-05 audit: no named consumer. - -## Required - -- [Archive consumer](/quest/m1/archive/consumer.md) - the stored broadcast this paces diff --git a/rs/moq-archive/Cargo.toml b/rs/moq-archive/Cargo.toml index 5de2c6b1b0f..a3d6fa17971 100644 --- a/rs/moq-archive/Cargo.toml +++ b/rs/moq-archive/Cargo.toml @@ -15,10 +15,8 @@ categories = ["multimedia", "filesystem"] [dependencies] bytes = { workspace = true } futures = { workspace = true } -hang = { workspace = true } kio = { workspace = true } moq-clock = { workspace = true } -moq-json = { workspace = true } moq-net = { workspace = true } object_store = { workspace = true } percent-encoding = { workspace = true } @@ -26,7 +24,6 @@ quick_cache = "0.7" serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, features = ["macros", "sync", "time"] } tracing = { workspace = true } [dev-dependencies] @@ -37,7 +34,7 @@ object_store = { workspace = true, features = ["fs", "aws"] } s3s = "0.17" s3s-fs = "0.17" tempfile = { workspace = true } -tokio = { workspace = true, features = ["macros", "net", "rt", "test-util"] } +tokio = { workspace = true, features = ["macros", "net", "rt", "sync", "test-util", "time"] } [[bench]] name = "producer" diff --git a/rs/moq-archive/README.md b/rs/moq-archive/README.md index af917e9a37f..c29d0f43833 100644 --- a/rs/moq-archive/README.md +++ b/rs/moq-archive/README.md @@ -14,6 +14,12 @@ group becomes a gap instead of a stall, a restarted producer resumes each track where it stopped, and an optional DVR window prunes old objects. The host picks the prefix and runs one producer per broadcast. +`Consumer` serves a stored prefix back as an ordinary broadcast: every track, +the archived catalog included, with its original groups, frames, timescale, +and priority. Each track is live from its newest stored group, older groups +are fetched from the store on request, and a growing archive is followed until +its producer marks it complete. + ```bash cargo add moq-archive ``` @@ -23,7 +29,11 @@ use moq_archive::object_store::memory::InMemory; // `broadcast` is a `moq_net::broadcast::Consumer`. let store = moq_archive::Store::new(InMemory::new(), "recordings/demo"); -moq_archive::Producer::new(store, broadcast, Default::default()).run().await?; +moq_archive::Producer::new(store.clone(), broadcast, Default::default()).run().await?; + +// `served` is a `moq_net::broadcast::Producer`, announced once `open` returns. +let consumer = moq_archive::Consumer::open(store, served, Default::default()).await?; +consumer.run().await?; ``` See [docs.rs/moq-archive](https://docs.rs/moq-archive) for the object layout. diff --git a/rs/moq-archive/src/consumer/index.rs b/rs/moq-archive/src/consumer/index.rs new file mode 100644 index 00000000000..a93e50ae47a --- /dev/null +++ b/rs/moq-archive/src/consumer/index.rs @@ -0,0 +1,136 @@ +//! One track's object keys, listed once and extended after its newest key. + +use std::collections::BTreeMap; + +use futures::TryStreamExt; +use moq_net::Timescale; +use object_store::path::Path; +use object_store::{ObjectStore, ObjectStoreExt}; + +use crate::layout::{self, End, Name}; +use crate::{Error, Info, Result}; + +/// Where an object starts: its first sequence and that sequence's first frame index. +pub(super) type Start = (u64, u64); + +/// What one listing of a track's directory found. +#[derive(Default)] +pub(super) struct Listed { + /// The objects new since the cursor, oldest first. + pub objects: Vec<(Start, Path)>, + /// The track's `end`, when asked for. + pub end: Option, +} + +/// A track's stored objects, by where each starts. +pub(super) struct Index { + pub dir: Path, + pub timescale: Option, + objects: BTreeMap, + /// The newest key listed, which the next listing starts after. + cursor: Path, +} + +impl Index { + pub fn new(dir: Path, info: &Info) -> Result { + let cursor = crate::path::push(&dir, layout::TRACK_INFO)?; + Ok(Self { + dir, + timescale: super::track::timescale(info)?, + objects: BTreeMap::new(), + cursor, + }) + } + + pub fn timed(&self) -> bool { + self.timescale.is_some() + } + + /// The newest key listed, for the next listing to start after. + pub fn cursor(&self) -> &Path { + &self.cursor + } + + /// Take in the objects a listing after [`Self::cursor`] found. + pub fn extend(&mut self, objects: &[(Start, Path)]) { + for (start, path) in objects { + self.objects.insert(*start, path.clone()); + self.cursor = path.clone(); + } + } + + /// The newest object, which holds the newest stored sequence. + pub fn newest(&self) -> Option<(Start, Path)> { + self.objects + .last_key_value() + .map(|(start, path)| (*start, path.clone())) + } + + /// The objects holding group `sequence` from frame `frame` on, oldest first: the last one + /// starting at or before it, then each continuing the group. Also the start of the first + /// object after them, which bounds a gap. + pub fn holding(&self, sequence: u64, frame: u64) -> (Vec<(Start, Path)>, Option) { + let mut holding = Vec::new(); + if let Some((start, path)) = self.objects.range(..=(sequence, frame)).next_back() { + holding.push((*start, path.clone())); + } + let mut after = None; + for (start, path) in self.objects.range((sequence, frame.saturating_add(1))..) { + if start.0 != sequence { + after = Some(*start); + break; + } + holding.push((*start, path.clone())); + } + (holding, after) + } + + /// Forget an object the store no longer has, such as one a DVR pruned. + pub fn forget(&mut self, start: Start) { + self.objects.remove(&start); + } +} + +/// List a track's directory after `cursor`, refusing anything the layout does not name, and +/// read its `end` when `end` is set. `.info` sorts before every object key and `end` after, +/// so listing after `.info` lists everything else. Object stores promise no listing order, so +/// it is sorted here. +pub(super) async fn list( + store: &S, + dir: &Path, + cursor: &Path, + timed: bool, + end: bool, +) -> Result { + let metas: Vec<_> = store.list_with_offset(Some(dir), cursor).try_collect().await?; + let mut listed = Listed::default(); + let mut found = None; + for meta in metas { + // A listing is recursive; anything deeper than the track's own entries is foreign. + if meta.location.prefix_match(dir).is_none_or(|parts| parts.count() != 1) { + return Err(Error::Path(meta.location.to_string())); + } + let filename = meta + .location + .filename() + .ok_or_else(|| Error::Path(meta.location.to_string()))?; + match Name::parse(filename, timed)? { + Name::Info => {} + Name::End => found = Some(meta.location), + Name::Object(start) => listed.objects.push(((start.group, start.frame), meta.location)), + } + } + listed.objects.sort(); + if end { + // The producer writes every track's `end` before the broadcast's `.end`. + let path = found.ok_or_else(|| Error::NotFound(format!("{dir}/{}", layout::TRACK_END)))?; + listed.end = Some(End::decode(&store.get(&path).await?.bytes().await?)?); + } + Ok(listed) +} + +/// A track's `.info`. +pub(super) async fn info(store: &S, dir: &Path) -> Result { + let path = crate::path::push(dir, layout::TRACK_INFO)?; + Info::decode(&store.get(&path).await?.bytes().await?) +} diff --git a/rs/moq-archive/src/consumer/mod.rs b/rs/moq-archive/src/consumer/mod.rs new file mode 100644 index 00000000000..832472b86e5 --- /dev/null +++ b/rs/moq-archive/src/consumer/mod.rs @@ -0,0 +1,385 @@ +//! Serve one broadcast's archive as an ordinary broadcast. +//! +//! A [`Consumer`] lists a [`Store`] prefix a [`Producer`](crate::Producer) wrote and creates every +//! stored track on a `moq_net` broadcast, the archived catalog included, with its stored +//! timescale and priority and no max age: the store's retention bounds what it holds. Each +//! track's live feed starts at its newest stored sequence, and every older group is served on +//! demand from the store through [`track::Dynamic`](moq_net::track::Dynamic) fills, each sequence in its recorded form: a +//! group as a group, a datagram as a datagram. +//! +//! A growing archive is followed: every [`Config::refresh`], one delimiter LIST of the prefix +//! finds added tracks, and one LIST per track after its newest key finds new objects. Once the +//! prefix holds `.end`, each track ends as recorded and refreshing stops. Without it the archive +//! stays live, which is also how one whose producer stopped early reads. +//! +//! ```no_run +//! # async fn example(broadcast: moq_net::broadcast::Producer) -> moq_archive::Result<()> { +//! use moq_archive::object_store::memory::InMemory; +//! +//! let store = moq_archive::Store::new(InMemory::new(), "recordings/demo"); +//! let consumer = moq_archive::Consumer::open(store, broadcast, Default::default()).await?; +//! // Every stored track exists now: announce the broadcast, then serve it. +//! consumer.run().await?; +//! # Ok(()) +//! # } +//! ``` + +mod index; +mod track; + +#[cfg(test)] +mod tests; + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::task::Poll; +use std::time::Duration; + +use futures::FutureExt; +use futures::future::BoxFuture; +use moq_net::broadcast; +use object_store::path::Path; +use object_store::{ObjectStore, ObjectStoreExt}; + +use crate::layout::{self, End, Object}; +use crate::{Error, Result, Store}; +use index::{Index, Start}; + +/// How a [`Consumer`] follows and caches its archive. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct Config { + /// How often a growing archive is listed for new objects and tracks. + pub refresh: Duration, + /// The most object bytes kept in memory, so neighbouring groups come from one GET. + pub cache: u64, +} + +impl Default for Config { + fn default() -> Self { + Self { + refresh: Duration::from_secs(1), + cache: 64 * 1024 * 1024, + } + } +} + +impl Config { + /// Set [`refresh`](Self::refresh). + pub fn with_refresh(mut self, refresh: Duration) -> Self { + self.refresh = refresh; + self + } + + /// Set [`cache`](Self::cache). + pub fn with_cache(mut self, bytes: u64) -> Self { + self.cache = bytes; + self + } +} + +/// Serves one archived broadcast until the broadcast closes. +pub struct Consumer { + shared: Arc>, + broadcast: broadcast::Producer, + /// Answers requests for names the archive does not hold, and says when the broadcast closed. + requests: broadcast::Dynamic, + refresh: Duration, + tracks: HashMap, + /// Whether the prefix held `.end`, so every track ended and nothing is refreshed. + ended: bool, +} + +struct Shared { + store: Store, + cache: quick_cache::sync::Cache, Weight>, +} + +struct Track { + index: Arc>, + live: track::Live, + dynamic: moq_net::track::Dynamic, +} + +/// What one refresh found, applied to the served tracks at once. +#[derive(Default)] +struct Refreshed { + /// Tracks new since the last refresh, with their index. + added: Vec<(String, moq_net::track::Info, Arc>)>, + /// Each track's new objects, fed live from the sequence given. + pushes: Vec<(String, Vec>, u64)>, + /// Each track's end, once the prefix holds `.end`. + ends: Option>, +} + +/// A track as a refresh starts: where its listing resumes and whether its feed started. +struct Snapshot { + name: String, + index: Arc>, + started: bool, +} + +impl Consumer { + /// List `store`'s prefix and create every stored track on `broadcast`, so a host can announce + /// it with every track listed. + /// + /// Fails with [`Error::NotFound`] when the prefix holds no track, and with [`Error::Path`] + /// when it holds a key the layout does not name. + pub async fn open(store: Store, broadcast: broadcast::Producer, config: Config) -> Result { + let shared = Arc::new(Shared { + store, + cache: quick_cache::sync::Cache::with_weighter( + // Estimated item count; the byte capacity below is the real bound. + 1024, + config.cache, + Weight, + ), + }); + let mut consumer = Self { + shared, + requests: broadcast.dynamic(), + broadcast, + refresh: config.refresh, + tracks: HashMap::new(), + ended: false, + }; + let refreshed = refresh(consumer.shared.clone(), Vec::new()).await?; + if refreshed.added.is_empty() { + return Err(Error::NotFound(consumer.shared.store.prefix().to_string())); + } + if let Err(err) = consumer.apply(refreshed) { + consumer.abort(moq_net::Error::Stream(moq_net::StreamError::Internal)); + return Err(err); + } + Ok(consumer) + } + + /// Serve the archive until the broadcast closes, following it while it grows. + /// + /// Fails when the store does, or when it holds a key or object the layout does not name. + /// Every track still live is then aborted with that failure; one left live when the + /// broadcast closes, or when the consumer is dropped, is cancelled. + pub async fn run(mut self) -> Result<()> { + let result = self.serve().await; + if result.is_err() { + self.abort(moq_net::Error::Stream(moq_net::StreamError::Internal)); + } + result + } + + async fn serve(&mut self) -> Result<()> { + let mut fills = kio::Tasks::new(); + let mut refreshing: Option>> = None; + + enum Event { + Refreshed(Result), + Closed, + } + + loop { + if refreshing.is_none() && !self.ended { + let snapshots = self.snapshots(); + let (shared, period) = (self.shared.clone(), self.refresh); + refreshing = Some( + async move { + moq_clock::sleep(period).await; + refresh(shared, snapshots).await + } + .boxed(), + ); + } + + let event = kio::wait(|waiter| { + for track in self.tracks.values() { + while let Poll::Ready(Ok(fill)) = track.dynamic.poll_requested_fill(waiter) { + let serve = track::serve(self.shared.clone(), track.index.clone(), fill); + fills.push(task(serve.boxed())); + } + } + let _ = fills.poll(waiter); + if let Some(future) = refreshing.as_mut() + && let Poll::Ready(refreshed) = waiter.poll_future(future.as_mut()) + { + return Poll::Ready(Event::Refreshed(refreshed)); + } + loop { + match self.requests.poll_requested_track(waiter) { + // Every stored track already exists, so the archive holds no such name. + Poll::Ready(Ok(request)) => request.reject(moq_net::Error::NotFound), + Poll::Ready(Err(_)) => return Poll::Ready(Event::Closed), + Poll::Pending => return Poll::Pending, + } + } + }) + .await; + + match event { + Event::Refreshed(refreshed) => { + refreshing = None; + self.apply(refreshed?)?; + } + Event::Closed => return Ok(()), + } + } + } + + fn snapshots(&self) -> Vec { + self.tracks + .iter() + .map(|(name, track)| Snapshot { + name: name.clone(), + index: track.index.clone(), + started: track.live.is_started(), + }) + .collect() + } + + fn apply(&mut self, refreshed: Refreshed) -> Result<()> { + for (name, info, index) in refreshed.added { + let producer = self.broadcast.create_track(name.as_str(), info)?; + let dynamic = producer.dynamic(); + self.tracks.insert( + name, + Track { + index, + live: track::Live::new(producer), + dynamic, + }, + ); + } + for (name, objects, from) in refreshed.pushes { + if let Some(track) = self.tracks.get_mut(&name) { + track.live.push(&objects, from)?; + } + } + if let Some(ends) = refreshed.ends { + for (name, end) in ends { + if let Some(track) = self.tracks.get_mut(&name) { + track.live.end(end, &name)?; + } + } + self.ended = true; + } + Ok(()) + } +} + +impl Consumer { + /// Abort every served track with `err`, its open group included. + fn abort(&mut self, err: moq_net::Error) { + for (_, track) in self.tracks.drain() { + track.live.abort(err.clone()); + } + } +} + +impl Drop for Consumer { + fn drop(&mut self) { + // An archive that ended already finished every track as recorded. + if !self.ended { + self.abort(moq_net::Error::Cancel); + } + } +} + +/// List the prefix for added tracks and `.end`, then each track after its newest key, and GET +/// what each track's live feed takes next. +async fn refresh(shared: Arc>, mut tracks: Vec) -> Result { + let store = &shared.store; + let prefix = store.prefix(); + let scope = (!prefix.as_ref().is_empty()).then_some(prefix); + let listed = store.inner().list_with_delimiter(scope).await?; + + let mut ended = false; + for object in &listed.objects { + match object.location.filename() { + Some(layout::BROADCAST_END) => ended = true, + _ => return Err(Error::Path(object.location.to_string())), + } + } + + let mut refreshed = Refreshed::default(); + for dir in listed.common_prefixes { + let segment = dir.filename().ok_or_else(|| Error::Path(dir.to_string()))?; + let name = layout::parse_track(segment)?; + if tracks.iter().any(|track| track.name == name) { + continue; + } + let stored = index::info(store.inner(), &dir).await?; + let info = track::info(&stored)?; + let index = Arc::new(Mutex::new(Index::new(dir, &stored)?)); + refreshed.added.push((name.clone(), info, index.clone())); + tracks.push(Snapshot { + name, + index, + started: false, + }); + } + + let mut ends = Vec::new(); + for track in tracks { + let (dir, cursor, timed) = { + let index = track.index.lock().unwrap(); + (index.dir.clone(), index.cursor().clone(), index.timed()) + }; + let listed = index::list(store.inner(), &dir, &cursor, timed, ended).await?; + track.index.lock().unwrap().extend(&listed.objects); + if let Some(end) = listed.end { + ends.push((track.name.clone(), end)); + } + + let (objects, from) = match track.started { + // Everything new, in order. + true => (listed.objects, 0), + // Only the newest sequence: fills serve the rest. + false => { + let Some((start, path)) = track.index.lock().unwrap().newest() else { + continue; + }; + let newest = object(&shared, &path, start, timed).await?; + let sequence = newest.entries.last().ok_or(Error::Empty)?.sequence(); + (track.index.lock().unwrap().holding(sequence, 0).0, sequence) + } + }; + let mut fetched = Vec::with_capacity(objects.len()); + for (start, path) in objects { + fetched.push(object(&shared, &path, start, timed).await?); + } + if !fetched.is_empty() { + refreshed.pushes.push((track.name, fetched, from)); + } + } + refreshed.ends = ended.then_some(ends); + Ok(refreshed) +} + +/// The object at `path`, from the cache or one GET, checked against the key it is listed under. +async fn object(shared: &Shared, path: &Path, start: Start, timed: bool) -> Result> { + shared + .cache + .get_or_insert_async(path, async { + let bytes = shared.store.inner().get(path).await?.bytes().await?; + let object = Object::decode(bytes, timed)?; + let found = object.start()?; + if (found.group, found.frame) != start { + return Err(Error::Path(path.to_string())); + } + Ok(Arc::new(object)) + }) + .await +} + +/// Adapt a boxed future into a [`kio::Tasks`] task. +fn task(mut future: BoxFuture<'static, ()>) -> impl FnMut(&kio::Waiter) -> Poll<()> + Send { + move |waiter| waiter.poll_future(future.as_mut()) +} + +/// Weighs a cached object by its payload bytes plus a small per-frame overhead. +#[derive(Clone)] +struct Weight; + +impl quick_cache::Weighter> for Weight { + fn weight(&self, _path: &Path, object: &Arc) -> u64 { + object.weight() + } +} diff --git a/rs/moq-archive/src/consumer/tests.rs b/rs/moq-archive/src/consumer/tests.rs new file mode 100644 index 00000000000..63707ff281d --- /dev/null +++ b/rs/moq-archive/src/consumer/tests.rs @@ -0,0 +1,732 @@ +use std::time::Duration; + +use bytes::Bytes; +use moq_clock::{sleep, timeout}; +use moq_net::track::{Order, Position, Subscription}; +use moq_net::{Timescale, Timestamp, broadcast, group, track}; +use object_store::ObjectStoreExt; +use tokio::task::JoinHandle; + +use super::*; +use crate::Info; +use crate::layout::{At, Entry, Frame}; +use crate::mock::Mock; + +const REFRESH: Duration = Duration::from_secs(1); + +/// Writes a prefix the way a [`crate::Producer`] lays it out, object by object. +struct Archive { + store: Store, +} + +impl Archive { + fn new() -> Self { + Self { + store: Store::new(Mock::memory(), "rec"), + } + } + + /// A timed track, timescale 1000, or an untimed one. + async fn track(&self, track: &str, priority: i8, timed: bool) { + let info = Info::new(priority, timed.then_some(1000)).unwrap(); + self.store.put_info(track, &info).await.unwrap(); + } + + /// Store one object whose first entry starts at `frame_start`. + async fn object(&self, track: &str, frame_start: u64, entries: Vec) { + let timed = self.store.get_info(track).await.unwrap().timescale.is_some(); + let object = Object { frame_start, entries }; + let name = object.start().unwrap().name().unwrap(); + let path = crate::path::push(&self.dir(track), &name).unwrap(); + let bytes = object.encode(timed).unwrap(); + self.store.inner().put(&path, bytes.into()).await.unwrap(); + } + + async fn end(&self, track: &str, end: End) { + let path = crate::path::push(&self.dir(track), layout::TRACK_END).unwrap(); + let bytes = end.encode().unwrap(); + self.store.inner().put(&path, bytes.into()).await.unwrap(); + } + + /// Write the broadcast's `.end`, after every track's `end`. + async fn complete(&self) { + self.raw(layout::BROADCAST_END, b"").await; + } + + async fn raw(&self, key: &str, bytes: &'static [u8]) { + let path = object_store::path::Path::parse(format!("rec/{key}")).unwrap(); + self.store + .inner() + .put(&path, Bytes::from_static(bytes).into()) + .await + .unwrap(); + } + + fn dir(&self, track: &str) -> object_store::path::Path { + layout::track_dir(self.store.prefix(), track).unwrap() + } +} + +/// Frame `index` of `sequence`, its timestamp and payload naming both. +fn frame(timed: bool, sequence: u64, index: u64) -> Frame { + Frame { + timestamp: timed.then_some(sequence * 1000 + index), + payload: Bytes::from(format!("{sequence}/{index}")), + } +} + +/// Group `sequence`'s frames `frames`. +fn frames(timed: bool, sequence: u64, frames: std::ops::Range) -> Entry { + Entry::Group { + sequence, + frames: frames.map(|index| frame(timed, sequence, index)).collect(), + } +} + +fn datagram(timed: bool, sequence: u64) -> Entry { + Entry::Datagram { + sequence, + frame: frame(timed, sequence, 0), + } +} + +/// What a reader sees of `frames(timed, sequence, indices)`. +fn expected(timed: bool, sequence: u64, indices: std::ops::Range) -> Vec { + indices.map(|index| describe(&frame(timed, sequence, index))).collect() +} + +fn describe(frame: &Frame) -> String { + let payload = String::from_utf8(frame.payload.to_vec()).unwrap(); + match frame.timestamp { + Some(pts) => format!("{pts}@{payload}"), + None => payload, + } +} + +fn read(timestamp: Option, payload: Bytes) -> String { + describe(&Frame { + timestamp: timestamp.map(Timestamp::value), + payload, + }) +} + +async fn serve(store: &Store) -> (broadcast::Producer, JoinHandle>) { + let broadcast = broadcast::Info::new().produce(); + let config = Config::default().with_refresh(REFRESH); + let consumer = Consumer::open(store.clone(), broadcast.clone(), config).await.unwrap(); + (broadcast, tokio::spawn(consumer.run())) +} + +/// FETCH group `sequence`. +async fn fetch(broadcast: &broadcast::Producer, track: &str, sequence: u64) -> moq_net::Result> { + let track = broadcast.consume().track(track)?; + let mut group = timeout(Duration::from_secs(10), track.fetch_group(sequence, None)) + .await + .expect("the fetch is answered")?; + let mut frames = Vec::new(); + while let Some(frame) = timeout(Duration::from_secs(10), group.read_frame()) + .await + .expect("the group ends")? + { + frames.push(read(frame.timestamp, frame.payload)); + } + Ok(frames) +} + +/// Read every group from 0 on, in order, with no age limit. +async fn subscribe(broadcast: &broadcast::Producer, track: &str) -> track::Ordered { + let track = broadcast.consume().track(track).unwrap(); + let subscription = Subscription::default() + .with_range(0..) + .with_order(Order::Ascending) + .with_max_delay(None); + track.subscribe(subscription).await.unwrap().ordered() +} + +/// The next group's sequence, without waiting for it to end. +async fn sequence(reader: &mut track::Ordered) -> Option { + timeout(Duration::from_secs(10), reader.next_group()) + .await + .expect("a group arrives") + .unwrap() + .map(|group| group.sequence) +} + +/// The next group and every frame in it, once the group ends. +async fn next(reader: &mut track::Ordered) -> Option<(u64, Vec)> { + let mut group = timeout(Duration::from_secs(10), reader.next_group()) + .await + .expect("a group arrives") + .unwrap()?; + let mut frames = Vec::new(); + while let Some(frame) = timeout(Duration::from_secs(10), group.read_frame()) + .await + .expect("the group ends") + .unwrap() + { + frames.push(read(frame.timestamp, frame.payload)); + } + Some((group.sequence, frames)) +} + +fn not_found(result: moq_net::Result>) { + assert!(matches!(result, Err(moq_net::Error::NotFound)), "{result:?}"); +} + +/// Recorded through the real producer, served back with every group, frame, form, and +/// property it had, byte for byte. +#[tokio::test(start_paused = true)] +async fn a_recording_round_trips() { + let store = Store::new(Mock::memory(), "rec"); + let source = broadcast::Info::new().produce(); + // The source keeps a second: the archive outlives it. + let timed = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_priority(3) + .with_max_age(Duration::from_secs(1)); + let video = source.create_track("video", timed).unwrap(); + let mut log = source + .create_track("log", track::Info::default().with_priority(9)) + .unwrap(); + let config = crate::producer::Config::default() + .with_duration(Duration::from_secs(2)) + .with_flush(Duration::from_secs(100)); + let recording = tokio::spawn(crate::Producer::new(store.clone(), source.consume(), config).run()); + while video.subscription().is_none() || log.subscription().is_none() { + sleep(Duration::from_millis(1)).await; + } + for sequence in 0..6 { + let mut group = video.create_group(group::Info { sequence }).unwrap(); + for index in 0..3 { + let pts = Timestamp::from_millis(sequence * 10_000 + index).unwrap(); + group + .write_frame(pts, Bytes::from(vec![sequence as u8, index as u8, 0xff])) + .unwrap(); + } + group.finish().unwrap(); + sleep(Duration::from_millis(10)).await; + } + let mut group = log.create_group(group::Info { sequence: 0 }).unwrap(); + group.write_frame(None, "a").unwrap(); + group.finish().unwrap(); + sleep(Duration::from_millis(10)).await; + log.insert_datagram(1, None, "b").unwrap(); + sleep(Duration::from_millis(10)).await; + video.finish().unwrap(); + log.finish().unwrap(); + sleep(Duration::from_millis(10)).await; + source.close(); + recording.await.unwrap().unwrap(); + + let (broadcast, _run) = serve(&store).await; + let served = broadcast.consume().track("video").unwrap(); + let info = served.query().await.unwrap(); + assert_eq!(info.priority, 3); + assert_eq!(info.timescale, Some(Timescale::MILLI)); + assert_eq!(info.max_age, None); + + let mut reader = subscribe(&broadcast, "video").await; + for sequence in 0..6 { + let mut group = reader.next_group().await.unwrap().unwrap(); + assert_eq!(group.sequence, sequence); + for index in 0..3 { + let frame = group.read_frame().await.unwrap().unwrap(); + assert_eq!(frame.timestamp.unwrap().value(), sequence * 10_000 + index); + assert_eq!(&frame.payload[..], &[sequence as u8, index as u8, 0xff]); + } + assert!(group.read_frame().await.unwrap().is_none()); + } + assert!(reader.next_group().await.unwrap().is_none()); + assert_eq!(reader.finished().await.unwrap(), Position::group(6)); + + // The untimed track keeps its datagram a datagram. + let info = broadcast.consume().track("log").unwrap().query().await.unwrap(); + assert_eq!((info.priority, info.timescale), (9, None)); + assert_eq!(fetch(&broadcast, "log", 0).await.unwrap(), ["a"]); + assert!(matches!( + fetch(&broadcast, "log", 1).await, + Err(moq_net::Error::NotFetchable | moq_net::Error::NotFound) + )); +} + +/// History older than the source's max age relative to the newest group is still served. +#[tokio::test(start_paused = true)] +async fn history_older_than_any_max_age_is_served() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + archive.object("video", 0, vec![frames(true, 0, 0..2)]).await; + // Hours of media later. + archive.object("video", 0, vec![frames(true, 10_000, 0..2)]).await; + + let (broadcast, _run) = serve(&archive.store).await; + assert_eq!(fetch(&broadcast, "video", 0).await.unwrap(), expected(true, 0, 0..2)); + let mut reader = subscribe(&broadcast, "video").await; + assert_eq!(next(&mut reader).await.unwrap(), (0, expected(true, 0, 0..2))); +} + +/// One GET serves every group of an object, and a group split across objects is served whole. +#[tokio::test(start_paused = true)] +async fn groups_come_from_cached_objects_and_span_them() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + archive + .object( + "video", + 0, + vec![frames(true, 0, 0..2), frames(true, 1, 0..2), frames(true, 2, 0..3)], + ) + .await; + archive.object("video", 3, vec![frames(true, 2, 3..5)]).await; + archive.object("video", 0, vec![frames(true, 3, 0..1)]).await; + + let (broadcast, _run) = serve(&archive.store).await; + let mock = archive.store.inner().clone(); + mock.take(); + assert_eq!(fetch(&broadcast, "video", 0).await.unwrap(), expected(true, 0, 0..2)); + assert_eq!(fetch(&broadcast, "video", 1).await.unwrap(), expected(true, 1, 0..2)); + assert_eq!(mock.gets().len(), 1, "both groups come from one object"); + + assert_eq!(fetch(&broadcast, "video", 2).await.unwrap(), expected(true, 2, 0..5)); +} + +/// A group every reader narrowed past stops its GET, so the groups the fill still owes are read +/// rather than waiting behind it. +#[tokio::test(start_paused = true)] +async fn a_narrowed_read_skips_the_group_it_left() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + for sequence in 0..4 { + archive.object("video", 0, vec![frames(true, sequence, 0..1)]).await; + } + + let (broadcast, _run) = serve(&archive.store).await; + let mock = archive.store.inner().clone(); + mock.take(); + let gate = mock.stall_gets("rec/video/"); + let ascending = |groups| { + Subscription::default() + .with_max_delay(None) + .with_range(groups) + .with_order(Order::Ascending) + }; + let mut subscriber = broadcast + .consume() + .track("video") + .unwrap() + .subscribe(ascending(0..)) + .await + .unwrap(); + let waiter = kio::Waiter::noop(); + let _ = subscriber.poll_recv_group(&waiter); + assert!(object_read(&mock, 0).await, "group 0 is read from its object"); + + subscriber.update(ascending(1..)).unwrap(); + let _ = subscriber.poll_recv_group(&waiter); + assert!( + object_read(&mock, 1).await, + "group 1 waited behind group 0, which nobody reads" + ); + gate.add_permits(16); +} + +/// Whether the video object starting at `sequence` is requested once the runtime settles. +/// Bounded by yields rather than time, so the paused clock stays put. +async fn object_read(mock: &Mock, sequence: u64) -> bool { + let prefix = format!("rec/video/{sequence:019}-"); + for _ in 0..1000 { + if mock.gets().iter().any(|get| get.starts_with(&prefix)) { + return true; + } + tokio::task::yield_now().await; + } + false +} + +#[tokio::test(start_paused = true)] +async fn a_gap_reads_as_dropped() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + archive + .object("video", 0, vec![frames(true, 0, 0..1), frames(true, 2, 0..1)]) + .await; + archive.object("video", 0, vec![frames(true, 5, 0..1)]).await; + archive.object("video", 0, vec![frames(true, 6, 0..1)]).await; + + let (broadcast, _run) = serve(&archive.store).await; + // Inside an object, and between two. + not_found(fetch(&broadcast, "video", 1).await); + not_found(fetch(&broadcast, "video", 3).await); + not_found(fetch(&broadcast, "video", 4).await); + + let mut reader = subscribe(&broadcast, "video").await; + for expected in [0, 2, 5, 6] { + assert_eq!(sequence(&mut reader).await, Some(expected)); + } +} + +/// A pruned start answers with what remains: the groups below it, and a group's pruned head, +/// read as dropped. +#[tokio::test(start_paused = true)] +async fn a_pruned_start_reads_as_dropped() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + archive + .object("video", 2, vec![frames(true, 5, 2..4), frames(true, 6, 0..2)]) + .await; + archive.object("video", 0, vec![frames(true, 7, 0..2)]).await; + archive.object("video", 0, vec![frames(true, 8, 0..1)]).await; + + let (broadcast, _run) = serve(&archive.store).await; + not_found(fetch(&broadcast, "video", 0).await); + not_found(fetch(&broadcast, "video", 4).await); + not_found(fetch(&broadcast, "video", 5).await); + assert_eq!(fetch(&broadcast, "video", 6).await.unwrap(), expected(true, 6, 0..2)); + + // An object pruned after it was listed reads as dropped too. + archive.store.inner().hide_gets("rec/video/0000000000000000007-"); + not_found(fetch(&broadcast, "video", 7).await); +} + +/// A GET that fails while the store still lists the object refuses its group only for now, so +/// a later read asks again once the store recovers. +#[tokio::test(start_paused = true)] +async fn a_failed_get_refuses_its_group_for_now() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + archive.object("video", 0, vec![frames(true, 0, 0..2)]).await; + archive.object("video", 0, vec![frames(true, 1, 0..1)]).await; + + let (broadcast, _run) = serve(&archive.store).await; + archive.store.inner().fail_gets("rec/video/0000000000000000000-"); + let err = fetch(&broadcast, "video", 0).await.unwrap_err(); + assert!(!matches!(err, moq_net::Error::NotFound), "{err}"); + archive.store.inner().heal_gets(); + assert_eq!(fetch(&broadcast, "video", 0).await.unwrap(), expected(true, 0, 0..2)); +} + +/// A growing archive is followed through each track's own cursor, a key sorting before +/// another track's cursor included, and a track added mid-recording joins. +#[tokio::test(start_paused = true)] +async fn a_growing_archive_is_followed() { + let archive = Archive::new(); + archive.track("a", 0, true).await; + archive.track("b", 0, true).await; + archive.object("a", 0, vec![frames(true, 0, 0..1)]).await; + archive.object("b", 0, vec![frames(true, 50, 0..1)]).await; + + let (broadcast, _run) = serve(&archive.store).await; + let mut a = subscribe(&broadcast, "a").await; + let mut b = subscribe(&broadcast, "b").await; + assert_eq!(sequence(&mut b).await, Some(50)); + + // `a/1-...` sorts before `b/50-...`: one prefix-wide cursor would skip it. + archive.object("a", 0, vec![frames(true, 1, 0..2)]).await; + archive.object("b", 0, vec![frames(true, 51, 0..1)]).await; + // A new object continuing the open group 51. + archive + .object("b", 1, vec![frames(true, 51, 1..2), frames(true, 52, 0..1)]) + .await; + assert_eq!(next(&mut a).await.unwrap(), (0, expected(true, 0, 0..1))); + assert_eq!(sequence(&mut a).await, Some(1)); + assert_eq!(next(&mut b).await.unwrap(), (51, expected(true, 51, 0..2))); + + assert!(matches!( + broadcast.consume().track("c").unwrap().query().await, + Err(moq_net::Error::NotFound) + )); + archive.track("c", 0, false).await; + archive.object("c", 0, vec![frames(false, 0, 0..1)]).await; + sleep(2 * REFRESH).await; + let mut c = subscribe(&broadcast, "c").await; + archive.object("c", 0, vec![frames(false, 1, 0..1)]).await; + assert_eq!(next(&mut c).await.unwrap(), (0, expected(false, 0, 0..1))); +} + +/// Without markers, an archive whose producer stopped stays live. +#[tokio::test(start_paused = true)] +async fn an_archive_without_markers_stays_open() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + archive + .object("video", 0, vec![frames(true, 0, 0..1), frames(true, 1, 0..1)]) + .await; + // A track `end` alone is not final: the broadcast never wrote `.end`. + archive.end("video", End::Finish(At { group: 2, frame: 0 })).await; + + let (broadcast, _run) = serve(&archive.store).await; + let mut reader = subscribe(&broadcast, "video").await; + assert_eq!(next(&mut reader).await.unwrap(), (0, expected(true, 0, 0..1))); + let mut group = reader.next_group().await.unwrap().unwrap(); + assert!(group.read_frame().await.unwrap().is_some()); + let pending = timeout(30 * REFRESH, group.read_frame()).await; + assert!(pending.is_err(), "the newest group stays open"); +} + +/// Closing the broadcast ends a followed archive and cancels the tracks it still served live, +/// the open newest group included. +#[tokio::test(start_paused = true)] +async fn closing_the_broadcast_cancels_the_live_tracks() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + archive.object("video", 0, vec![frames(true, 0, 0..1)]).await; + + let (broadcast, run) = serve(&archive.store).await; + let mut reader = subscribe(&broadcast, "video").await; + let mut group = reader.next_group().await.unwrap().unwrap(); + assert!(group.read_frame().await.unwrap().is_some()); + + broadcast.close(); + run.await.unwrap().unwrap(); + let err = group.read_frame().await.unwrap_err(); + assert!(matches!(err, moq_net::Error::Cancel), "{err}"); +} + +/// A continuation that skips frames of the open group fails the replay, and the open group is +/// aborted with the track rather than dropped. +#[tokio::test(start_paused = true)] +async fn a_gapped_continuation_aborts_the_open_group() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + archive.object("video", 0, vec![frames(true, 0, 0..1)]).await; + + let (broadcast, run) = serve(&archive.store).await; + let mut reader = subscribe(&broadcast, "video").await; + let mut group = reader.next_group().await.unwrap().unwrap(); + assert!(group.read_frame().await.unwrap().is_some()); + + // Frame 1 of group 0 is missing. + archive.object("video", 2, vec![frames(true, 0, 2..3)]).await; + assert!(matches!(run.await.unwrap(), Err(Error::Sequence))); + let err = group.read_frame().await.unwrap_err(); + assert!( + matches!(err, moq_net::Error::Stream(moq_net::StreamError::Internal)), + "{err}" + ); +} + +/// A store failure ends the replay and aborts every served track with it, the open newest group +/// included, so readers see the failure rather than a dropped producer. +#[tokio::test(start_paused = true)] +async fn a_store_failure_aborts_the_served_tracks() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + archive.object("video", 0, vec![frames(true, 0, 0..1)]).await; + + let (broadcast, run) = serve(&archive.store).await; + let mut reader = subscribe(&broadcast, "video").await; + let mut group = reader.next_group().await.unwrap().unwrap(); + assert!(group.read_frame().await.unwrap().is_some()); + + archive.store.inner().fail_lists(); + assert!(run.await.unwrap().is_err()); + let failed = |err: &moq_net::Error| matches!(err, moq_net::Error::Stream(moq_net::StreamError::Internal)); + let err = group.read_frame().await.unwrap_err(); + assert!(failed(&err), "{err}"); + let Err(err) = reader.next_group().await else { + panic!("the track is aborted"); + }; + assert!(failed(&err), "{err}"); +} + +/// A completed archive ends each track at its recorded end, mid-group included, and an +/// errored track replays its stored groups. +#[tokio::test(start_paused = true)] +async fn a_completed_archive_ends_as_recorded() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + archive.track("audio", 0, true).await; + archive.track("errored", 0, true).await; + archive.track("tail", 0, true).await; + archive + .object("video", 0, vec![frames(true, 0, 0..2), frames(true, 1, 0..2)]) + .await; + archive + .object("audio", 0, vec![frames(true, 0, 0..2), frames(true, 1, 0..2)]) + .await; + archive + .object("errored", 0, vec![frames(true, 0, 0..1), frames(true, 1, 0..1)]) + .await; + archive.end("video", End::Finish(At { group: 2, frame: 0 })).await; + archive.end("audio", End::Finish(At { group: 1, frame: 2 })).await; + archive.end("errored", End::Error(64 + 7)).await; + // Empty groups store no object, only the end past them. + archive.object("tail", 0, vec![frames(true, 0, 0..1)]).await; + archive.end("tail", End::Finish(At { group: 4, frame: 0 })).await; + archive.complete().await; + + let (broadcast, _run) = serve(&archive.store).await; + for (track, end) in [ + ("video", Position::group(2)), + ("audio", Position { group: 1, frame: 2 }), + ] { + let mut reader = subscribe(&broadcast, track).await; + assert_eq!(next(&mut reader).await.unwrap(), (0, expected(true, 0, 0..2))); + assert_eq!(next(&mut reader).await.unwrap(), (1, expected(true, 1, 0..2))); + assert!(next(&mut reader).await.is_none()); + assert_eq!(reader.finished().await.unwrap(), end, "{track}"); + } + + let mut reader = subscribe(&broadcast, "errored").await; + assert_eq!(next(&mut reader).await.unwrap().0, 0); + assert_eq!(next(&mut reader).await.unwrap().0, 1); + assert!(next(&mut reader).await.is_none()); + + // The sequence below the end replays as an empty group, and those before it as dropped. + let mut reader = subscribe(&broadcast, "tail").await; + assert_eq!(next(&mut reader).await.unwrap(), (0, expected(true, 0, 0..1))); + assert_eq!(next(&mut reader).await.unwrap(), (3, Vec::new())); + assert!(next(&mut reader).await.is_none()); + assert_eq!(reader.finished().await.unwrap(), Position::group(4)); +} + +/// Markers written while the consumer follows end the tracks then. +#[tokio::test(start_paused = true)] +async fn markers_end_a_followed_archive() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + archive.object("video", 0, vec![frames(true, 0, 0..2)]).await; + + let (broadcast, _run) = serve(&archive.store).await; + let mut reader = subscribe(&broadcast, "video").await; + archive.object("video", 2, vec![frames(true, 0, 2..3)]).await; + archive.end("video", End::Finish(At { group: 1, frame: 0 })).await; + archive.complete().await; + assert_eq!(next(&mut reader).await.unwrap(), (0, expected(true, 0, 0..3))); + assert!(next(&mut reader).await.is_none()); +} + +/// Groups and datagrams sharing one sequence space replay in their recorded forms. +#[tokio::test(start_paused = true)] +async fn groups_and_datagrams_keep_their_forms() { + let archive = Archive::new(); + archive.track("mixed", 0, false).await; + archive + .object( + "mixed", + 0, + vec![frames(false, 0, 0..1), datagram(false, 1), frames(false, 2, 0..1)], + ) + .await; + + let (broadcast, _run) = serve(&archive.store).await; + let track = broadcast.consume().track("mixed").unwrap(); + let subscription = Subscription::default() + .with_range(0..) + .with_order(Order::Ascending) + .with_max_delay(None); + let mut reader = track.subscribe(subscription).await.unwrap(); + + archive + .object("mixed", 0, vec![datagram(false, 3), frames(false, 4, 0..1)]) + .await; + let datagram = timeout(10 * REFRESH, reader.recv_datagram()) + .await + .expect("the datagram arrives") + .unwrap() + .unwrap(); + assert_eq!((datagram.sequence, &datagram.payload[..]), (3, &b"3/0"[..])); + + // A stored datagram is never fetched as a group. + assert!(fetch(&broadcast, "mixed", 1).await.is_err()); + assert_eq!(fetch(&broadcast, "mixed", 0).await.unwrap(), ["0/0"]); + assert_eq!(fetch(&broadcast, "mixed", 2).await.unwrap(), ["2/0"]); + + // The newest stored group arrives live as a group, still open. + let mut sequences = Vec::new(); + while sequences.last() != Some(&4) { + let group = timeout(10 * REFRESH, reader.recv_group()) + .await + .unwrap() + .unwrap() + .unwrap(); + sequences.push(group.sequence); + } + assert!(!sequences.contains(&1) && !sequences.contains(&3), "{sequences:?}"); +} + +/// A track removed and re-added under its name replays both runs in sequence. +#[tokio::test(start_paused = true)] +async fn a_re_added_track_replays_both_runs() { + let store = Store::new(Mock::memory(), "rec"); + let source = broadcast::Info::new().produce(); + let info = track::Info::default().with_timescale(Timescale::MILLI); + let config = crate::producer::Config::default().with_flush(Duration::from_secs(1)); + let recording = tokio::spawn(crate::Producer::new(store.clone(), source.consume(), config).run()); + + for (run, sequences) in [(0, 0..2), (1, 2..4)] { + let track = source.create_track("video", info.clone()).unwrap(); + while track.subscription().is_none() { + sleep(Duration::from_millis(1)).await; + } + for sequence in sequences { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + let pts = Timestamp::from_millis(sequence * 1000).unwrap(); + group.write_frame(pts, format!("run{run}/{sequence}")).unwrap(); + group.finish().unwrap(); + } + track.finish().unwrap(); + sleep(Duration::from_secs(2)).await; + drop(track); + sleep(Duration::from_millis(10)).await; + } + source.close(); + recording.await.unwrap().unwrap(); + + let (broadcast, _run) = serve(&store).await; + let mut reader = subscribe(&broadcast, "video").await; + let mut read = Vec::new(); + while let Some((_, frames)) = next(&mut reader).await { + read.extend(frames); + } + assert_eq!(read, ["0@run0/0", "1000@run0/1", "2000@run1/2", "3000@run1/3"]); +} + +#[tokio::test(start_paused = true)] +async fn an_empty_or_unrecognized_prefix_is_refused() { + let open = |store: Store| async move { + let broadcast = broadcast::Info::new().produce(); + Consumer::open(store, broadcast, Config::default()).await.err() + }; + + let archive = Archive::new(); + assert!(matches!(open(archive.store.clone()).await, Some(Error::NotFound(_)))); + + // A key the layout does not name, at the root or under a track. + let archive = Archive::new(); + archive.track("video", 0, true).await; + archive.raw("stray", b"x").await; + assert!(matches!(open(archive.store.clone()).await, Some(Error::Path(_)))); + + let archive = Archive::new(); + archive.track("video", 0, true).await; + archive.raw("video/segments/0000000000000000000", b"x").await; + assert!(matches!(open(archive.store.clone()).await, Some(Error::Path(_)))); + + // A directory that is not a canonical track name. + let archive = Archive::new(); + archive + .raw("catalog.json/.info", br#"{"version":2,"priority":0}"#) + .await; + assert!(matches!(open(archive.store.clone()).await, Some(Error::Track))); +} + +/// Object stores promise no listing order. +#[tokio::test(start_paused = true)] +async fn an_unordered_listing_is_sorted() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + for sequence in 0..4 { + archive.object("video", 0, vec![frames(true, sequence, 0..1)]).await; + } + archive.store.inner().unordered(); + + let (broadcast, _run) = serve(&archive.store).await; + let mut reader = subscribe(&broadcast, "video").await; + for sequence in 0..3 { + assert_eq!( + next(&mut reader).await.unwrap(), + (sequence, expected(true, sequence, 0..1)) + ); + } + assert_eq!(super::tests::sequence(&mut reader).await, Some(3)); +} diff --git a/rs/moq-archive/src/consumer/track.rs b/rs/moq-archive/src/consumer/track.rs new file mode 100644 index 00000000000..914afc5b5dd --- /dev/null +++ b/rs/moq-archive/src/consumer/track.rs @@ -0,0 +1,320 @@ +//! One served track: its live feed of what the archive gains, and fills of what it holds. + +use std::sync::{Arc, Mutex}; + +use futures::future::{self, Either}; +use moq_net::track::{Fill, Position}; +use moq_net::{Timescale, Timestamp, group, track}; +use object_store::ObjectStore; + +use super::Shared; +use super::index::Index; +use crate::layout::{End, Entry, Frame, Object}; +use crate::{Error, Info, Result}; + +/// A track's live feed: its newest stored sequence when found, then everything stored after. +pub(super) struct Live { + producer: track::Producer, + timescale: Option, + /// The newest group, kept open since a later object may continue it. + open: Option, + /// One past the newest sequence fed, once the feed declared where it starts; fills serve + /// everything before that start. + next: Option, +} + +struct Open { + group: group::Producer, + sequence: u64, + /// The index of the next frame it takes. + next: u64, +} + +impl Live { + pub fn new(producer: track::Producer) -> Self { + let timescale = producer.info().timescale; + Self { + producer, + timescale, + open: None, + next: None, + } + } + + pub fn is_started(&self) -> bool { + self.next.is_some() + } + + /// Feed `objects`' entries from sequence `from` on, in order. + pub fn push(&mut self, objects: &[Arc], from: u64) -> Result<()> { + for object in objects { + for (index, entry) in object.entries.iter().enumerate() { + if entry.sequence() < from { + continue; + } + if self.next.is_none() { + // Readers ask fills for everything older. + self.producer.start_at(entry.sequence())?; + } + let first = match index { + 0 => object.frame_start, + _ => 0, + }; + self.entry(entry, first)?; + self.next = Some(entry.sequence() + 1); + } + } + Ok(()) + } + + fn entry(&mut self, entry: &Entry, first: u64) -> Result<()> { + match entry { + Entry::Datagram { sequence, frame } => { + // A later sequence completes the open group. + self.close()?; + let timestamp = timestamp(frame.timestamp, self.timescale)?; + self.producer + .insert_datagram(*sequence, timestamp, frame.payload.clone())?; + } + Entry::Group { sequence, frames } => { + if self.open.as_ref().is_some_and(|open| open.sequence != *sequence) { + self.close()?; + } + // The group stays in `open` while it is written, so a failure aborts it with + // the track rather than dropping it. + let open = match &mut self.open { + Some(open) => { + if open.next != first { + return Err(Error::Sequence); + } + open + } + None => { + let group = self.producer.create_group(group::Info { sequence: *sequence })?; + let open = self.open.insert(Open { + group, + sequence: *sequence, + next: first, + }); + if first > 0 { + // Its head was pruned. + open.group.start_at(first)?; + } + open + } + }; + for frame in frames { + let timestamp = timestamp(frame.timestamp, self.timescale)?; + open.group.write_frame(timestamp, frame.payload.clone())?; + open.next += 1; + } + } + } + Ok(()) + } + + fn close(&mut self) -> Result<()> { + if let Some(mut open) = self.open.take() { + open.group.finish()?; + } + Ok(()) + } + + /// End the feed with `err` when serving the archive failed, the open group included, so + /// readers see why rather than a dropped producer. + pub fn abort(self, err: moq_net::Error) { + if let Some(open) = self.open { + let _ = open.group.abort(err.clone()); + } + let _ = self.producer.abort(err); + } + + /// End the feed as the archive recorded. + pub fn end(&mut self, end: End, track: &str) -> Result<()> { + self.close()?; + match end { + End::Finish(at) => { + self.producer.finish_at(Position { + group: at.group, + frame: at.frame, + })?; + // An empty group stores no object, yet the end only settles once the live edge + // reaches it. The sequence just below the end replays as the empty group a + // source most often ended with, and any before it as dropped. + let fin = at.group + u64::from(at.frame != 0); + let from = self.next.unwrap_or(0); + if from < fin { + self.producer.drop_groups(from..fin - 1, moq_net::StreamError::NotFound); + self.producer + .create_group(group::Info { sequence: fin - 1 })? + .finish()?; + } + } + End::Error(code) => { + // An abort would refuse every later reader the stored history, so the track + // ends cleanly after its newest stored sequence instead. + tracing::warn!(track, code, "archived track ended with an error"); + self.producer.finish()?; + } + } + Ok(()) + } +} + +/// Serve `fill` from the archive, a group at a time in the fill's order, until every group is +/// answered or nobody waits on it. +pub(super) async fn serve(shared: Arc>, index: Arc>, fill: Fill) { + let mut last = None; + while fill.is_used() + && let Some(sequence) = fill.next() + && last != Some(sequence) + { + last = Some(sequence); + serve_group(&shared, &index, &fill, sequence).await; + } +} + +/// Answer group `sequence` of `fill` from the objects holding it, drop the run of sequences +/// around it that the archive does not hold, or refuse it for now when the store fails. +async fn serve_group(shared: &Shared, index: &Mutex, fill: &Fill, sequence: u64) { + let frame = fill.frame_start(sequence); + let (holding, after, timed, timescale) = { + let index = index.lock().unwrap(); + let (holding, after) = index.holding(sequence, frame); + (holding, after, index.timed(), index.timescale) + }; + let next = after.map_or(u64::MAX, |start| start.0); + + let mut objects = Vec::with_capacity(holding.len()); + for (start, path) in &holding { + let unwanted = std::pin::pin!(fill.unwanted(sequence)); + let load = std::pin::pin!(super::object(shared, path, *start, timed)); + let loaded = match future::select(unwanted, load).await { + // Every reader narrowed past it: the fill moves on to the groups it still owes. + Either::Left(_) => return, + Either::Right((loaded, _)) => loaded, + }; + match loaded { + Ok(object) => objects.push(object), + Err(Error::NotFound(_)) => { + // Pruned since it was listed. + index.lock().unwrap().forget(*start); + fill.drop_groups(sequence..sequence + 1); + return; + } + Err(err) => { + // The store still lists it, so a later reader asks again. + tracing::warn!(%path, sequence, %err, "archived group is unavailable"); + let err = moq_net::Error::Stream(moq_net::StreamError::Internal); + fill.refuse_groups(sequence..sequence + 1, err); + return; + } + } + } + + let Some(first) = objects.first() else { + // Older than the oldest stored object: pruned, or never stored. + fill.drop_groups(0..next); + return; + }; + let Some(position) = first.entries.iter().position(|entry| entry.sequence() >= sequence) else { + // A gap after the object's last entry. + let last = first.entries.last().map_or(0, |entry| entry.sequence() + 1); + fill.drop_groups(last..next); + return; + }; + let frames = match &first.entries[position] { + Entry::Group { + sequence: found, + frames, + } if *found == sequence => frames, + // A datagram stays lossy and uncached, never a fetched group. + Entry::Datagram { sequence: found, .. } if *found == sequence => { + fill.drop_groups(sequence..sequence + 1); + return; + } + // A gap between two entries. + other => { + let gap = match position { + 0 => sequence, + _ => first.entries[position - 1].sequence() + 1, + }; + fill.drop_groups(gap..other.sequence()); + return; + } + }; + let start = match position { + 0 => first.frame_start, + _ => 0, + }; + if start > frame { + // Its head was pruned. + fill.drop_groups(sequence..sequence + 1); + return; + } + + // The group's frames, with their indices, across every object holding it. + let mut stored: Vec<(u64, &Frame)> = (start..).zip(frames).collect(); + for object in &objects[1..] { + let next = start + stored.len() as u64; + match object.entries.first() { + Some(Entry::Group { + sequence: found, + frames, + }) if *found == sequence && object.frame_start == next => { + stored.extend((next..).zip(frames)); + } + _ => { + // A continuation went missing, so the group can't be served whole. + fill.drop_groups(sequence..sequence + 1); + return; + } + } + } + + if !fill.is_wanted(sequence) { + return; + } + let Ok(mut producer) = fill.accept(sequence, None) else { + return; + }; + for (index, stored) in stored { + if index < frame { + continue; + } + let written = timestamp(stored.timestamp, timescale) + .map_err(|_| moq_net::Error::ProtocolViolation) + .and_then(|timestamp| producer.write_frame(timestamp, stored.payload.clone())); + if let Err(err) = written { + tracing::warn!(sequence, %err, "archived group is malformed"); + let _ = producer.abort(err); + return; + } + } + let _ = producer.finish(); +} + +/// The served properties of a track stored with `info`: no max age, since the archive's +/// retention bounds what it holds and the source's would expire archived history. +pub(super) fn info(info: &Info) -> Result { + Ok(track::Info::default() + .with_priority(info.priority) + .with_max_age(None) + .with_timescale(timescale(info)?)) +} + +/// A stored track's timescale, `None` when untimed. +pub(super) fn timescale(info: &Info) -> Result> { + info.timescale + .map(|timescale| Timescale::new(timescale).map_err(|_| Error::Timescale(timescale))) + .transpose() +} + +/// A stored timestamp on its track, which must be timed exactly when the frame is. +fn timestamp(value: Option, timescale: Option) -> Result> { + match (value, timescale) { + (Some(value), Some(timescale)) => Ok(Some(Timestamp::new(value, timescale).map_err(|_| Error::Id(value))?)), + (None, None) => Ok(None), + _ => Err(Error::Timing), + } +} diff --git a/rs/moq-archive/src/error.rs b/rs/moq-archive/src/error.rs index db44446b2ea..0d733ba7df5 100644 --- a/rs/moq-archive/src/error.rs +++ b/rs/moq-archive/src/error.rs @@ -14,7 +14,7 @@ pub enum Error { #[error("unsupported version {0}")] Version(u64), - /// A group ID, segment ID, or frame timestamp is outside 0 through 2^53 - 1. + /// A sequence, frame index, or timestamp is outside 0 through 2^53 - 1. #[error("{0} exceeds the recording id range")] Id(u64), @@ -48,10 +48,6 @@ pub enum Error { #[error("group sequences are not strictly ascending")] Sequence, - /// An object's table does not hold exactly the frames its record names. - #[error("object does not match its record")] - Span, - /// The binary table is truncated, overlapping, gapped, or out of range. #[error("malformed table")] Table, @@ -68,10 +64,6 @@ pub enum Error { #[error("invalid path: {0}")] Path(String), - /// A backend returned a directory from the flat object listing API. - #[error("directory listing is unsupported: {0}")] - Directory(String), - /// A create collided with different object bytes. #[error("conflicting object: {0}")] Conflict(String), @@ -80,7 +72,7 @@ pub enum Error { #[error("json: {0}")] Json(String), - /// Publishing the replayed archive failed. + /// Serving the archive on its broadcast failed. #[error("moq: {0}")] Moq(String), diff --git a/rs/moq-archive/src/info.rs b/rs/moq-archive/src/info.rs index 38024bd3733..286f0a80582 100644 --- a/rs/moq-archive/src/info.rs +++ b/rs/moq-archive/src/info.rs @@ -19,9 +19,7 @@ pub struct Info { pub timescale: Option, } -/// [`Info`] as stored, before its priority is read in its version's encoding. It also takes the -/// `epoch` an older `Writer` recorded, so [`Reader`](crate::Reader) still replays those -/// recordings; that goes when `Reader` does. +/// [`Info`] as stored, before its priority is read in its version's encoding. #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct Stored { @@ -29,8 +27,6 @@ struct Stored { priority: i64, #[serde(default)] timescale: Option, - #[serde(default, rename = "epoch")] - _epoch: Option, } impl TryFrom for Info { @@ -174,14 +170,12 @@ mod tests { assert!(matches!(Info::new(0, Some(ID_MAX + 1)), Err(Error::Timescale(_)))); } - #[test] - fn an_older_writers_epoch_is_ignored() { - let info = Info::decode(br#"{"version":2,"priority":128,"timescale":1000,"epoch":"x"}"#).unwrap(); - assert_eq!((info.version, info.priority, info.timescale), (2, 0, Some(1000))); - } - #[test] fn unknown_members_and_missing_fields_are_refused() { + assert!(matches!( + Info::decode(br#"{"version":2,"priority":128,"timescale":1000,"epoch":"x"}"#), + Err(Error::Json(_)) + )); assert!(matches!( Info::decode(br#"{"version":3,"priority":0,"timescale":1,"extra":true}"#), Err(Error::Json(_)) diff --git a/rs/moq-archive/src/layout.rs b/rs/moq-archive/src/layout.rs index a2c44366129..12edbf93491 100644 --- a/rs/moq-archive/src/layout.rs +++ b/rs/moq-archive/src/layout.rs @@ -9,8 +9,7 @@ use object_store::path::Path; use serde::{Deserialize, Serialize}; use crate::path::{check_id, decode_track, encode_track, push}; -use crate::segment::Frame; -use crate::{Error, ID_WIDTH, Result, VERSION}; +use crate::{Error, ID_WIDTH, Result, VERSION, VERSION_MIN}; /// The broadcast's completion marker, beside its track segments. pub(crate) const BROADCAST_END: &str = ".end"; @@ -144,6 +143,15 @@ impl End { } } +/// One frame's timestamp and original payload. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Frame { + /// Absolute timestamp in the track's timescale, or `None` on an untimed track. + pub timestamp: Option, + /// Original frame payload, including any group-scoped compression. + pub payload: Bytes, +} + /// One stored sequence, in the form it was forwarded in. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum Entry { @@ -190,6 +198,15 @@ impl Object { }) } + /// Its payload bytes plus a small per-frame overhead, for a byte-bounded cache. + pub(crate) fn weight(&self) -> u64 { + let frames = self.entries.iter().flat_map(|entry| match entry { + Entry::Group { frames, .. } => frames.as_slice(), + Entry::Datagram { frame, .. } => std::slice::from_ref(frame), + }); + frames.map(|frame| frame.payload.len() as u64 + 32).sum::() + 64 + } + /// The position just past the last stored frame or datagram. pub(crate) fn end(&self) -> Result { let last = self.entries.last().ok_or(Error::Empty)?; @@ -257,10 +274,11 @@ impl Object { } /// Decode a whole table before slicing any payload; `timed` is whether the track's `.info` - /// declares a timescale, which says whether each frame stores a timestamp. + /// declares a timescale, which says whether each frame stores a timestamp. Version 2 objects + /// are identical to the current ones. pub(crate) fn decode(mut buf: impl Buf, timed: bool) -> Result { let version = read(&mut buf)?; - if version != VERSION { + if !(VERSION_MIN..=VERSION).contains(&version) { return Err(Error::Version(version)); } let frame_start = read(&mut buf)?; @@ -535,8 +553,12 @@ mod tests { let bytes = object.encode(false).unwrap(); // version, frame start, one entry: sequence 2, a group of two frames, then the payloads. assert_eq!(bytes.as_ref(), &[VERSION as u8, 0, 1, 2, 0, 2, 1, 2, b'x', b'y', b'z']); - assert_eq!(Object::decode(bytes, false).unwrap(), object); + assert_eq!(Object::decode(bytes.clone(), false).unwrap(), object); assert!(matches!(object.encode(true), Err(Error::Timing))); + + let mut v2 = bytes.to_vec(); + v2[0] = 2; + assert_eq!(Object::decode(&v2[..], false).unwrap(), object); } #[test] @@ -589,9 +611,11 @@ mod tests { Object::decode(&[VERSION as u8, 0, 0x7f][..], false), Err(Error::Table) )); - assert!(matches!( - Object::decode(&[VERSION as u8 + 1, 0, 0][..], false), - Err(Error::Version(_)) - )); + for version in [1, VERSION as u8 + 1] { + assert!(matches!( + Object::decode(&[version, 0, 0][..], false), + Err(Error::Version(_)) + )); + } } } diff --git a/rs/moq-archive/src/lib.rs b/rs/moq-archive/src/lib.rs index 1aea80ccf2f..f50a4a1426e 100644 --- a/rs/moq-archive/src/lib.rs +++ b/rs/moq-archive/src/lib.rs @@ -1,7 +1,7 @@ //! Broadcast archives on any [`object_store::ObjectStore`]. //! //! A [`Producer`] stores every track of one broadcast under a key prefix the host picks, and a -//! [`Reader`] serves a recording back through a `moq_net` broadcast. Callers that need runtime +//! [`Consumer`] serves that prefix back as an ordinary `moq_net` broadcast. Callers that need runtime //! dispatch supply `Arc`; the archive API itself stays generic. //! //! # Layout @@ -59,6 +59,7 @@ pub use object_store; +pub mod consumer; mod error; /// The `.info` object holding a track's immutable properties. pub mod info; @@ -67,29 +68,24 @@ mod layout; mod mock; mod path; pub mod producer; -pub mod reader; -/// The binary segment object holding a record's groups and frames. -mod segment; -/// Recording objects on an object store. +/// The object store and key prefix an archive lives under. mod store; +pub use consumer::Consumer; pub use error::{Error, Result}; pub use info::Info; -pub(crate) use path::Key; pub use producer::Producer; -pub use reader::Reader; -pub(crate) use segment::Object; pub use store::Store; /// Recording format version written into `.info` and the binary envelope. pub const VERSION: u64 = 3; -/// The oldest recording format version [`Reader`] accepts. Version 2 stored `.info`'s +/// The oldest recording format version a [`Consumer`] accepts. Version 2 stored `.info`'s /// priority as its moq-lite byte and is otherwise identical. pub(crate) const VERSION_MIN: u64 = 2; -/// Largest group, segment, or timestamp value the format allows (`2^53 - 1`). +/// Largest sequence, frame index, or timestamp value the format allows (`2^53 - 1`). pub const ID_MAX: u64 = (1 << 53) - 1; -/// Decimal width of segment ID filename fields. +/// Decimal width of each object key field. pub(crate) const ID_WIDTH: usize = 19; diff --git a/rs/moq-archive/src/mock.rs b/rs/moq-archive/src/mock.rs index 003ef4e5882..b0e82f80654 100644 --- a/rs/moq-archive/src/mock.rs +++ b/rs/moq-archive/src/mock.rs @@ -31,6 +31,8 @@ struct State { fail_puts: Vec, /// GETs whose path contains any of these return Not Found. hide_gets: Vec, + /// GETs whose path contains any of these fail, as a store outage would. + fail_gets: Vec, /// GETs whose path contains the pattern wait for a permit before reading. stall_gets: Vec<(String, Arc)>, /// Deletes whose path contains any of these fail. @@ -99,6 +101,17 @@ impl Mock { self } + pub fn fail_gets(&self, pattern: &str) -> &Self { + self.state().fail_gets.push(pattern.to_string()); + self + } + + /// End every [`fail_gets`](Self::fail_gets) outage. + pub fn heal_gets(&self) -> &Self { + self.state().fail_gets.clear(); + self + } + /// Hold every GET whose path contains `pattern` until the returned semaphore gets a permit. pub fn stall_gets(&self, pattern: &str) -> Arc { let gate = Arc::new(tokio::sync::Semaphore::new(0)); @@ -133,19 +146,6 @@ impl Mock { self.hold.notify_waiters(); } - /// Clear every injected failure. - pub fn heal(&self) { - let mut state = self.state(); - state.fail_puts.clear(); - state.hide_gets.clear(); - state.stall_gets.clear(); - state.fail_deletes.clear(); - state.fail_lists = false; - state.holds.clear(); - drop(state); - self.hold.notify_waiters(); - } - /// Wait out a [`hold_puts`](Self::hold_puts) gate. /// /// The waiter is armed before the check, so a `release` in between is not missed. @@ -250,6 +250,12 @@ impl ObjectStore for Mock { source: "hidden".into(), }); } + if self.state().fail_gets.iter().any(|p| location.as_ref().contains(p)) { + return Err(object_store::Error::Generic { + store: "mock", + source: "outage".into(), + }); + } let stall = self .state() .stall_gets diff --git a/rs/moq-archive/src/path.rs b/rs/moq-archive/src/path.rs index 576067c3ff3..2f5488fd869 100644 --- a/rs/moq-archive/src/path.rs +++ b/rs/moq-archive/src/path.rs @@ -1,7 +1,7 @@ use object_store::path::{Path, PathPart}; use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode}; -use crate::{Error, ID_MAX, ID_WIDTH, Result}; +use crate::{Error, ID_MAX, Result}; /// Bytes that are not `A-Z a-z 0-9 _ -` and must be percent-encoded in a track name. const TRACK: &AsciiSet = &NON_ALPHANUMERIC.remove(b'_').remove(b'-'); @@ -26,131 +26,11 @@ pub(crate) fn decode_track(encoded: &str) -> Result { Ok(decoded.into_owned()) } -/// Write a segment ID as 19 zero-padded decimal digits. -pub(crate) fn format_id(id: u64) -> Result { - check_id(id)?; - Ok(format!("{id:0width$}", width = ID_WIDTH)) -} - -/// Parse a 19-digit decimal ID field, refusing values outside the recording range. -pub(crate) fn parse_id(field: &str) -> Result { - if field.len() != ID_WIDTH || !field.bytes().all(|b| b.is_ascii_digit()) { - return Err(Error::Path(field.to_string())); - } - let id: u64 = field.parse().map_err(|_| Error::Path(field.to_string()))?; - check_id(id) -} - /// Refuse an ID outside 0 through 2^53 - 1. pub(crate) fn check_id(id: u64) -> Result { if id > ID_MAX { Err(Error::Id(id)) } else { Ok(id) } } -/// A recording object key under an application prefix. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum Key { - /// `/.info` - Info { - /// The unencoded track name. - track: String, - }, - /// `/segments/`: record `segment` of a track's timeline, or for a - /// timeline track, the window groups closed while committing that record. - Segments { - /// The unencoded track name. - track: String, - /// The record's sequence. - segment: u64, - }, -} - -impl Key { - /// A track's `.info` object. - pub fn info(track: impl Into) -> Result { - let track = track.into(); - encode_track(&track)?; - Ok(Self::Info { track }) - } - - /// The object at `segments/`. - pub fn segments(track: impl Into, segment: u64) -> Result { - let track = track.into(); - encode_track(&track)?; - check_id(segment)?; - Ok(Self::Segments { track, segment }) - } - - /// The unencoded track name this key belongs to. - pub fn track(&self) -> &str { - match self { - Self::Info { track } | Self::Segments { track, .. } => track, - } - } - - /// Encode this key under `prefix`. - pub(crate) fn path(&self, prefix: &Path) -> Result { - let path = push(prefix, &encode_track(self.track())?)?; - match self { - Self::Info { .. } => push(&path, ".info"), - Self::Segments { segment, .. } => { - let path = push(&path, "segments")?; - push(&path, &format_id(*segment)?) - } - } - } - - /// Parse a store location relative to `prefix`. - pub(crate) fn parse(prefix: &Path, location: &Path) -> Result { - if prefix.as_ref().is_empty() { - return parse_parts(location.parts(), location); - } - let parts = location - .prefix_match(prefix) - .ok_or_else(|| Error::Path(location.to_string()))?; - parse_parts(parts, location) - } -} - -fn parse_parts<'a>(mut parts: impl Iterator>, location: &Path) -> Result { - let encoded = parts.next().ok_or_else(|| Error::Path(location.to_string()))?; - let track = decode_track(encoded.as_ref())?; - let kind = parts.next().ok_or_else(|| Error::Path(location.to_string()))?; - match kind.as_ref() { - ".info" => { - if parts.next().is_some() { - return Err(Error::Path(location.to_string())); - } - Ok(Key::Info { track }) - } - "segments" => { - let name = parts.next().ok_or_else(|| Error::Path(location.to_string()))?; - if parts.next().is_some() { - return Err(Error::Path(location.to_string())); - } - Ok(Key::Segments { - track, - segment: parse_id(name.as_ref())?, - }) - } - _ => Err(Error::Path(location.to_string())), - } -} - -pub(crate) fn track_prefix(track: &str) -> Result { - push(&Path::ROOT, &encode_track(track)?) -} - -pub(crate) fn segments_prefix(prefix: &Path, track: &str) -> Result { - let path = append(prefix, &track_prefix(track)?); - push(&path, "segments") -} - -fn append(base: &Path, relative: &Path) -> Path { - let mut path = base.clone(); - path.extend(relative.parts()); - path -} - pub(crate) fn push(base: &Path, segment: &str) -> Result { let part = PathPart::parse(segment).map_err(|err| Error::Path(err.to_string()))?; Ok(base.clone().join(part)) @@ -196,92 +76,4 @@ mod tests { assert!(matches!(decode_track("a/b"), Err(Error::Track))); assert!(matches!(decode_track(".info"), Err(Error::Track))); } - - #[test] - fn id_fields_are_19_digits() { - assert_eq!(format_id(0).unwrap(), "0000000000000000000"); - assert_eq!(format_id(ID_MAX).unwrap(), "0009007199254740991"); - assert_eq!(parse_id("0000000000000000000").unwrap(), 0); - assert_eq!(parse_id("0009007199254740991").unwrap(), ID_MAX); - } - - #[test] - fn ids_outside_the_json_range_are_rejected() { - assert!(matches!(format_id(ID_MAX + 1), Err(Error::Id(_)))); - assert!(matches!(check_id(u64::MAX), Err(Error::Id(_)))); - assert!(matches!( - parse_id("0009007199254740992"), - Err(Error::Id(n)) if n == ID_MAX + 1 - )); - // The largest QUIC varint still fits the 19-digit field. - let varint = (1u64 << 62) - 1; - assert!(matches!(format_id(varint), Err(Error::Id(n)) if n == varint)); - assert!(matches!(parse_id("4611686018427387903"), Err(Error::Id(n)) if n == varint)); - assert!(matches!(parse_id("5"), Err(Error::Path(_)))); - assert!(matches!(parse_id("000000000000000000X"), Err(Error::Path(_)))); - } - - #[test] - fn keys_roundtrip_under_a_prefix() { - let prefix = Path::from("rec/1"); - for key in [ - Key::info("catalog.json").unwrap(), - Key::segments("video", 5).unwrap(), - Key::segments("timeline.z", 0).unwrap(), - Key::segments("timeline.z", ID_MAX).unwrap(), - ] { - let path = key.path(&prefix).unwrap(); - assert_eq!(Key::parse(&prefix, &path).unwrap(), key); - } - } - - #[test] - fn encoded_track_is_not_double_encoded() { - let path = Key::info("catalog.json").unwrap().path(&Path::from("rec")).unwrap(); - assert_eq!(path.as_ref(), "rec/catalog%2Ejson/.info"); - } - - #[test] - fn direct_construction_is_validated_when_serialized() { - let prefix = Path::from("rec"); - let empty = Key::Info { track: String::new() }; - assert!(matches!(empty.path(&prefix), Err(Error::Track))); - let segment = Key::Segments { - track: "t".to_string(), - segment: ID_MAX + 1, - }; - assert!(matches!(segment.path(&prefix), Err(Error::Id(_)))); - - // Everything that does serialize parses back to the same key. - for key in [ - Key::Info { - track: "a/b".to_string(), - }, - Key::Segments { - track: ".x".to_string(), - segment: ID_MAX, - }, - Key::Segments { - track: "é".to_string(), - segment: 0, - }, - ] { - let path = key.path(&prefix).unwrap(); - assert_eq!(Key::parse(&prefix, &path).unwrap(), key); - } - } - - #[test] - fn keys_parse_under_an_empty_prefix() { - let key = Key::segments("catalog.json", 2).unwrap(); - let path = key.path(&Path::ROOT).unwrap(); - assert_eq!(path.as_ref(), "catalog%2Ejson/segments/0000000000000000002"); - assert_eq!(Key::parse(&Path::ROOT, &path).unwrap(), key); - } - - #[test] - fn a_retired_directory_is_not_a_key() { - let path = Path::from("video/groups/0000000000000000001.0000000000000000001"); - assert!(matches!(Key::parse(&Path::ROOT, &path), Err(Error::Path(_)))); - } } diff --git a/rs/moq-archive/src/producer/track.rs b/rs/moq-archive/src/producer/track.rs index 23ab90a7e65..e92d338601d 100644 --- a/rs/moq-archive/src/producer/track.rs +++ b/rs/moq-archive/src/producer/track.rs @@ -16,8 +16,7 @@ use object_store::{ObjectStore, ObjectStoreExt, PutMode, PutPayload}; use super::Shared; use super::ordered::{Item, Ordered}; use super::resume::Stored; -use crate::layout::{self, End, Entry, Object}; -use crate::segment::Frame; +use crate::layout::{self, End, Entry, Frame, Object}; use crate::{Error, Info, Result}; /// Record each run of the track `name` reported through `queue`, until the queue closes with the diff --git a/rs/moq-archive/src/reader/index.rs b/rs/moq-archive/src/reader/index.rs deleted file mode 100644 index 4cafc63d2c9..00000000000 --- a/rs/moq-archive/src/reader/index.rs +++ /dev/null @@ -1,170 +0,0 @@ -use std::collections::{BTreeMap, HashMap}; -use std::ops::Range; - -use hang::timeline::{Position, Record}; - -use crate::path::check_id; - -/// One committed record: the object at `segments/` and the frames it holds. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) struct Span { - /// The record's sequence, which names its object. - pub sequence: u64, - /// The first frame, inclusive. - pub start: Position, - /// The end, exclusive. - pub end: Position, -} - -impl Span { - /// Build from a record, refusing an empty span or IDs past the recording range. - fn new(record: &Record) -> Option { - check_id(record.sequence).ok()?; - check_id(record.start.group).ok()?; - check_id(record.end.group).ok()?; - (record.start < record.end).then_some(Self { - sequence: record.sequence, - start: record.start, - end: record.end, - }) - } -} - -/// One track's committed spans. -#[derive(Default)] -struct Spans { - /// Each span keyed by its start. - spans: BTreeMap, - /// Each span's start keyed by its sequence, so a pop can find it. - sequences: BTreeMap, -} - -/// The spans each track's replayed timeline window commits. -#[derive(Default)] -pub(super) struct Index { - tracks: HashMap, -} - -impl Index { - /// Add `track`'s record. - /// - /// A malformed record or one overlapping another is left out, so its frames behave like ones - /// the source never delivered; the track's other records stay usable. - pub fn push(&mut self, track: &str, record: &Record) { - let Some(span) = Span::new(record) else { - tracing::warn!(track, sequence = record.sequence, "ignoring a malformed archive record"); - return; - }; - - let spans = self.tracks.entry(track.to_string()).or_default(); - let before = spans.spans.range(..span.end).next_back(); - if before.is_some_and(|(_, prev)| prev.end > span.start) || spans.sequences.contains_key(&span.sequence) { - tracing::warn!( - track, - sequence = record.sequence, - "ignoring an overlapping archive record" - ); - return; - } - spans.spans.insert(span.start, span); - spans.sequences.insert(span.sequence, span.start); - } - - /// Remove `track`'s records with these sequences, returning the evicted spans. - pub fn pop(&mut self, track: &str, sequences: Range) -> Vec { - let Some(spans) = self.tracks.get_mut(track) else { - return Vec::new(); - }; - let popped: Vec = spans - .sequences - .range(sequences) - .map(|(sequence, _)| *sequence) - .collect(); - popped - .into_iter() - .filter_map(|sequence| { - let start = spans.sequences.remove(&sequence)?; - spans.spans.remove(&start) - }) - .collect() - } - - /// The spans holding frames of `group` on `track`, in order. - pub fn group(&self, track: &str, group: u64) -> Vec { - let Some(spans) = self.tracks.get(track) else { - return Vec::new(); - }; - let head = Position::group(group); - let mut found: Vec = spans - .spans - .range(..Position::group(group.saturating_add(1))) - .rev() - .map(|(_, span)| *span) - .take_while(|span| span.end > head) - .collect(); - found.reverse(); - found - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ID_MAX; - - fn record(sequence: u64, start: Position, end: Position) -> Record { - Record::new(sequence, sequence * 1000, 1000, start, end) - } - - fn at(group: u64) -> Position { - Position::group(group) - } - - #[test] - fn lookup_finds_every_span_of_a_group() { - let mut index = Index::default(); - index.push("audio", &record(0, at(0), at(3))); - index.push("audio", &record(1, at(5), at(6))); - // Group 7 split across two records, then whole groups again. - index.push("video", &record(0, at(7), Position::new(7, 3))); - index.push("video", &record(1, Position::new(7, 3), at(9))); - - let sequences = |spans: Vec| spans.iter().map(|span| span.sequence).collect::>(); - assert_eq!(sequences(index.group("audio", 1)), vec![0]); - assert!(index.group("audio", 3).is_empty(), "the gap between records"); - assert_eq!(sequences(index.group("audio", 5)), vec![1]); - assert!(index.group("audio", 6).is_empty()); - assert_eq!(sequences(index.group("video", 7)), vec![0, 1]); - assert_eq!(sequences(index.group("video", 8)), vec![1]); - assert!(index.group("chat", 0).is_empty()); - } - - #[test] - fn pop_evicts_only_the_popped_records() { - let mut index = Index::default(); - index.push("video", &record(0, at(0), at(2))); - index.push("video", &record(1, at(2), at(4))); - - let evicted = index.pop("video", 0..1); - assert_eq!(evicted.len(), 1); - assert_eq!(evicted[0].start, at(0)); - assert!(index.group("video", 0).is_empty()); - assert_eq!(index.group("video", 2).len(), 1); - assert!(index.pop("video", 0..1).is_empty(), "already popped"); - } - - #[test] - fn malformed_and_overlapping_records_are_ignored() { - let mut index = Index::default(); - index.push("video", &record(0, at(4), at(7))); - index.push("video", &record(1, at(6), at(9))); - assert_eq!(index.group("video", 8), Vec::new(), "overlapping the earlier span"); - index.push("video", &record(2, at(3), at(5))); - assert_eq!(index.group("video", 3), Vec::new(), "overlapping the later span"); - - index.push("audio", &record(0, at(3), at(3))); - assert!(index.group("audio", 3).is_empty(), "an empty span"); - index.push("data", &record(0, at(ID_MAX + 1), at(ID_MAX + 2))); - assert!(index.group("data", ID_MAX + 1).is_empty(), "past the recording range"); - } -} diff --git a/rs/moq-archive/src/reader/mod.rs b/rs/moq-archive/src/reader/mod.rs deleted file mode 100644 index 1b8348fa5b0..00000000000 --- a/rs/moq-archive/src/reader/mod.rs +++ /dev/null @@ -1,708 +0,0 @@ -//! Serve an archive's groups through a caller-supplied broadcast. -//! -//! A [`Reader`] replays each track's timeline onto its timeline track and answers FETCH for every -//! frame those timelines commit. A group maps to the records holding it, each one object fetched -//! with a single GET, validated against its record, and kept in a byte-bounded cache so adjacent -//! groups reuse it. A request for one track never downloads another track's object. A group whose -//! records do not reach its end yet, such as an append-only log, grows as later records commit. -//! Timeline groups remain FETCHable from their original objects after broadcast-cache eviction. -//! -//! The reader never infers that a recording ended: it keeps following until the caller, who may -//! know finality out of band, calls [`Reader::finish`]. -//! -//! A track whose `.info` declares no timescale replays untimed, along with its timeline: its frames -//! carry no timestamps, and nothing stands in for one. -//! -//! ```no_run -//! # async fn example(store: moq_archive::Store) -> moq_archive::Result<()> { -//! use moq_archive::reader::{Config, Reader}; -//! -//! let broadcast = moq_net::broadcast::Info::new().produce(); -//! let timelines = [("video".to_string(), "video.timeline.z".to_string())].into(); -//! let mut reader = Reader::open(store, &broadcast, Config::new(timelines)).await?; -//! let serve = reader.serve(); -//! // Spawn `serve` on any executor, then call `reader.refresh()` to follow a growing archive. -//! # drop(serve); -//! # Ok(()) -//! # } -//! ``` - -mod index; - -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::future::Future; -use std::sync::{Arc, Mutex}; -use std::task::Poll; - -use futures::TryStreamExt; -use futures::future::BoxFuture; -use hang::timeline::{Position, Record}; -use moq_json::window; -use moq_net::{Timescale, Timestamp, broadcast, track}; -use object_store::ObjectStore; -use tokio::sync::watch; - -use self::index::{Index, Span}; -use crate::store::list::Query; -use crate::{Error, Key, Object, Result, Store}; - -/// Configuration for [`Reader::open`]. -#[derive(Debug, Clone)] -#[non_exhaustive] -pub struct Config { - /// Each served track's timeline track, from the catalog's `archive` entry. - pub timelines: BTreeMap, - /// Upper bound on cached object bytes shared by every track. - pub cache: u64, -} - -impl Config { - /// Serve the tracks `timelines` indexes with a 64 MiB object cache. - pub fn new(timelines: BTreeMap) -> Self { - Self { - timelines, - cache: 64 * 1024 * 1024, - } - } - - /// Set [`cache`](Self::cache). - pub fn with_cache(mut self, bytes: u64) -> Self { - self.cache = bytes; - self - } -} - -/// Serves an archive through a [`broadcast::Producer`] and follows its timelines. -pub struct Reader { - shared: Arc>, - broadcast: broadcast::Producer, - dynamic: broadcast::Dynamic, - timelines: Vec, -} - -/// One track's timeline, replayed onto its own timeline track. -struct Timeline { - /// The track it indexes. - track: String, - producer: track::Producer, - /// `None` when the timeline, like the track it indexes, is untimed. - timescale: Option, - decoder: window::Decoder, - /// The last timeline segment replayed. - cursor: Option, -} - -struct TimelineRequests { - name: String, - timescale: Option, - dynamic: track::Dynamic, -} - -struct Shared { - store: Store, - index: Mutex, - /// Stored object sequence for every validated timeline group. - timelines: Mutex>>, - /// New timeline handlers, including tracks enrolled after serve starts. - requests: kio::Queue, - cache: quick_cache::sync::Cache, Weight>, - /// Each track's `.info`, which is immutable, so a re-requested track costs no GET. - infos: Mutex>, - /// The tracks a timeline indexes, which are the only ones served. - served: Mutex>, - /// Notified whenever the index changes; holds whether the caller declared the recording ended. - changed: watch::Sender, -} - -impl Reader { - /// Create every timeline track on `broadcast`, then replay each track's retained timeline. - /// - /// Fails if a timeline's `.info` is missing or invalid, a timeline track already exists, or the - /// store cannot list a timeline. - pub async fn open(store: Store, broadcast: &broadcast::Producer, config: Config) -> Result { - let items = usize::try_from(config.cache / (1024 * 1024)) - .unwrap_or(usize::MAX) - .max(16); - let shared = Arc::new(Shared { - store, - index: Mutex::default(), - timelines: Mutex::default(), - requests: kio::Queue::new(), - cache: quick_cache::sync::Cache::with_weighter(items, config.cache, Weight), - infos: Mutex::default(), - served: Mutex::default(), - changed: watch::Sender::new(false), - }); - - let mut reader = Self { - shared, - broadcast: broadcast.clone(), - dynamic: broadcast.dynamic(), - timelines: Vec::new(), - }; - for (track, timeline) in config.timelines { - reader.add(track, timeline).await?; - } - reader.refresh().await?; - Ok(reader) - } - - /// Serve and follow `track`, whose timeline is the track `timeline`, such as one a recording - /// enrolled after this reader opened. Its timeline replays on the next [`refresh`](Self::refresh). - /// - /// Fails if the timeline's `.info` is missing or invalid, or `track` is already served. - pub async fn track(&mut self, track: &str, timeline: &str) -> Result<()> { - self.add(track.to_string(), timeline.to_string()).await - } - - async fn add(&mut self, track: String, timeline: String) -> Result<()> { - if self.shared.served.lock().unwrap().contains(&track) { - return Err(moq_net::Error::Duplicate.into()); - } - let info = track_info(&self.shared.store.get_info(&timeline).await?)?; - let timescale = info.timescale; - let producer = self.broadcast.create_track(timeline.clone(), info)?; - self.shared - .requests - .try_push(TimelineRequests { - name: timeline, - timescale, - dynamic: producer.dynamic(), - }) - .expect("the unbounded handler queue stays open while the reader lives"); - self.shared.served.lock().unwrap().insert(track.clone()); - self.timelines.push(Timeline { - track, - producer, - timescale, - decoder: window::Decoder::new(window::ConsumerConfig::default().with_compression(true)), - cursor: None, - }); - Ok(()) - } - - /// Replay timeline objects committed since the last refresh. - /// - /// Lists each timeline's keys after its last replayed segment, then GETs them in segment order. - /// Each stored group opens with a window checkpoint, so a segment that expired or is malformed - /// is skipped and the next one recovers the retained window, evicting whatever it popped. A - /// malformed newest object is retried on the next refresh. - pub async fn refresh(&mut self) -> Result<()> { - for timeline in &mut self.timelines { - timeline.refresh(&self.shared).await?; - } - Ok(()) - } - - /// Finish every replayed timeline track once the caller knows the recording ended. - /// - /// Following stops, and a group still growing is served as it stands. Groups already committed - /// stay servable through [`serve`](Self::serve). - pub fn finish(self) -> Result<()> { - self.shared.changed.send_replace(true); - for timeline in self.timelines { - timeline.producer.finish()?; - } - Ok(()) - } - - /// Answer track and group requests on the broadcast until it closes. - /// - /// The future is independent of this reader, so it keeps serving after - /// [`finish`](Self::finish); run it on any executor. - pub fn serve(&self) -> impl Future + Send + 'static - where - T: 'static, - { - serve(self.shared.clone(), self.dynamic.clone()) - } -} - -impl Timeline { - async fn refresh(&mut self, shared: &Shared) -> Result<()> { - let name = self.producer.name().to_string(); - let mut query = Query::segments(&name)?; - if let Some(cursor) = self.cursor { - query = query.after(&Key::segments(&name, cursor)?)?; - } - - let entries: Vec<_> = shared.store.list(&query).try_collect().await?; - let mut segments: Vec = entries - .into_iter() - .filter_map(|entry| match entry.key { - Key::Segments { segment, .. } => Some(segment), - _ => None, - }) - .filter(|segment| self.cursor.is_none_or(|cursor| *segment > cursor)) - .collect(); - segments.sort_unstable(); - segments.dedup(); - - for segment in segments { - match shared - .store - .get_segments(&name, segment, self.timescale.is_some()) - .await - { - Ok(object) => self.replay(shared, segment, &object)?, - Err(Error::Store(err)) => return Err(Error::Store(err)), - Err(err) => { - tracing::warn!(timeline = name, segment, %err, "skipping unreadable timeline segment"); - continue; - } - } - self.cursor = Some(segment); - } - - Ok(()) - } - - /// Index one timeline object's groups, then republish each on the timeline track. - fn replay(&mut self, shared: &Shared, segment: u64, object: &Object) -> Result<()> { - let timed = self.timescale.is_some(); - for stored in &object.groups { - let mut group = self.decoder.group(); - let decoded = stored.frames.iter().try_for_each(|frame| group.decode(&frame.payload)); - - // Apply whatever decoded, even from a group that failed partway: the next group's - // checkpoint restates the window regardless. - let mut index = shared.index.lock().unwrap(); - while let Some(event) = group.next_event() { - match event { - window::Event::Push { value, .. } - if value.pts.is_some() != timed || value.duration.is_some() != timed => - { - tracing::warn!( - track = self.track, - sequence = value.sequence, - "ignoring a record whose timing contradicts its timeline" - ); - } - window::Event::Push { index: at, value } if at == value.sequence => index.push(&self.track, &value), - window::Event::Push { index: at, value } => { - tracing::warn!( - track = self.track, - at, - sequence = value.sequence, - "ignoring a misnumbered record" - ); - } - window::Event::Pop(range) | window::Event::Skip(range) => { - for span in index.pop(&self.track, range) { - if let Ok(key) = Key::segments(self.track.clone(), span.sequence) { - shared.cache.remove(&key); - } - } - } - _ => {} - } - } - drop(index); - shared.changed.send_modify(|_| {}); - - if let Err(err) = decoded { - tracing::warn!(group = stored.sequence, %err, "skipping malformed timeline group"); - continue; - } - - shared - .timelines - .lock() - .unwrap() - .entry(self.producer.name().to_string()) - .or_default() - .insert(stored.sequence, segment); - - let mut producer = match self.producer.create_group(stored.sequence.into()) { - Ok(producer) => producer, - Err(moq_net::Error::Duplicate) => continue, - Err(err) => return Err(err.into()), - }; - for frame in &stored.frames { - producer.write_frame(timestamp(frame.timestamp, self.timescale)?, frame.payload.clone())?; - } - producer.finish()?; - } - Ok(()) - } -} - -/// Serve requested tracks until the broadcast closes. -async fn serve(shared: Arc>, mut dynamic: broadcast::Dynamic) { - let mut serving = kio::Tasks::new(); - kio::wait(|waiter| { - while let Poll::Ready(Ok(requests)) = shared.requests.poll_pop(waiter) { - serving.push(task(Box::pin(serve_timeline(shared.clone(), requests)))); - } - loop { - match dynamic.poll_requested_track(waiter) { - Poll::Ready(Ok(request)) => { - let known = shared.served.lock().unwrap().contains(request.name()); - serving.push(task(Box::pin(serve_track(shared.clone(), request, known)))) - } - Poll::Ready(Err(_)) => return Poll::Ready(()), - Poll::Pending => break, - } - } - let _ = serving.poll(waiter); - Poll::Pending - }) - .await -} - -/// Serve a timeline's evicted groups from their original stored objects. -async fn serve_timeline(shared: Arc>, requests: TimelineRequests) { - let mut fills = kio::Tasks::new(); - kio::wait(|waiter| { - loop { - match requests.dynamic.poll_requested_fill(waiter) { - Poll::Ready(Ok(fill)) => { - let (shared, name, timescale) = (shared.clone(), requests.name.clone(), requests.timescale); - fills.push(task(Box::pin(async move { - let mut last = None; - while fill.is_used() - && let Some(sequence) = fill.next() - && last != Some(sequence) - { - last = Some(sequence); - serve_timeline_group(&shared, &name, timescale, &fill, sequence).await; - } - }))); - } - Poll::Ready(Err(_)) => return Poll::Ready(()), - Poll::Pending => break, - } - } - let _ = fills.poll(waiter); - Poll::Pending - }) - .await; -} - -/// Answer group `sequence`, which the archive failed to load. A store failure may pass, so -/// only the readers waiting now are refused; anything else means the archive can't serve it. -fn unavailable(fill: &track::Fill, sequence: u64, err: &Error) { - let groups = sequence..sequence + 1; - match err { - Error::Store(_) => fill.refuse_groups(groups, moq_net::Error::Stream(moq_net::StreamError::Internal)), - _ => fill.drop_groups(groups), - } -} - -async fn serve_timeline_group( - shared: &Shared, - name: &str, - timescale: Option, - fill: &track::Fill, - sequence: u64, -) { - let reject = || fill.drop_groups(sequence..sequence + 1); - let segment = shared - .timelines - .lock() - .unwrap() - .get(name) - .and_then(|groups| groups.get(&sequence)) - .copied(); - let Some(segment) = segment else { - reject(); - return; - }; - let key = match Key::segments(name, segment) { - Ok(key) => key, - Err(_) => { - reject(); - return; - } - }; - let object = tokio::select! { - biased; - // Every reader narrowed past it: the fill moves on to the groups it still owes. - _ = fill.unwanted(sequence) => return, - object = shared.cache.get_or_insert_async(&key, async { - shared - .store - .get_segments(name, segment, timescale.is_some()) - .await - .map(Arc::new) - }) => object, - }; - let object = match object { - Ok(object) => object, - Err(err) => { - tracing::warn!(track = name, sequence, %err, "archived timeline group is unavailable"); - unavailable(fill, sequence, &err); - return; - } - }; - let start = fill.frame_start(sequence); - let Some(stored) = object.groups.iter().find(|group| group.sequence == sequence) else { - reject(); - return; - }; - if object.frame_start != 0 { - reject(); - return; - } - if !fill.is_wanted(sequence) { - return; - } - let Ok(mut producer) = fill.accept(sequence, None) else { - return; - }; - if producer.start_at(start).is_err() { - return; - } - for frame in stored.frames.iter().skip(start as usize) { - let Ok(timestamp) = timestamp(frame.timestamp, timescale) else { - let _ = producer.abort(moq_net::Error::NotFound); - return; - }; - if producer.write_frame(timestamp, frame.payload.clone()).is_err() { - return; - } - } - let _ = producer.finish(); -} - -/// Accept a track a timeline indexes, then serve its fills until nobody uses it. -async fn serve_track(shared: Arc>, request: track::Request, known: bool) { - let name = request.name().to_string(); - if !known { - request.reject(moq_net::Error::NotFound); - return; - } - - let cached = shared.infos.lock().unwrap().get(&name).cloned(); - let info = match cached { - Some(info) => info, - None => match shared.store.get_info(&name).await.and_then(|info| track_info(&info)) { - Ok(info) => { - shared.infos.lock().unwrap().insert(name.clone(), info.clone()); - info - } - Err(err) => { - tracing::warn!(track = name, %err, "archived track has no usable .info"); - request.reject(moq_net::Error::NotFound); - return; - } - }, - }; - - let timescale = info.timescale; - let dynamic = request.dynamic(); - let mut demand = dynamic.demand(); - let _producer = request.accept(info); - let mut fills = kio::Tasks::new(); - - kio::wait(|waiter| { - loop { - match dynamic.poll_requested_fill(waiter) { - Poll::Ready(Ok(fill)) => { - let (shared, name) = (shared.clone(), name.clone()); - fills.push(task(Box::pin(async move { - // One group at a time: a fill's groups precede its readers' live edge. - let mut last = None; - while fill.is_used() - && let Some(sequence) = fill.next() - && last != Some(sequence) - { - last = Some(sequence); - serve_group(&shared, &name, timescale, &fill, sequence).await; - } - }))); - } - Poll::Ready(Err(_)) => return Poll::Ready(()), - Poll::Pending => break, - } - } - if fills.poll(waiter).is_pending() { - return Poll::Pending; - } - // Drain the changes so the waiter stays registered, then read where demand settled. - while let Poll::Ready(Ok(Some(_))) = demand.poll_wanted_changed(waiter) {} - match demand.wanted() { - None => Poll::Ready(()), - Some(_) => Poll::Pending, - } - }) - .await -} - -/// Answer one group of a fill from the records holding it, or drop it as never delivered. -/// -/// A group whose records stop short of its end keeps its producer open and grows as later records -/// commit, until one reaches the group's end or the caller finishes the reader. -async fn serve_group( - shared: &Shared, - track: &str, - timescale: Option, - fill: &track::Fill, - sequence: u64, -) { - let start = fill.frame_start(sequence); - let mut changed = shared.changed.subscribe(); - let mut spans = shared.index.lock().unwrap().group(track, sequence); - - // The earliest retained record must hold the requested frame. Nothing retained is gone; - // an expired head is refused for this request alone, since a later one may ask the tail. - match spans.first() { - None => { - fill.drop_groups(sequence..sequence + 1); - return; - } - Some(span) if span.start > Position::new(sequence, start) => { - fill.refuse_groups(sequence..sequence + 1, moq_net::Error::NotFound); - return; - } - Some(_) => {} - } - // Validate the first object before accepting, so an unreadable group is a plain miss. Keep it, - // so a cache too small to hold it costs no second GET. - let timed = timescale.is_some(); - let loaded = tokio::select! { - biased; - // Every reader narrowed past it: the fill moves on to the groups it still owes. - _ = fill.unwanted(sequence) => return, - loaded = object(shared, track, timed, &spans[0]) => loaded, - }; - let mut first = match loaded { - Ok(object) => Some((spans[0].sequence, object)), - Err(err) => { - tracing::warn!(track, sequence, %err, "archived group is unavailable"); - unavailable(fill, sequence, &err); - return; - } - }; - if !fill.is_wanted(sequence) { - return; - } - let Ok(mut producer) = fill.accept(sequence, None) else { - return; - }; - if producer.start_at(start).is_err() { - return; - } - - let mut next = start; - loop { - for span in &spans { - if span.end <= Position::new(sequence, next) { - continue; - } - let loaded = match first.take_if(|(loaded, _)| *loaded == span.sequence) { - Some((_, object)) => Ok(object), - None => object(shared, track, timed, span).await, - }; - let object = match loaded { - Ok(object) if span.start <= Position::new(sequence, next) => object, - Ok(_) => { - let _ = producer.abort(moq_net::Error::NotFound); - return; - } - Err(err) => { - tracing::warn!(track, sequence, %err, "archived group is unavailable"); - let _ = producer.abort(moq_net::Error::NotFound); - return; - } - }; - // The span was validated against its object, so the group is present. - let Some(stored) = object.groups.iter().find(|group| group.sequence == sequence) else { - continue; - }; - let first = match object.groups[0].sequence == sequence { - true => object.frame_start, - false => 0, - }; - for (index, frame) in (first..).zip(&stored.frames) { - if index < next { - continue; - } - let Ok(timestamp) = timestamp(frame.timestamp, timescale) else { - let _ = producer.abort(moq_net::Error::TimestampMismatch); - return; - }; - if let Err(err) = producer.write_frame(timestamp, frame.payload.clone()) { - let _ = producer.abort(err); - return; - } - next = index + 1; - } - } - - let complete = spans.last().is_some_and(|span| span.end.group > sequence); - if complete { - let _ = producer.finish(); - return; - } - // A change that landed while objects loaded is unseen, so the spans above predate it: - // reload before finishing or waiting, or its records would never be served. - if !changed.has_changed().unwrap_or(false) { - // The caller finished the recording, and these spans are its final index. - if *changed.borrow() { - let _ = producer.finish(); - return; - } - if changed.changed().await.is_err() { - let _ = producer.finish(); - return; - } - } - changed.mark_unchanged(); - spans = shared.index.lock().unwrap().group(track, sequence); - } -} - -/// A span's object, from the cache or one GET validated against the span. -async fn object(shared: &Shared, track: &str, timed: bool, span: &Span) -> Result> { - let key = Key::segments(track.to_string(), span.sequence)?; - shared - .cache - .get_or_insert_async(&key, async { - let object = shared.store.get_segments(track, span.sequence, timed).await?; - object.check_span(span.start, span.end)?; - Ok(Arc::new(object)) - }) - .await -} - -fn track_info(info: &crate::Info) -> Result { - let mut track = track::Info::default().with_priority(info.priority); - if let Some(timescale) = info.timescale { - track = track.with_timescale(Timescale::new(timescale).map_err(|_| Error::Timescale(timescale))?); - } - Ok(track) -} - -/// A stored timestamp on its track, which must be timed exactly when the frame is. -fn timestamp(value: Option, timescale: Option) -> Result> { - match (value, timescale) { - (Some(value), Some(timescale)) => Ok(Some(Timestamp::new(value, timescale).map_err(|_| Error::Id(value))?)), - (None, None) => Ok(None), - _ => Err(Error::Timing), - } -} - -/// Adapt a boxed future into a [`kio::Tasks`] task. -fn task(mut future: BoxFuture<'static, ()>) -> impl FnMut(&kio::Waiter) -> Poll<()> + Send { - move |waiter| waiter.poll_future(future.as_mut()) -} - -/// Weighs a cached object by its payload bytes plus a small per-frame overhead. -#[derive(Clone)] -struct Weight; - -impl quick_cache::Weighter> for Weight { - fn weight(&self, _key: &Key, object: &Arc) -> u64 { - let frames = object.groups.iter().flat_map(|group| &group.frames); - frames.map(|frame| frame.payload.len() as u64 + 32).sum::() + 64 - } -} - -#[cfg(test)] -#[expect( - clippy::disallowed_methods, - clippy::disallowed_types, - reason = "tests read the host clock directly" -)] -mod tests; diff --git a/rs/moq-archive/src/reader/tests.rs b/rs/moq-archive/src/reader/tests.rs deleted file mode 100644 index 1f6d72ff79c..00000000000 --- a/rs/moq-archive/src/reader/tests.rs +++ /dev/null @@ -1,888 +0,0 @@ -use std::collections::HashMap; -use std::time::Duration; - -use bytes::Bytes; -use hang::timeline::{Position, Record}; -use moq_json::window; -use moq_net::{broadcast, group}; -use object_store::ObjectStoreExt; - -use super::*; -use crate::Info; -use crate::mock::{Mock, Op}; -use crate::segment::{Frame, Group}; - -fn timeline(track: &str) -> String { - hang::timeline::default_name(track) -} - -/// One track's timeline encoder, as a recording writer drives it. -struct Encoder { - encoder: window::Encoder, - /// Next timeline group sequence. - group: u64, - /// Next record sequence. - record: u64, -} - -/// Writes archive objects the way a recording writer lays them out. -struct Archive { - store: Store, - ops: bool, - encoders: HashMap, -} - -impl Archive { - async fn new() -> Self { - Self::with_ops(false).await - } - - /// Without ops every window edit is its own checkpoint group. - async fn with_ops(ops: bool) -> Self { - Self { - store: Store::new(Mock::memory(), "rec"), - ops, - encoders: HashMap::new(), - } - } - - async fn encoder(&mut self, track: &str) -> &mut Encoder { - if !self.encoders.contains_key(track) { - self.store - .put_info(track, &Info::new(1, Some(1000)).unwrap()) - .await - .unwrap(); - self.store - .put_info(&timeline(track), &Info::new(0, Some(1000)).unwrap()) - .await - .unwrap(); - let config = window::ProducerConfig::default() - .with_compression(true) - .with_ops(self.ops); - self.encoders.insert( - track.to_string(), - Encoder { - encoder: window::Encoder::new(config), - group: 0, - record: 0, - }, - ); - } - self.encoders.get_mut(track).unwrap() - } - - /// The next record of `track`, spanning `start..end` where every group holds `frames` frames. - /// Stores its object unless `store` is false. - async fn record(&mut self, track: &str, start: Position, end: Position, frames: u64) -> Record { - let encoder = self.encoder(track).await; - let sequence = encoder.record; - encoder.record += 1; - let record = Record::new(sequence, start.group * 100, 100, start, end); - - let last = match end.frame { - 0 => end.group - 1, - _ => end.group, - }; - let groups = (start.group..=last) - .map(|group| { - let first = if group == start.group { start.frame } else { 0 }; - let stop = if group == end.group { end.frame } else { frames }; - Group { - sequence: group, - frames: (first..stop).map(|i| frame(track, group, i)).collect(), - } - }) - .collect(); - let object = Object { - frame_start: start.frame, - groups, - }; - self.store.put_segments(track, sequence, &object).await.unwrap(); - record - } - - /// Store and commit a record of whole groups `groups`, each holding `frames` frames. - async fn add(&mut self, track: &str, groups: std::ops::Range, frames: u64) -> Record { - let record = self - .record( - track, - Position::group(groups.start), - Position::group(groups.end), - frames, - ) - .await; - self.commit(track, &record, 0).await; - record - } - - /// Push `record`, pop `pop` records, and return the resulting timeline groups. - async fn edit(&mut self, track: &str, record: &Record, pop: u64) -> Vec { - let encoder = self.encoder(track).await; - let mut frames = Vec::new(); - let pending = encoder.encoder.push(record).unwrap(); - frames.push((pending.keyframe, pending.payload.clone())); - pending.commit(); - if let Some(pending) = encoder.encoder.pop(pop).unwrap() { - frames.push((pending.keyframe, pending.payload.clone())); - pending.commit(); - } - - let mut groups: Vec = Vec::new(); - for (keyframe, payload) in frames { - if keyframe { - encoder.group += 1; - } - if groups.last().is_none_or(|group| group.sequence != encoder.group - 1) { - groups.push(Group { - sequence: encoder.group - 1, - frames: Vec::new(), - }); - } - let timestamp = record.pts; - groups.last_mut().unwrap().frames.push(Frame { timestamp, payload }); - } - groups - } - - async fn timeline(&self, track: &str, segment: u64, groups: Vec) { - self.store - .put_segments(&timeline(track), segment, &Object::new(groups)) - .await - .unwrap(); - } - - /// Commit `record` as its track's timeline segment, popping `pop` older records. - async fn commit(&mut self, track: &str, record: &Record, pop: u64) { - let groups = self.edit(track, record, pop).await; - self.timeline(track, record.sequence, groups).await; - } - - async fn raw(&self, key: &Key, bytes: &'static [u8]) { - let path = self.store.path(key).unwrap(); - self.store - .inner() - .put(&path, Bytes::from_static(bytes).into()) - .await - .unwrap(); - } - - fn config(&self) -> Config { - Config::new( - self.encoders - .keys() - .map(|track| (track.clone(), timeline(track))) - .collect(), - ) - } -} - -fn frame(track: &str, sequence: u64, index: u64) -> Frame { - Frame { - timestamp: Some(sequence * 100 + index), - payload: Bytes::from(format!("{track}/{sequence}/{index}")), - } -} - -async fn open(archive: &Archive) -> (broadcast::Producer, Reader) { - open_with(archive, archive.config()).await -} - -async fn open_with(archive: &Archive, config: Config) -> (broadcast::Producer, Reader) { - let broadcast = broadcast::Info::new().produce(); - let reader = Reader::open(archive.store.clone(), &broadcast, config).await.unwrap(); - tokio::spawn(reader.serve()); - (broadcast, reader) -} - -/// FETCH one group, returning each frame's timestamp and payload. -async fn fetch( - broadcast: &broadcast::Producer, - track: &str, - sequence: u64, -) -> std::result::Result, moq_net::Error> { - let track = broadcast.consume().track(track)?; - let mut group = track.fetch_group(sequence, None).await?; - let mut frames = Vec::new(); - while let Some(frame) = group.read_frame().await? { - frames.push((frame.timestamp.unwrap().value(), frame.payload)); - } - Ok(frames) -} - -/// Whether reading the whole fetched group from past its `len` frames ends cleanly. -async fn fetch_past_end(broadcast: &broadcast::Producer, track: &str, sequence: u64, len: usize) -> bool { - let track = broadcast.consume().track(track).unwrap(); - let group = track.fetch_group(sequence, None).await.unwrap(); - let mut past = group.with_range(len as u64 + 1..); - past.read_frame().await.unwrap().is_none() -} - -fn expected(track: &str, sequence: u64, frames: std::ops::Range) -> Vec<(u64, Bytes)> { - frames - .map(|i| { - let frame = frame(track, sequence, i); - (frame.timestamp.unwrap(), frame.payload) - }) - .collect() -} - -fn not_found(result: std::result::Result, moq_net::Error>) { - assert!(matches!(result, Err(moq_net::Error::NotFound)), "{result:?}"); -} - -#[tokio::test] -async fn fetch_replays_original_groups() { - let mut archive = Archive::new().await; - archive.add("video", 0..2, 3).await; - archive.add("video", 2..3, 2).await; - archive.add("audio", 0..2, 1).await; - archive.add("audio", 5..6, 2).await; - - let (broadcast, _reader) = open(&archive).await; - - assert_eq!(fetch(&broadcast, "video", 0).await.unwrap(), expected("video", 0, 0..3)); - assert_eq!(fetch(&broadcast, "video", 2).await.unwrap(), expected("video", 2, 0..2)); - assert_eq!(fetch(&broadcast, "audio", 5).await.unwrap(), expected("audio", 5, 0..2)); - - let track = broadcast.consume().track("video").unwrap(); - let info = track.query().await.unwrap(); - assert_eq!(info.priority, 1); - assert_eq!(info.timescale.unwrap().as_u64(), 1000); -} - -#[tokio::test] -async fn a_group_split_across_records_is_served_whole() { - let mut archive = Archive::new().await; - let head = archive - .record("video", Position::group(7), Position::new(7, 3), 5) - .await; - archive.commit("video", &head, 0).await; - let tail = archive - .record("video", Position::new(7, 3), Position::group(9), 5) - .await; - archive.commit("video", &tail, 0).await; - - let (broadcast, _reader) = open(&archive).await; - assert_eq!(fetch(&broadcast, "video", 7).await.unwrap(), expected("video", 7, 0..5)); - assert_eq!(fetch(&broadcast, "video", 8).await.unwrap(), expected("video", 8, 0..5)); -} - -#[tokio::test] -async fn an_open_group_grows_as_records_commit() { - let mut archive = Archive::new().await; - let head = archive.record("log", Position::group(0), Position::new(0, 2), 0).await; - archive.commit("log", &head, 0).await; - - let (broadcast, mut reader) = open(&archive).await; - let track = broadcast.consume().track("log").unwrap(); - let mut group = tokio::time::timeout(Duration::from_secs(1), track.fetch_group(0, group::Fetch::default())) - .await - .expect("fetch") - .unwrap(); - for index in 0..2 { - let frame = group.read_frame().await.unwrap().unwrap(); - assert_eq!(frame.payload, frame_payload("log", 0, index)); - } - let pending = tokio::time::timeout(Duration::from_millis(50), group.read_frame()).await; - assert!(pending.is_err(), "the group waits for its next record"); - - let more = archive.record("log", Position::new(0, 2), Position::new(0, 3), 0).await; - archive.commit("log", &more, 0).await; - reader.refresh().await.unwrap(); - let frame = group.read_frame().await.unwrap().unwrap(); - assert_eq!(frame.payload, frame_payload("log", 0, 2)); - - // Finishing the reader ends the group where the recording stopped. - reader.finish().unwrap(); - assert!(group.read_frame().await.unwrap().is_none()); -} - -/// A refresh that lands while a growing group loads an object is still served, not swallowed -/// by the wait for the next one. -#[tokio::test(start_paused = true)] -async fn a_refresh_during_an_object_load_is_served() { - let mut archive = Archive::new().await; - let head = archive.record("log", Position::group(0), Position::new(0, 2), 0).await; - archive.commit("log", &head, 0).await; - - let (broadcast, mut reader) = open(&archive).await; - let mock = archive.store.inner().clone(); - mock.take(); - let gate = mock.stall_gets("log/segments/"); - let track = broadcast.consume().track("log").unwrap(); - let fetch = tokio::spawn(async move { track.fetch_group(0, group::Fetch::default()).await }); - - // Wait until the head object's GET is in flight, then commit the tail behind it. - while !mock.gets().iter().any(|path| path.contains("log/segments/")) { - tokio::task::yield_now().await; - } - let more = archive.record("log", Position::new(0, 2), Position::new(0, 3), 0).await; - archive.commit("log", &more, 0).await; - reader.refresh().await.unwrap(); - gate.add_permits(16); - - let mut group = fetch.await.unwrap().unwrap(); - for index in 0..3 { - let frame = tokio::time::timeout(Duration::from_secs(1), group.read_frame()) - .await - .expect("the refreshed tail is served") - .unwrap() - .unwrap(); - assert_eq!(frame.payload, frame_payload("log", 0, index)); - } -} - -fn frame_payload(track: &str, sequence: u64, index: u64) -> Bytes { - frame(track, sequence, index).payload -} - -#[tokio::test] -async fn an_expired_head_is_not_found() { - let mut archive = Archive::new().await; - let head = archive - .record("video", Position::group(7), Position::new(7, 3), 5) - .await; - archive.commit("video", &head, 0).await; - let tail = archive - .record("video", Position::new(7, 3), Position::group(8), 5) - .await; - archive.commit("video", &tail, 1).await; - - let (broadcast, _reader) = open(&archive).await; - not_found(fetch(&broadcast, "video", 7).await); -} - -#[tokio::test] -async fn requests_download_only_their_object() { - let mut archive = Archive::new().await; - archive.add("video", 0..1, 1).await; - archive.add("audio", 0..3, 1).await; - - let (broadcast, _reader) = open(&archive).await; - archive.store.inner().take(); - - for sequence in 0..3 { - assert_eq!( - fetch(&broadcast, "audio", sequence).await.unwrap(), - expected("audio", sequence, 0..1) - ); - } - - let gets = archive.store.inner().gets(); - assert_eq!( - gets, - vec![ - "rec/audio/.info".to_string(), - "rec/audio/segments/0000000000000000000".to_string(), - ], - "adjacent groups reuse one GET and never touch video" - ); -} - -#[tokio::test] -async fn unadvertised_or_unreadable_groups_are_not_found() { - let mut archive = Archive::new().await; - archive.add("audio", 0..2, 1).await; - archive.add("audio", 4..5, 1).await; - // The object holds groups 0 and 1, but its record names group 2 too. - let mismatched = archive.record("video", Position::group(0), Position::group(2), 1).await; - archive - .commit( - "video", - &Record::new(0, 0, 100, Position::group(0), Position::group(3)), - 0, - ) - .await; - let bad = archive.record("bad", Position::group(0), Position::group(1), 1).await; - archive.raw(&Key::segments("bad", 0).unwrap(), b"\x02garbage").await; - archive.commit("bad", &bad, 0).await; - let missing = Record::new(0, 0, 100, Position::group(0), Position::group(1)); - archive.encoder("missing").await; - archive.commit("missing", &missing, 0).await; - assert_eq!(mismatched.end, Position::group(2)); - - let (broadcast, _reader) = open(&archive).await; - - not_found(fetch(&broadcast, "audio", 2).await); // between records - not_found(fetch(&broadcast, "audio", 5).await); // past the last record - not_found(fetch(&broadcast, "video", 0).await); // table disagrees with the record - not_found(fetch(&broadcast, "bad", 0).await); // malformed envelope - not_found(fetch(&broadcast, "missing", 0).await); // no object - not_found(fetch(&broadcast, "chat", 0).await); // a track no timeline indexes - - // Siblings remain usable. - assert_eq!(fetch(&broadcast, "audio", 4).await.unwrap(), expected("audio", 4, 0..1)); -} - -#[tokio::test] -async fn refresh_follows_new_records_and_pops() { - let mut archive = Archive::new().await; - archive.add("video", 0..2, 1).await; - - let (broadcast, mut reader) = open(&archive).await; - assert!(fetch(&broadcast, "video", 0).await.is_ok()); - - let next = archive.record("video", Position::group(2), Position::group(3), 1).await; - // Refreshing before the commit finds nothing new; Not Found is not finality. - reader.refresh().await.unwrap(); - not_found(fetch(&broadcast, "video", 2).await); - - archive.commit("video", &next, 1).await; - reader.refresh().await.unwrap(); - assert_eq!(fetch(&broadcast, "video", 2).await.unwrap(), expected("video", 2, 0..1)); - - // The popped record's groups are gone, although its object is still stored and was cached. - archive.store.inner().take(); - not_found(fetch(&broadcast, "video", 1).await); - assert_eq!(archive.store.inner().gets(), Vec::::new()); -} - -/// A track the recording enrolled after the reader opened is found by listing the store again, -/// then served and followed like the rest. -#[tokio::test] -async fn a_track_enrolled_after_opening_is_followed() { - let mut archive = Archive::new().await; - archive.add("audio", 0..2, 1).await; - - let (broadcast, mut reader) = open(&archive).await; - not_found(fetch(&broadcast, "video", 0).await); - - archive.add("video", 0..2, 2).await; - let timelines = archive.store.timelines().await.unwrap(); - assert_eq!(timelines.keys().collect::>(), ["audio", "video"]); - reader.track("video", &timelines["video"]).await.unwrap(); - assert!( - reader.track("video", &timelines["video"]).await.is_err(), - "a served track is refused" - ); - reader.refresh().await.unwrap(); - - assert_eq!(fetch(&broadcast, "video", 1).await.unwrap(), expected("video", 1, 0..2)); - let timeline = broadcast.consume().track(&timeline("video")).unwrap(); - assert!(timeline.subscribe(None).await.is_ok(), "the late timeline replays"); -} - -/// A timeline directory without its `.info`, such as one interrupted mid-enrollment, isn't -/// advertised, so following the recording doesn't fail on it. -#[tokio::test] -async fn a_timeline_without_info_is_not_listed() { - let mut archive = Archive::new().await; - archive.add("audio", 0..2, 1).await; - archive - .raw(&Key::segments(timeline("video"), 0).unwrap(), b"partial") - .await; - - let timelines = archive.store.timelines().await.unwrap(); - assert_eq!(timelines.keys().collect::>(), ["audio"]); -} - -#[tokio::test] -async fn a_missing_timeline_segment_recovers_from_the_next_checkpoint() { - let mut archive = Archive::new().await; - archive.add("video", 0..1, 1).await; - // Record 1 is committed to the window but its timeline object is unreadable. - let lost = archive.record("video", Position::group(1), Position::group(2), 1).await; - archive.edit("video", &lost, 0).await; - archive - .raw(&Key::segments(timeline("video"), 1).unwrap(), b"\x02") - .await; - archive.add("video", 2..3, 1).await; - - let (broadcast, _reader) = open(&archive).await; - - // Record 2's checkpoint restates record 1, so its group is still served. - for sequence in 0..3 { - assert_eq!( - fetch(&broadcast, "video", sequence).await.unwrap(), - expected("video", sequence, 0..1) - ); - } -} - -#[tokio::test] -async fn a_missing_tail_is_retried_on_the_next_refresh() { - let mut archive = Archive::new().await; - for group in 0..3 { - archive.add("video", group..group + 1, 1).await; - } - // Listed, but not yet readable. - archive.store.inner().hide_gets("segments/0000000000000000002"); - - let (broadcast, mut reader) = open(&archive).await; - assert!(fetch(&broadcast, "video", 1).await.is_ok()); - not_found(fetch(&broadcast, "video", 2).await); - - archive.store.inner().heal(); - archive.store.inner().take(); - reader.refresh().await.unwrap(); - assert_eq!( - archive.store.inner().take(), - [ - Op::List { - prefix: "rec/video%2Etimeline%2Ez/segments".to_string(), - offset: Some("rec/video%2Etimeline%2Ez/segments/0000000000000000001".to_string()), - }, - Op::Get("rec/video%2Etimeline%2Ez/segments/0000000000000000002".to_string()), - ], - "the cursor stays before the missing tail" - ); - assert_eq!(fetch(&broadcast, "video", 2).await.unwrap(), expected("video", 2, 0..1)); -} - -#[tokio::test] -async fn following_lists_only_new_timeline_keys() { - let mut archive = Archive::new().await; - archive.add("video", 0..1, 1).await; - let (broadcast, mut reader) = open(&archive).await; - - for group in 1..4 { - // A media object stored ahead of its commit is invisible until the timeline names it. - let record = archive - .record("video", Position::group(group), Position::group(group + 1), 1) - .await; - reader.refresh().await.unwrap(); - not_found(fetch(&broadcast, "video", group).await); - - archive.commit("video", &record, 0).await; - archive.store.inner().take(); - reader.refresh().await.unwrap(); - let previous = format!("rec/video%2Etimeline%2Ez/segments/{:019}", group - 1); - assert_eq!( - archive.store.inner().take(), - [ - Op::List { - prefix: "rec/video%2Etimeline%2Ez/segments".to_string(), - offset: Some(previous), - }, - Op::Get(format!("rec/video%2Etimeline%2Ez/segments/{group:019}")), - ], - "following record {group} touches no media listing" - ); - assert_eq!( - fetch(&broadcast, "video", group).await.unwrap(), - expected("video", group, 0..1) - ); - } -} - -#[tokio::test] -async fn an_unordered_listing_replays_in_segment_order() { - let mut archive = Archive::new().await; - archive.store.inner().unordered(); - for group in 0..4 { - let record = archive - .record("video", Position::group(group), Position::group(group + 1), 1) - .await; - archive.commit("video", &record, u64::from(group >= 2)).await; - } - - let (broadcast, mut reader) = open(&archive).await; - - // The cursor is the newest segment, not the last one listed. - let record = archive.record("video", Position::group(4), Position::group(5), 1).await; - archive.commit("video", &record, 1).await; - archive.store.inner().take(); - reader.refresh().await.unwrap(); - assert_eq!( - archive.store.inner().gets(), - ["rec/video%2Etimeline%2Ez/segments/0000000000000000004"] - ); - - for group in 0..3 { - not_found(fetch(&broadcast, "video", group).await); - } - for group in 3..5 { - assert_eq!( - fetch(&broadcast, "video", group).await.unwrap(), - expected("video", group, 0..1) - ); - } -} - -#[tokio::test] -async fn a_track_without_usable_info_is_not_found() { - let mut archive = Archive::new().await; - archive.add("audio", 0..1, 1).await; - archive.add("bare", 0..1, 1).await; - archive.add("future", 0..1, 1).await; - let path = archive.store.path(&Key::info("bare").unwrap()).unwrap(); - archive.store.inner().delete(&path).await.unwrap(); - let path = archive.store.path(&Key::info("future").unwrap()).unwrap(); - archive.store.inner().delete(&path).await.unwrap(); - archive - .raw( - &Key::info("future").unwrap(), - br#"{"version":4,"priority":0,"timescale":1000}"#, - ) - .await; - - let (broadcast, _reader) = open(&archive).await; - for track in ["bare", "future"] { - not_found(fetch(&broadcast, track, 0).await); - } - assert_eq!(fetch(&broadcast, "audio", 0).await.unwrap(), expected("audio", 0, 0..1)); -} - -#[tokio::test] -async fn multi_frame_timeline_groups_decode() { - let mut archive = Archive::with_ops(true).await; - let first = archive.record("video", Position::group(0), Position::group(1), 1).await; - let second = archive.record("video", Position::group(1), Position::group(2), 1).await; - let mut groups = archive.edit("video", &first, 0).await; - let ops = archive.edit("video", &second, 0).await; - assert!( - ops[0].sequence == groups[0].sequence, - "the push stays in the open group" - ); - groups[0].frames.extend(ops.into_iter().flat_map(|group| group.frames)); - archive.timeline("video", 0, groups).await; - - let (broadcast, _reader) = open(&archive).await; - assert!(fetch(&broadcast, "video", 1).await.is_ok()); -} - -#[tokio::test] -async fn timeline_tracks_are_republished_and_finished_on_request() { - let mut archive = Archive::new().await; - archive.add("video", 0..1, 1).await; - archive.add("video", 1..2, 1).await; - archive.add("audio", 0..1, 1).await; - - let (broadcast, reader) = open(&archive).await; - let track = broadcast.consume().track(&timeline("video")).unwrap(); - let subscriber = track.subscribe(None).await.unwrap(); - let mut timeline = - window::Consumer::::new(subscriber, window::ConsumerConfig::default().with_compression(true)); - - reader.finish().unwrap(); - - let mut records = Vec::new(); - while let Some(event) = timeline.next().await.unwrap() { - if let window::Event::Push { value, .. } = event.value { - records.push(value.sequence); - } - } - assert_eq!(records, vec![0, 1]); -} - -/// A record whose timing contradicts its timeline is not indexed, though its object is stored. -#[tokio::test] -async fn a_record_contradicting_its_timeline_is_not_indexed() { - let mut archive = Archive::new().await; - archive.add("video", 0..1, 1).await; - let stored = archive.record("video", Position::group(1), Position::group(2), 1).await; - let untimed = Record::untimed(stored.sequence, stored.start, stored.end); - let mut groups = archive.edit("video", &untimed, 0).await; - // The timeline envelope stays timed, as its `.info` declares. - for frame in groups.iter_mut().flat_map(|group| &mut group.frames) { - frame.timestamp = Some(100); - } - archive.timeline("video", untimed.sequence, groups).await; - - let (broadcast, _reader) = open(&archive).await; - assert_eq!(fetch(&broadcast, "video", 0).await.unwrap(), expected("video", 0, 0..1)); - not_found(fetch(&broadcast, "video", 1).await); -} - -#[tokio::test] -async fn open_requires_every_timeline_info() { - let store = Store::new(Mock::memory(), "rec"); - let broadcast = broadcast::Info::new().produce(); - let config = Config::new([("video".to_string(), timeline("video"))].into()); - let result = Reader::open(store, &broadcast, config).await; - assert!(matches!(result, Err(Error::NotFound(_)))); -} - -#[tokio::test(start_paused = true)] -async fn evicted_timeline_groups_remain_fetchable() { - let mut archive = Archive::new().await; - archive.add("video", 0..1, 1).await; - archive.add("video", 1..2, 1).await; - let mut info = broadcast::Info::new(); - info.pool = moq_net::cache::Pool::new(moq_net::cache::Config::default().with_expiry(Duration::from_millis(100))); - let pool = info.pool.clone(); - let broadcast = info.produce(); - let reader = Reader::open(archive.store.clone(), &broadcast, archive.config()) - .await - .unwrap(); - let serve = tokio::spawn(reader.serve()); - let expected = fetch(&broadcast, &timeline("video"), 0).await.unwrap(); - let now = moq_net::time::Instant::now(); - pool.poll(now, &kio::Waiter::noop()); - pool.poll(now + Duration::from_secs(1), &kio::Waiter::noop()); - assert_eq!(fetch(&broadcast, &timeline("video"), 0).await.unwrap(), expected); - // Past the end of an evicted complete group is EOF, as it is for a cached one. - pool.poll(now + Duration::from_secs(2), &kio::Waiter::noop()); - pool.poll(now + Duration::from_secs(3), &kio::Waiter::noop()); - assert!(fetch_past_end(&broadcast, &timeline("video"), 0, expected.len()).await); - serve.abort(); - let _ = serve.await; -} - -/// A timeline group every reader narrowed past stops its GET, so the groups the fill still -/// owes are read rather than waiting behind it. -#[tokio::test(start_paused = true)] -async fn a_narrowed_timeline_read_skips_the_group_it_left() { - let mut archive = Archive::new().await; - for group in 0..3 { - archive.add("video", group..group + 1, 1).await; - } - let mut info = broadcast::Info::new(); - info.pool = moq_net::cache::Pool::new(moq_net::cache::Config::default().with_expiry(Duration::from_millis(100))); - let pool = info.pool.clone(); - let broadcast = info.produce(); - let reader = Reader::open(archive.store.clone(), &broadcast, archive.config()) - .await - .unwrap(); - let serve = tokio::spawn(reader.serve()); - let now = moq_net::time::Instant::now(); - pool.poll(now, &kio::Waiter::noop()); - pool.poll(now + Duration::from_secs(1), &kio::Waiter::noop()); - - let mock = archive.store.inner().clone(); - mock.take(); - let gate = mock.stall_gets("video%2Etimeline%2Ez/segments/"); - let ascending = |groups| { - moq_net::track::Subscription::default() - .with_max_delay(None) - .with_range(groups) - .with_order(moq_net::track::Order::Ascending) - }; - let mut subscriber = broadcast - .consume() - .track(&timeline("video")) - .unwrap() - .subscribe(ascending(0..)) - .await - .unwrap(); - let waiter = kio::Waiter::noop(); - let mut gets = Vec::new(); - let _ = subscriber.poll_recv_group(&waiter); - assert!( - segment_read(&mock, &mut gets, 0).await, - "group 0 is read from its segment" - ); - - subscriber.update(ascending(1..)).unwrap(); - let _ = subscriber.poll_recv_group(&waiter); - assert!( - segment_read(&mock, &mut gets, 1).await, - "group 1 waited behind group 0, which nobody reads" - ); - gate.add_permits(16); - serve.abort(); - let _ = serve.await; -} - -/// Whether the video timeline's `segment` is requested once the runtime settles, collecting -/// GETs. Bounded by yields rather than time, so the paused clock stays put. -async fn segment_read(mock: &Mock, gets: &mut Vec, segment: u64) -> bool { - let path = format!("rec/video%2Etimeline%2Ez/segments/{segment:019}"); - for _ in 0..1000 { - gets.extend(mock.gets()); - if gets.contains(&path) { - return true; - } - tokio::task::yield_now().await; - } - false -} - -#[tokio::test(start_paused = true)] -async fn enrolled_timeline_fetch_survives_finish() { - let mut archive = Archive::new().await; - archive.add("video", 0..1, 1).await; - let mut info = broadcast::Info::new(); - info.pool = moq_net::cache::Pool::new(moq_net::cache::Config::default().with_expiry(Duration::from_secs(30))); - let broadcast = info.produce(); - let mut reader = Reader::open(archive.store.clone(), &broadcast, archive.config()) - .await - .unwrap(); - let serving = tokio::spawn(reader.serve()); - archive.add("audio", 0..1, 1).await; - archive.add("audio", 1..2, 1).await; - reader.track("audio", &timeline("audio")).await.unwrap(); - reader.refresh().await.unwrap(); - let expected = fetch(&broadcast, &timeline("audio"), 0).await.unwrap(); - reader.finish().unwrap(); - let now = moq_net::time::Instant::now(); - broadcast.info().pool.poll(now, &kio::Waiter::noop()); - broadcast - .info() - .pool - .poll(now + Duration::from_secs(31), &kio::Waiter::noop()); - assert_eq!(fetch(&broadcast, &timeline("audio"), 0).await.unwrap(), expected); - let missing = fetch(&broadcast, &timeline("audio"), 2).await; - assert!( - matches!(missing, Err(moq_net::Error::NotFound)), - "unknown timeline group: {missing:?}" - ); - // Complete groups retain their known end, so a read past it is EOF. - assert!(fetch_past_end(&broadcast, &timeline("audio"), 0, expected.len()).await); - serving.abort(); - let _ = serving.await; -} - -#[tokio::test(start_paused = true)] -#[ignore = "run explicitly to measure timeline FETCH fanout"] -async fn benchmark_timeline_fetch_fanout() { - for tracks in [1, 16, 128] { - for readers in [1, 16, 128] { - let mut archive = Archive::new().await; - for index in 0..tracks { - let name = format!("video-{index}"); - archive.add(&name, 0..1, 1).await; - archive.add(&name, 1..2, 1).await; - } - let mut info = broadcast::Info::new(); - info.pool = - moq_net::cache::Pool::new(moq_net::cache::Config::default().with_expiry(Duration::from_secs(30))); - let broadcast = info.produce(); - let reader = Reader::open(archive.store.clone(), &broadcast, archive.config()) - .await - .unwrap(); - let serving = tokio::spawn(reader.serve()); - let name = timeline("video-0"); - let mut now = moq_net::time::Instant::now(); - broadcast.info().pool.poll(now, &kio::Waiter::noop()); - let mut elapsed = Duration::ZERO; - for _ in 0..20 { - // One sweep dates recent reads; the next expires them. - for _ in 0..2 { - now += Duration::from_secs(31); - broadcast.info().pool.poll(now, &kio::Waiter::noop()); - } - let subscription = moq_net::track::Subscription::default() - .with_range(0..) - .with_max_delay(Duration::from_secs(60)); - let mut retained = broadcast - .consume() - .track(&name) - .unwrap() - .subscribe(subscription) - .await - .unwrap() - .ordered(); - assert_eq!( - retained.next_group().await.unwrap().unwrap().sequence, - 1, - "benchmark must fetch an evicted group" - ); - let started = std::time::Instant::now(); - let results = futures::future::join_all((0..readers).map(|_| fetch(&broadcast, &name, 0))).await; - elapsed += started.elapsed(); - assert!( - results - .iter() - .all(|result| result.as_ref().is_ok_and(|frames| !frames.is_empty())) - ); - } - println!( - "timeline-fetch tracks={tracks} readers={readers} mean_us={:.1}", - elapsed.as_secs_f64() * 1_000_000.0 / 20.0 - ); - serving.abort(); - let _ = serving.await; - } - } -} diff --git a/rs/moq-archive/src/segment.rs b/rs/moq-archive/src/segment.rs deleted file mode 100644 index 0411392dd80..00000000000 --- a/rs/moq-archive/src/segment.rs +++ /dev/null @@ -1,648 +0,0 @@ -use bytes::{Buf, Bytes}; -#[cfg(test)] -use bytes::{BufMut, BytesMut}; -use hang::timeline::Position; -use moq_net::varint; - -use crate::path::check_id; -use crate::{Error, Result, VERSION, VERSION_MIN}; - -/// One group's stored frames in a segment object, in sequence order. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Group { - /// Group sequence number. - pub sequence: u64, - /// Frames in their original order within the group. - pub frames: Vec, -} - -/// One frame's timestamp and original payload. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Frame { - /// Absolute timestamp in the track's timescale, or `None` on an untimed track. - pub timestamp: Option, - /// Original frame payload, including any group-scoped compression. - pub payload: Bytes, -} - -/// A versioned group/frame table followed by concatenated payload bytes. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Object { - /// The index of the first group's first stored frame within that group; later groups start at - /// frame zero. - pub frame_start: u64, - /// Groups in strictly ascending sequence order. - pub groups: Vec, -} - -impl Object { - /// An object whose first group starts at frame zero. - #[cfg(test)] - pub fn new(groups: Vec) -> Self { - Self { frame_start: 0, groups } - } - - /// The position of the first stored frame. - pub fn start(&self) -> Result { - validate(&self.groups)?; - Ok(Position::new(self.groups[0].sequence, self.frame_start)) - } - - /// Require this object to hold exactly the frames `start..end` of a record, with no group - /// missing or empty. - pub fn check_span(&self, start: Position, end: Position) -> Result<()> { - if self.start()? != start { - return Err(Error::Span); - } - for (expected, group) in (start.group..).zip(&self.groups) { - if group.sequence != expected || group.frames.is_empty() { - return Err(Error::Span); - } - } - let last = self.groups.last().expect("validated as non-empty"); - let first = match self.groups.len() { - 1 => self.frame_start, - _ => 0, - }; - let stop = first.checked_add(last.frames.len() as u64).ok_or(Error::Overflow)?; - let exact = match end.frame { - 0 => end.group == last.sequence + 1, - frame => end.group == last.sequence && frame == stop, - }; - if !exact { - return Err(Error::Span); - } - Ok(()) - } - - /// Encode the binary envelope. The first sequence is absolute; later ones are `current - previous - 1`. - /// - /// Frame timestamps are stored only when the frames have them, so every frame must agree. - #[cfg(test)] - pub fn encode(&self) -> Result { - validate(&self.groups)?; - - let mut table = BytesMut::new(); - write_varint(&mut table, VERSION)?; - write_varint(&mut table, self.frame_start)?; - write_varint(&mut table, self.groups.len() as u64)?; - - let mut payload = BytesMut::new(); - let mut prev = None; - for group in &self.groups { - let delta = match prev { - None => group.sequence, - Some(previous) => group - .sequence - .checked_sub(previous) - .ok_or(Error::Overflow)? - .checked_sub(1) - .ok_or(Error::Overflow)?, - }; - prev = Some(group.sequence); - write_varint(&mut table, delta)?; - write_varint(&mut table, group.frames.len() as u64)?; - for frame in &group.frames { - let offset = u64::try_from(payload.len()).map_err(|_| Error::Overflow)?; - let length = u64::try_from(frame.payload.len()).map_err(|_| Error::Overflow)?; - if let Some(timestamp) = frame.timestamp { - write_varint(&mut table, timestamp)?; - } - write_varint(&mut table, offset)?; - write_varint(&mut table, length)?; - payload.extend_from_slice(&frame.payload); - } - } - - table.extend_from_slice(&payload); - Ok(table.freeze()) - } - - /// Decode a complete table before slicing any payload, and refuse an unknown version. - /// - /// Version 2 objects are identical to the current ones. - /// - /// `timed` is whether the track's `.info` declares a timescale, which says whether the table - /// stores a timestamp per frame. - pub fn decode(mut buf: impl Buf, timed: bool) -> Result { - let version = read_varint(&mut buf)?; - if !(VERSION_MIN..=VERSION).contains(&version) { - return Err(Error::Version(version)); - } - - let frame_start = read_varint(&mut buf)?; - let group_count = read_count(&mut buf, 2)?; - if group_count == 0 { - return Err(Error::Empty); - } - - struct Entry { - sequence: u64, - frames: Vec<(Option, u64, u64)>, - } - - let mut entries = Vec::new(); - let mut prev: Option = None; - for i in 0..group_count { - let delta = read_varint(&mut buf)?; - let sequence = if i == 0 { - check_id(delta)? - } else { - let previous = prev.unwrap(); - let sequence = previous - .checked_add(1) - .ok_or(Error::Overflow)? - .checked_add(delta) - .ok_or(Error::Overflow)?; - check_id(sequence)? - }; - if let Some(previous) = prev - && sequence <= previous - { - return Err(Error::Sequence); - } - prev = Some(sequence); - - let frame_count = read_count(&mut buf, if timed { 3 } else { 2 })?; - let mut frames = Vec::new(); - for _ in 0..frame_count { - let timestamp = match timed { - true => Some(check_id(read_varint(&mut buf)?)?), - false => None, - }; - let offset = read_varint(&mut buf)?; - let length = read_varint(&mut buf)?; - frames.push((timestamp, offset, length)); - } - entries.push(Entry { sequence, frames }); - } - - let payload = buf.copy_to_bytes(buf.remaining()); - let payload_len = u64::try_from(payload.len()).map_err(|_| Error::Overflow)?; - - let mut expected = 0u64; - let mut groups = Vec::with_capacity(entries.len()); - for entry in entries { - let mut frames = Vec::with_capacity(entry.frames.len()); - for (timestamp, offset, length) in entry.frames { - if offset != expected { - return Err(Error::Table); - } - let end = offset.checked_add(length).ok_or(Error::Overflow)?; - if end > payload_len { - return Err(Error::Table); - } - let start = usize::try_from(offset).map_err(|_| Error::Overflow)?; - let stop = usize::try_from(end).map_err(|_| Error::Overflow)?; - frames.push(Frame { - timestamp, - payload: payload.slice(start..stop), - }); - expected = end; - } - groups.push(Group { - sequence: entry.sequence, - frames, - }); - } - if expected != payload_len { - return Err(Error::Table); - } - - Ok(Self { frame_start, groups }) - } -} - -fn validate(groups: &[Group]) -> Result<()> { - if groups.is_empty() { - return Err(Error::Empty); - } - let mut prev = None; - let mut timed = None; - for group in groups { - check_id(group.sequence)?; - if let Some(previous) = prev - && group.sequence <= previous - { - return Err(Error::Sequence); - } - prev = Some(group.sequence); - for frame in &group.frames { - if *timed.get_or_insert(frame.timestamp.is_some()) != frame.timestamp.is_some() { - return Err(Error::Timing); - } - if let Some(timestamp) = frame.timestamp { - check_id(timestamp)?; - } - } - } - Ok(()) -} - -#[cfg(test)] -fn write_varint(buf: &mut impl BufMut, value: u64) -> Result<()> { - varint::encode_quic(value, buf).map_err(|_| Error::Overflow) -} - -fn read_varint(buf: &mut impl Buf) -> Result { - varint::decode_quic(buf).map_err(|_| Error::Table) -} - -fn read_count(buf: &mut impl Buf, min_entry: usize) -> Result { - let n = read_varint(buf)?; - let n = usize::try_from(n).map_err(|_| Error::Overflow)?; - if min_entry == 0 || n > buf.remaining() / min_entry { - return Err(Error::Table); - } - Ok(n) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ID_MAX; - - fn frame(timestamp: u64, payload: &'static [u8]) -> Frame { - Frame { - timestamp: Some(timestamp), - payload: Bytes::from_static(payload), - } - } - - fn untimed(payload: &'static [u8]) -> Frame { - Frame { - timestamp: None, - payload: Bytes::from_static(payload), - } - } - - fn object(groups: Vec) -> Object { - Object::new(groups) - } - - fn group(sequence: u64, frames: usize) -> Group { - Group { - sequence, - frames: (0..frames as u64).map(|i| frame(i, b"x")).collect(), - } - } - - #[test] - fn roundtrip_consecutive_and_sparse() { - let original = object(vec![ - Group { - sequence: 0, - frames: vec![frame(0, b"a"), frame(1, b"bb")], - }, - Group { - sequence: 1, - frames: vec![frame(2, b"ccc")], - }, - Group { - sequence: 4, - frames: vec![frame(ID_MAX, b"d")], - }, - ]); - let bytes = original.encode().unwrap(); - assert_eq!(Object::decode(&bytes[..], true).unwrap(), original); - assert_eq!(original.start().unwrap(), Position::group(0)); - - let split = Object { - frame_start: 300, - ..original - }; - assert_eq!(Object::decode(split.encode().unwrap(), true).unwrap(), split); - assert_eq!(split.start().unwrap(), Position::new(0, 300)); - } - - #[test] - fn an_untimed_object_stores_no_timestamps() { - let original = object(vec![ - Group { - sequence: 3, - frames: vec![untimed(b"a"), untimed(b"bb")], - }, - Group { - sequence: 4, - frames: vec![untimed(b"c")], - }, - ]); - let bytes = original.encode().unwrap(); - let mut buf = &bytes[..]; - for expected in [VERSION, 0, 2, 3, 2, 0, 1, 1, 2, 0, 1, 3, 1] { - assert_eq!(read_varint(&mut buf).unwrap(), expected); - } - assert_eq!(buf, b"abbc"); - assert_eq!(Object::decode(bytes.clone(), false).unwrap(), original); - assert!( - Object::decode(bytes, true).is_err(), - "a timed reader misreads the table" - ); - } - - #[test] - fn frames_agree_on_timing() { - let mixed = object(vec![ - Group { - sequence: 0, - frames: vec![frame(0, b"a")], - }, - Group { - sequence: 1, - frames: vec![untimed(b"b")], - }, - ]); - assert_eq!(mixed.encode(), Err(Error::Timing)); - } - - #[test] - fn first_id_is_absolute_then_minus_one_deltas() { - let bytes = object(vec![ - Group { - sequence: 5, - frames: vec![frame(0, b"x")], - }, - Group { - sequence: 6, - frames: vec![frame(1, b"y")], - }, - Group { - sequence: 10, - frames: vec![frame(2, b"z")], - }, - ]) - .encode() - .unwrap(); - - let mut buf = &bytes[..]; - assert_eq!(read_varint(&mut buf).unwrap(), VERSION); // version - assert_eq!(read_varint(&mut buf).unwrap(), 0); // frame start - assert_eq!(read_varint(&mut buf).unwrap(), 3); // group count - assert_eq!(read_varint(&mut buf).unwrap(), 5); // absolute - assert_eq!(read_varint(&mut buf).unwrap(), 1); // one frame - read_varint(&mut buf).unwrap(); // timestamp - read_varint(&mut buf).unwrap(); // offset - read_varint(&mut buf).unwrap(); // length - assert_eq!(read_varint(&mut buf).unwrap(), 0); // consecutive - assert_eq!(read_varint(&mut buf).unwrap(), 1); - read_varint(&mut buf).unwrap(); - read_varint(&mut buf).unwrap(); - read_varint(&mut buf).unwrap(); - assert_eq!(read_varint(&mut buf).unwrap(), 3); // 10 - 6 - 1 - } - - #[test] - fn empty_object_is_rejected() { - assert!(matches!(object(vec![]).encode(), Err(Error::Empty))); - let mut bytes = BytesMut::new(); - write_varint(&mut bytes, VERSION).unwrap(); - write_varint(&mut bytes, 0).unwrap(); - write_varint(&mut bytes, 0).unwrap(); - assert!(matches!(Object::decode(bytes.freeze(), true), Err(Error::Empty))); - } - - #[test] - fn sequences_must_ascend() { - assert!(matches!( - object(vec![ - Group { - sequence: 2, - frames: vec![frame(0, b"a")], - }, - Group { - sequence: 2, - frames: vec![frame(1, b"b")], - }, - ]) - .encode(), - Err(Error::Sequence) - )); - assert!(matches!( - object(vec![ - Group { - sequence: 2, - frames: vec![frame(0, b"a")], - }, - Group { - sequence: 1, - frames: vec![frame(1, b"b")], - }, - ]) - .encode(), - Err(Error::Sequence) - )); - } - - #[test] - fn reconstructed_ids_must_stay_in_range() { - let mut bytes = BytesMut::new(); - write_varint(&mut bytes, VERSION).unwrap(); - write_varint(&mut bytes, 0).unwrap(); - write_varint(&mut bytes, 2).unwrap(); - write_varint(&mut bytes, ID_MAX).unwrap(); - write_varint(&mut bytes, 0).unwrap(); // no frames - write_varint(&mut bytes, 0).unwrap(); // consecutive => ID_MAX + 1 - write_varint(&mut bytes, 0).unwrap(); - assert!(matches!(Object::decode(bytes.freeze(), true), Err(Error::Id(_)))); - } - - /// A table with one frameless group per delta. - fn deltas(deltas: &[u64]) -> Bytes { - let mut bytes = BytesMut::new(); - write_varint(&mut bytes, VERSION).unwrap(); - write_varint(&mut bytes, 0).unwrap(); - write_varint(&mut bytes, deltas.len() as u64).unwrap(); - for &delta in deltas { - write_varint(&mut bytes, delta).unwrap(); - write_varint(&mut bytes, 0).unwrap(); - } - bytes.freeze() - } - - fn sequences(bytes: Bytes) -> Result> { - Ok(Object::decode(bytes, true)? - .groups - .iter() - .map(|group| group.sequence) - .collect()) - } - - #[test] - fn delta_extremes() { - let varint = (1u64 << 62) - 1; - assert_eq!(sequences(deltas(&[0, 0, 0])).unwrap(), vec![0, 1, 2]); - assert_eq!(sequences(deltas(&[0, 9, 0])).unwrap(), vec![0, 10, 11]); - assert_eq!(sequences(deltas(&[0, ID_MAX - 1])).unwrap(), vec![0, ID_MAX]); - assert_eq!(sequences(deltas(&[ID_MAX])).unwrap(), vec![ID_MAX]); - assert!(matches!(sequences(deltas(&[0, ID_MAX])), Err(Error::Id(_)))); - assert!(matches!(sequences(deltas(&[ID_MAX + 1])), Err(Error::Id(_)))); - assert!(matches!(sequences(deltas(&[varint])), Err(Error::Id(_)))); - assert!(matches!(sequences(deltas(&[ID_MAX, varint])), Err(Error::Id(_)))); - } - - #[test] - fn timestamp_bounds() { - assert!( - object(vec![Group { - sequence: 0, - frames: vec![frame(ID_MAX, b"a")], - }]) - .encode() - .is_ok() - ); - assert!(matches!( - object(vec![Group { - sequence: 0, - frames: vec![frame(ID_MAX + 1, b"a")], - }]) - .encode(), - Err(Error::Id(_)) - )); - } - - #[test] - fn unknown_version_is_refused() { - // Version 1 objects held whole groups under range-named keys. - for version in [1, VERSION + 1] { - let mut bytes = BytesMut::new(); - write_varint(&mut bytes, version).unwrap(); - write_varint(&mut bytes, 1).unwrap(); - assert!(matches!(Object::decode(bytes.freeze(), true), Err(Error::Version(v)) if v == version)); - } - } - - #[test] - fn a_version_2_object_decodes_unchanged() { - let original = object(vec![Group { - sequence: 3, - frames: vec![untimed(b"a")], - }]); - let mut bytes = BytesMut::new(); - write_varint(&mut bytes, 2).unwrap(); - let encoded = original.encode().unwrap(); - let mut rest = &encoded[..]; - assert_eq!(read_varint(&mut rest).unwrap(), VERSION); - bytes.extend_from_slice(rest); - assert_eq!(Object::decode(bytes.freeze(), false).unwrap(), original); - } - - #[test] - fn payload_must_be_contiguous() { - let mut good = object(vec![Group { - sequence: 0, - frames: vec![frame(0, b"ab")], - }]) - .encode() - .unwrap() - .to_vec(); - good.push(b'x'); - assert!(matches!(Object::decode(Bytes::from(good), true), Err(Error::Table))); - } - - #[test] - fn advertised_counts_must_fit_minimum_entry_size() { - // group_count equals remaining bytes, so a remaining-bytes check would - // accept it, but each group needs at least two varints. - let mut bytes = BytesMut::new(); - write_varint(&mut bytes, VERSION).unwrap(); - write_varint(&mut bytes, 0).unwrap(); - write_varint(&mut bytes, 4).unwrap(); - bytes.extend_from_slice(&[0, 0, 0, 0]); - assert!(matches!(Object::decode(bytes.freeze(), true), Err(Error::Table))); - } - - #[test] - fn offset_overflow_and_truncated_table() { - let mut bytes = BytesMut::new(); - for value in [VERSION, 0, 1, 0, 1, 0, 0, 4] { - write_varint(&mut bytes, value).unwrap(); // length 4, no payload - } - assert!(matches!(Object::decode(bytes.freeze(), true), Err(Error::Table))); - - assert!(matches!(Object::decode(&b"\x02"[..], true), Err(Error::Table))); - } - - #[test] - fn frame_offsets_must_tile_the_payload() { - // One group of two frames at (offset, length), followed by `payload` bytes. - let table = |frames: [(u64, u64); 2], payload: usize| { - let mut bytes = BytesMut::new(); - for value in [VERSION, 0, 1, 0, 2] { - write_varint(&mut bytes, value).unwrap(); - } - for (offset, length) in frames { - for value in [0, offset, length] { - write_varint(&mut bytes, value).unwrap(); - } - } - bytes.extend_from_slice(&vec![b'x'; payload]); - Object::decode(bytes.freeze(), true) - }; - assert!(table([(0, 1), (1, 2)], 3).is_ok()); - assert!(matches!(table([(0, 1), (0, 2)], 3), Err(Error::Table)), "overlap"); - assert!(matches!(table([(0, 1), (2, 1)], 3), Err(Error::Table)), "gap"); - assert!(matches!(table([(1, 1), (2, 1)], 3), Err(Error::Table)), "late start"); - assert!(matches!(table([(0, 1), (1, 3)], 3), Err(Error::Table)), "past the end"); - let varint = (1u64 << 62) - 1; - assert!( - matches!(table([(0, 1), (1, varint)], 3), Err(Error::Table)), - "huge length" - ); - } - - #[test] - fn a_span_must_match_the_record() { - let whole = object(vec![group(5, 2), group(6, 1)]); - whole.check_span(Position::group(5), Position::group(7)).unwrap(); - assert_eq!( - whole.check_span(Position::group(5), Position::group(8)), - Err(Error::Span) - ); - assert_eq!( - whole.check_span(Position::group(4), Position::group(7)), - Err(Error::Span) - ); - assert_eq!( - whole.check_span(Position::group(5), Position::new(6, 2)), - Err(Error::Span), - "a partial end names its frame count" - ); - - // A frame split: frames 3..5 of group 7. - let split = Object { - frame_start: 3, - groups: vec![group(7, 2)], - }; - split.check_span(Position::new(7, 3), Position::new(7, 5)).unwrap(); - assert_eq!( - split.check_span(Position::new(7, 3), Position::new(7, 4)), - Err(Error::Span) - ); - - // A split's tail continues into whole groups. - let tail = Object { - frame_start: 3, - groups: vec![group(7, 2), group(8, 4)], - }; - tail.check_span(Position::new(7, 3), Position::new(8, 4)).unwrap(); - tail.check_span(Position::new(7, 3), Position::group(9)).unwrap(); - - // Gaps and empty groups never appear inside a record. - let gap = object(vec![group(5, 1), group(7, 1)]); - assert_eq!(gap.check_span(Position::group(5), Position::group(8)), Err(Error::Span)); - let empty = object(vec![group(5, 1), group(6, 0)]); - assert_eq!( - empty.check_span(Position::group(5), Position::group(7)), - Err(Error::Span) - ); - } - - #[test] - fn endpoints_roundtrip() { - let original = object(vec![Group { - sequence: ID_MAX, - frames: vec![frame(0, b"")], - }]); - assert_eq!(Object::decode(original.encode().unwrap(), true).unwrap(), original); - } -} diff --git a/rs/moq-archive/src/store.rs b/rs/moq-archive/src/store.rs index 281fe9aa223..8a9196ac40d 100644 --- a/rs/moq-archive/src/store.rs +++ b/rs/moq-archive/src/store.rs @@ -1,57 +1,12 @@ use bytes::Bytes; -use futures::StreamExt; -use futures::stream::BoxStream; use object_store::path::Path; -use object_store::{ObjectMeta, ObjectStore, ObjectStoreExt, PutMode, PutPayload}; +use object_store::{ObjectStore, ObjectStoreExt, PutMode, PutPayload}; use crate::info::Info; -use crate::path::Key; -use crate::segment::Object; +use crate::layout; use crate::{Error, Result}; -/// Recording object listing types, relative to the recording prefix supplied to [`Store::new`]. -pub(crate) mod list { - use object_store::path::Path; - - use crate::path; - use crate::{Key, Result}; - - /// A recording-relative object listing query. - #[derive(Debug, Clone, Default, PartialEq, Eq)] - pub struct Query { - pub(crate) prefix: Option, - pub(crate) offset: Option, - } - - impl Query { - /// List segment objects for one track. - pub fn segments(track: &str) -> Result { - Ok(Self { - prefix: Some(path::segments_prefix(&Path::ROOT, track)?), - ..Self::default() - }) - } - - /// Start strictly after this recording object. - pub fn after(mut self, key: &Key) -> Result { - self.offset = Some(key.path(&Path::ROOT)?); - Ok(self) - } - } - - /// One recording object returned by a listing. - #[derive(Debug, Clone, PartialEq, Eq)] - pub struct Entry { - /// Parsed recording key. - pub key: Key, - /// Object size in bytes. - pub size: u64, - } -} - -use list::{Entry, Query}; - -/// Versioned recording objects on a generic [`ObjectStore`]. +/// A broadcast's archive: an [`ObjectStore`] and the key prefix it is stored under. #[derive(Clone, Debug)] pub struct Store { inner: T, @@ -77,215 +32,67 @@ impl Store { &self.prefix } - /// Encode `key` under this store's prefix. - pub(crate) fn path(&self, key: &Key) -> Result { - key.path(&self.prefix) - } - - /// Create `.info`, or accept an existing object with the same parsed properties. + /// Create a track's `.info`, or accept an existing object with the same parsed properties. /// - /// An existing recording in an older format is refused with [`Error::Version`]: a reader - /// still reads it, but nothing appends to it. - pub(crate) async fn put_info(&self, track: &str, info: &Info) -> Result { - let key = Key::info(track)?; - let path = self.path(&key)?; + /// An existing recording in an older format is refused with [`Error::Version`]: a + /// [`Consumer`](crate::Consumer) still reads it, but nothing appends to it. + pub(crate) async fn put_info(&self, track: &str, info: &Info) -> Result<()> { + let path = self.info_path(track)?; let bytes = info.encode()?; - match self.create(&path, bytes).await? { - Create::Created => Ok(key), - Create::Exists => { - let existing = self.get_bytes(&path).await?; - let parsed = Info::decode(&existing)?; - if parsed.version != info.version { - return Err(Error::Version(parsed.version)); + match self + .inner + .put_opts(&path, PutPayload::from(bytes), PutMode::Create.into()) + .await + { + Ok(_) => Ok(()), + Err(object_store::Error::AlreadyExists { .. }) => { + let existing = Info::decode(&self.get_bytes(&path).await?)?; + if existing.version != info.version { + return Err(Error::Version(existing.version)); } - if parsed.priority != info.priority { + if existing.priority != info.priority { return Err(Error::Priority { - existing: parsed.priority, + existing: existing.priority, intended: info.priority, }); } - if parsed.timescale != info.timescale { + if existing.timescale != info.timescale { return Err(Error::TimescaleMismatch { - existing: parsed.timescale, + existing: existing.timescale, intended: info.timescale, }); } - Ok(key) + Ok(()) } + Err(err) => Err(err.into()), } } /// Fetch and validate a track's `.info`. - pub(crate) async fn get_info(&self, track: &str) -> Result { - let path = self.path(&Key::info(track)?)?; - Info::decode(&self.get_bytes(&path).await?) - } - - /// Create the object at `segments/`. A collision is accepted only when the bytes match. - #[cfg(test)] - pub(crate) async fn put_segments(&self, track: &str, segment: u64, object: &Object) -> Result { - let key = Key::segments(track, segment)?; - self.put_segment(&key, object.encode()?).await?; - Ok(key) - } - - /// Fetch and decode the object at `segments/`, whose frames store timestamps when - /// `timed`, as the track's `.info` says. - pub(crate) async fn get_segments(&self, track: &str, segment: u64, timed: bool) -> Result { - let path = self.path(&Key::segments(track, segment)?)?; - Object::decode(self.get_bytes(&path).await?, timed) - } - - /// Each recorded track's timeline track, found by listing the recording's track directories: - /// a timeline is named by [`hang::timeline::default_name`]. One listing of the - /// prefix's directories plus a HEAD per timeline, so following a long recording doesn't list - /// its objects. A timeline whose `.info` isn't stored yet, such as one still enrolling, is left - /// out until it is. - /// - /// For replaying a recording without its catalog; a catalog's `archive` entry names the same map. - pub async fn timelines(&self) -> Result> { - let listed = self.inner.list_with_delimiter(self.list_path(None).as_ref()).await?; - let mut timelines = std::collections::BTreeMap::new(); - for directory in listed.common_prefixes { - let encoded = directory - .filename() - .ok_or_else(|| Error::Directory(directory.to_string()))?; - let track = crate::path::decode_track(encoded)?; - let Some(indexed) = track.strip_suffix(hang::timeline::SUFFIX) else { - continue; - }; - match self.inner.head(&self.path(&Key::info(&track)?)?).await { - Ok(_) => timelines.insert(indexed.to_string(), track), - Err(object_store::Error::NotFound { .. }) => continue, - Err(err) => return Err(err.into()), - }; - } - Ok(timelines) - } - - /// Stream matching entries, in unspecified order. - pub(crate) fn list(&self, query: &Query) -> BoxStream<'static, Result> { - let prefix = self.list_path(query.prefix.as_ref()); - let offset = query.offset.as_ref().map(|offset| self.recording_path(offset)); - let store_prefix = self.prefix.clone(); - let stream = match offset.as_ref() { - Some(offset) => self.inner.list_with_offset(prefix.as_ref(), offset), - None => self.inner.list(prefix.as_ref()), - }; - stream.map(move |item| Entry::from_meta(&store_prefix, item?)).boxed() - } - #[cfg(test)] - async fn put_segment(&self, key: &Key, bytes: Bytes) -> Result<()> { - let path = self.path(key)?; - match self.create(&path, bytes.clone()).await? { - Create::Created => Ok(()), - Create::Exists => { - let existing = self.get_bytes(&path).await?; - if existing == bytes { - Ok(()) - } else { - Err(Error::Conflict(path.to_string())) - } - } - } + pub(crate) async fn get_info(&self, track: &str) -> Result { + Info::decode(&self.get_bytes(&self.info_path(track)?).await?) } - async fn create(&self, path: &Path, bytes: Bytes) -> Result { - match self - .inner - .put_opts(path, PutPayload::from(bytes), PutMode::Create.into()) - .await - { - Ok(_) => Ok(Create::Created), - Err(object_store::Error::AlreadyExists { .. }) => Ok(Create::Exists), - Err(err) => Err(err.into()), - } + fn info_path(&self, track: &str) -> Result { + crate::path::push(&layout::track_dir(&self.prefix, track)?, layout::TRACK_INFO) } async fn get_bytes(&self, path: &Path) -> Result { Ok(self.inner.get(path).await?.bytes().await?) } - - fn recording_path(&self, relative: &Path) -> Path { - let mut path = self.prefix.clone(); - path.extend(relative.parts()); - path - } - - fn list_path(&self, relative: Option<&Path>) -> Option { - let path = relative.map_or_else(|| self.prefix.clone(), |relative| self.recording_path(relative)); - (!path.as_ref().is_empty()).then_some(path) - } -} - -impl Entry { - fn from_meta(prefix: &Path, meta: ObjectMeta) -> Result { - let key = Key::parse(prefix, &meta.location)?; - Ok(Self { key, size: meta.size }) - } -} - -enum Create { - Created, - Exists, } #[cfg(test)] mod tests { - use std::collections::BTreeSet; - - use futures::TryStreamExt; - use object_store::ObjectStoreExt; use object_store::memory::InMemory; - use object_store::path::Path; use super::*; - use crate::ID_MAX; - use crate::mock::Mock; - use crate::path::encode_track; - use crate::segment::{Frame, Group}; fn memory() -> Store { Store::new(InMemory::new(), "rec") } - fn frame(timestamp: u64, payload: &'static [u8]) -> Frame { - Frame { - timestamp: Some(timestamp), - payload: Bytes::from_static(payload), - } - } - - fn one_group(sequence: u64, payload: &'static [u8]) -> Object { - Object::new(vec![Group { - sequence, - frames: vec![frame(sequence, payload)], - }]) - } - - fn two_groups() -> Object { - Object::new(vec![ - Group { - sequence: 5, - frames: vec![frame(0, b"a")], - }, - Group { - sequence: 6, - frames: vec![frame(1, b"bb")], - }, - ]) - } - - async fn names(store: &Store, query: &Query) -> BTreeSet { - store - .list(query) - .map_ok(|entry| entry.key.path(&Path::ROOT).unwrap().to_string()) - .try_collect() - .await - .unwrap() - } - #[tokio::test] async fn info_create_get_and_idempotent_retry() { let store = memory(); @@ -293,7 +100,7 @@ mod tests { store.put_info("catalog.json", &info).await.unwrap(); assert_eq!(store.get_info("catalog.json").await.unwrap(), info); - let path = store.path(&Key::info("catalog.json").unwrap()).unwrap(); + let path = Path::from("rec/catalog%2Ejson/.info"); store .inner() .put( @@ -330,7 +137,7 @@ mod tests { matches!(err, Error::Timescale(0)) }), ] { - let path = store.path(&Key::info(track).unwrap()).unwrap(); + let path = Path::from(format!("rec/{track}/.info")); store.inner().put(&path, existing.to_vec().into()).await.unwrap(); let err = store.put_info(track, &info).await.unwrap_err(); assert!(check(&err), "{track}: {err}"); @@ -365,124 +172,4 @@ mod tests { Info::new(0, Some(1_000)).unwrap() ); } - - #[tokio::test] - async fn segments_put_get_and_identical_collision() { - let store = memory(); - let object = two_groups(); - let key = store.put_segments("video", 3, &object).await.unwrap(); - assert_eq!(key, Key::segments("video", 3).unwrap()); - assert_eq!( - store.path(&key).unwrap().as_ref(), - "rec/video/segments/0000000000000000003" - ); - assert_eq!(store.get_segments("video", 3, true).await.unwrap(), object); - store.put_segments("video", 3, &object).await.unwrap(); - - let other = Object::new(vec![Group { - sequence: 5, - frames: vec![frame(0, b"X")], - }]); - assert!(matches!( - store.put_segments("video", 3, &other).await, - Err(Error::Conflict(_)) - )); - } - - #[tokio::test] - async fn listing_names_every_object() { - let store = memory(); - store.put_info("video", &Info::new(0, Some(1)).unwrap()).await.unwrap(); - store.put_segments("video", 1, &one_group(1, b"a")).await.unwrap(); - store.put_segments("video", 3, &one_group(3, b"b")).await.unwrap(); - store.put_segments("timeline.z", 2, &one_group(0, b"t")).await.unwrap(); - - let listed: Vec<_> = store.list(&Query::default()).try_collect().await.unwrap(); - let mut keys: Vec<_> = listed.into_iter().map(|listed| listed.key).collect(); - keys.sort_by(|a, b| store.path(a).unwrap().as_ref().cmp(store.path(b).unwrap().as_ref())); - assert_eq!( - keys, - vec![ - Key::segments("timeline.z", 2).unwrap(), - Key::info("video").unwrap(), - Key::segments("video", 1).unwrap(), - Key::segments("video", 3).unwrap(), - ] - ); - } - - #[tokio::test] - async fn list_with_offset_finishes_before_the_caller_sorts() -> Result<()> { - let store = memory(); - for segment in [0u64, 2, 1] { - store - .put_segments("timeline.z", segment, &one_group(segment, b"t")) - .await - .unwrap(); - } - let query = Query::segments("timeline.z")?.after(&Key::segments("timeline.z", 0)?)?; - let mut rest: Vec<_> = store - .list(&query) - .map_ok(|entry| match entry.key { - Key::Segments { segment, .. } => segment, - _ => panic!("expected a segment key"), - }) - .try_collect() - .await?; - rest.sort(); - assert_eq!(rest, vec![1, 2]); - Ok(()) - } - - #[tokio::test] - async fn percent_encoded_track_listing() { - let store = memory(); - store - .put_info("catalog.json", &Info::new(0, Some(1)).unwrap()) - .await - .unwrap(); - let locations = names(&store, &Query::default()).await; - assert!(locations.contains(&format!("{}/.info", encode_track("catalog.json").unwrap()))); - } - - #[tokio::test] - async fn empty_prefix_lists_the_whole_store() { - let store = Store::new(Mock::memory(), ""); - store - .put_info("catalog.json", &Info::new(0, Some(1)).unwrap()) - .await - .unwrap(); - store.put_segments("video", 4, &one_group(4, b"a")).await.unwrap(); - - let expected = - std::collections::HashSet::from([Key::info("catalog.json").unwrap(), Key::segments("video", 4).unwrap()]); - let streamed: std::collections::HashSet = store - .list(&Query::default()) - .map_ok(|entry| entry.key) - .try_collect() - .await - .unwrap(); - assert_eq!(streamed, expected); - } - - #[tokio::test] - async fn id_endpoints_are_valid_keys() { - let store = memory(); - let object = one_group(ID_MAX, b"z"); - store.put_segments("t", ID_MAX, &object).await.unwrap(); - assert_eq!(store.get_segments("t", ID_MAX, true).await.unwrap(), object); - } - - #[tokio::test] - async fn local_disk_roundtrip() { - let dir = tempfile::tempdir().unwrap(); - let inner = object_store::local::LocalFileSystem::new_with_prefix(dir.path()).unwrap(); - let store = Store::new(inner, Path::from("rec")); - let info = Info::new(3, Some(90_000)).unwrap(); - store.put_info("audio", &info).await.unwrap(); - let object = two_groups(); - store.put_segments("audio", 0, &object).await.unwrap(); - assert_eq!(store.get_info("audio").await.unwrap(), info); - assert_eq!(store.get_segments("audio", 0, true).await.unwrap(), object); - } } diff --git a/rs/moq-cli/MIGRATE.md b/rs/moq-cli/MIGRATE.md index 84620196579..92a92ed3cce 100644 --- a/rs/moq-cli/MIGRATE.md +++ b/rs/moq-cli/MIGRATE.md @@ -28,7 +28,7 @@ Add `--passthrough` to keep publishing CMAF. `moq export archive ` records every broadcast announced at or under `--broadcast`, each instance under `//@`, until the connection ends, instead of one broadcast at ``. Point `` at the root you want recordings under, and drop `--catalog-format`, which it now refuses. Untimed tracks record, and `--retention` works on them. -`import archive` does not read the new layout yet; it still replays recordings made by older `moq` versions. +`import archive` replays one recorded broadcast in the new layout, `//@`, and no longer reads recordings made by older `moq` versions. ### `--cluster-lan-secret` is a file path only (#5910) diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index 8254b97fe5f..bad6bd0b9c4 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -1,5 +1,5 @@ //! Archive endpoints: record every broadcast announced under a prefix into an object store -//! (export), or republish a recording from one (import), through `moq-archive`. +//! (export), or republish one recorded broadcast from one (import), through `moq-archive`. use std::collections::HashMap; use std::sync::Arc; @@ -38,13 +38,12 @@ pub struct ExportArgs { #[derive(usage_rs::Args, Clone)] #[usage(unknown_flags = "error", args_override_self = false)] pub struct ImportArgs { - /// The timeline recording to replay. + /// One recorded broadcast: the `/@` key prefix `export archive` wrote. pub store: Url, - /// Keep following a recording that is still growing, checking for new segments at this - /// interval. Unset replays what is stored now and ends the timeline there. + /// How often a recording still growing is checked for new objects and tracks. Defaults to 1s. #[usage(long)] - pub follow: Option, + pub refresh: Option, } /// One publisher instance being recorded: its path, and its epoch when its route has one. @@ -239,8 +238,9 @@ fn key(root: &Path, (path, epoch): &Instance) -> anyhow::Result { /// Republish the recording at `args.store` as the broadcast `name`, announced under `epoch`. /// -/// Each track's timeline replays as a live track and every other track's groups are served on -/// request. +/// Every stored track is served with its original groups: each live from its newest stored +/// group, and history on request. A recording still growing is followed until its producer +/// marks it complete. pub async fn import( origin: moq_net::origin::Producer, name: String, @@ -248,48 +248,24 @@ pub async fn import( epoch: moq_net::Epoch, ) -> anyhow::Result<()> { let store = open(&args.store)?; - // A second handle lists the recording's timelines while the reader owns the first. - let listing = open(&args.store)?; + let mut config = moq_archive::consumer::Config::default(); + if let Some(refresh) = args.refresh { + config = config.with_refresh(refresh.into()); + } let broadcast = origin.create_broadcast(&name).context("failed to create broadcast")?; - let mut timelines = store - .timelines() - .await - .with_context(|| format!("failed to list the recording at {}", args.store))?; - let config = moq_archive::reader::Config::new(timelines.clone()); - let mut reader = moq_archive::Reader::open(store, &broadcast, config) + let consumer = moq_archive::Consumer::open(store, broadcast.clone(), config) .await .with_context(|| format!("no readable recording at {}", args.store))?; - let serve = reader.serve(); broadcast .announce(moq_net::origin::Route::default().with_epoch(epoch)) .context("failed to announce broadcast")?; tracing::info!(%name, store = %args.store, "replaying"); notify_ready(); - - // The store holds no end marker, so only the caller can say the recording is complete. - let Some(interval) = args.follow.map(Into::into) else { - reader.finish()?; - serve.await; - return Ok(()); - }; - let refresh = async { - loop { - moq_clock::sleep(interval).await; - // A rendition the recording enrolled after the replay started. - for (track, timeline) in listing.timelines().await? { - if let std::collections::btree_map::Entry::Vacant(entry) = timelines.entry(track) { - reader.track(entry.key(), &timeline).await?; - entry.insert(timeline); - } - } - reader.refresh().await?; - } - }; - tokio::select! { - () = serve => Ok(()), - result = refresh => result, - } + consumer + .run() + .await + .with_context(|| format!("replaying {} failed", args.store)) } /// Open the object store `url` names, taking cloud credentials from the `AWS_*`, `GOOGLE_*`, @@ -303,7 +279,7 @@ fn open_store(url: &Url) -> anyhow::Result<(Arc, Path)> { Ok((Arc::from(store), prefix)) } -/// [`open_store`] as one recording. +/// [`open_store`] as one recorded broadcast. fn open(url: &Url) -> anyhow::Result>> { let (store, prefix) = open_store(url)?; Ok(moq_archive::Store::new(store, prefix)) @@ -405,6 +381,62 @@ mod tests { assert!(!files.iter().any(|file| file.starts_with("other")), "{files:?}"); } + /// `import archive` republishes a recording `export archive` wrote under the given name and + /// epoch, serving its stored groups. + #[tokio::test] + async fn an_exported_broadcast_imports() { + let dir = tempfile::tempdir().unwrap(); + let url = Url::from_directory_path(dir.path()).unwrap(); + + let source = moq_net::broadcast::Info::new().produce(); + let info = track::Info::default().with_timescale(Timescale::MILLI); + let video = source.create_track("video", info).unwrap(); + for sequence in 0..2 { + let mut group = video.create_group(group::Info { sequence }).unwrap(); + let pts = Timestamp::from_millis(sequence * 1000).unwrap(); + group.write_frame(pts, format!("frame {sequence}")).unwrap(); + group.finish().unwrap(); + } + video.finish().unwrap(); + let recording = + tokio::spawn(moq_archive::Producer::new(open(&url).unwrap(), source.consume(), Default::default()).run()); + // Its objects, then the markers once the broadcast ends. + tokio::time::timeout(Duration::from_secs(10), async { + while files(dir.path()).len() < 2 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the recording stores its track"); + source.close(); + recording.await.unwrap().unwrap(); + + let origin = moq_tokio::origin::spawn(); + let args = ImportArgs { + store: url, + refresh: None, + }; + let epoch = moq_net::Epoch::mint(); + let _import = tokio::spawn(import(origin.clone(), "replay".into(), args, epoch.clone())); + let mut announced = origin.consume().announced(); + tokio::time::timeout(Duration::from_secs(10), async { + loop { + if let Some(moq_net::announce::Event::Start(announce)) = announced.next().await { + assert_eq!(announce.prefix.as_str(), "replay"); + assert_eq!(announce.route.epoch, Some(epoch.clone())); + break; + } + } + }) + .await + .expect("the replay is announced"); + let broadcast = origin.consume().request_broadcast("replay", None).await.unwrap(); + let track = broadcast.track("video").unwrap(); + let mut group = track.fetch_group(0, None).await.unwrap(); + let frame = group.read_frame().await.unwrap().unwrap(); + assert_eq!(&frame.payload[..], b"frame 0"); + } + /// An epochless route announced again while its last recording drains is a new recording, and /// an epoch-pinned one records again once its recording ends. #[test] diff --git a/rs/moq-hls/Cargo.toml b/rs/moq-hls/Cargo.toml index 30b1f98e31b..2ff9f9fb647 100644 --- a/rs/moq-hls/Cargo.toml +++ b/rs/moq-hls/Cargo.toml @@ -37,7 +37,9 @@ tracing = { workspace = true } url = { workspace = true } [dev-dependencies] +async-trait = { workspace = true } futures = { workspace = true } +moq-archive = { path = "../moq-archive" } moq-json = { path = "../moq-json" } moq-tokio = { path = "../moq-tokio", default-features = false, features = ["tcp", "quic", "aws-lc-rs"] } tokio = { workspace = true, features = ["test-util"] } diff --git a/rs/moq-hls/src/export/archive_tests.rs b/rs/moq-hls/src/export/archive_tests.rs new file mode 100644 index 00000000000..3ddd9a405ef --- /dev/null +++ b/rs/moq-hls/src/export/archive_tests.rs @@ -0,0 +1,583 @@ +//! Serving a `moq-archive` recording as HLS. +//! +//! A source broadcast, its catalog and timelines included, is recorded by an +//! [`moq_archive::Producer`] and served back by an [`moq_archive::Consumer`], so the exporter +//! serves the archived catalog exactly like a live broadcast. These tests pin the storage traffic +//! that composition produces: playlists read only the timelines, and a segment GETs only objects +//! of the requested rendition. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bytes::{Bytes, BytesMut}; +use futures::stream::BoxStream; +use hang::timeline::{Position, Record}; +use moq_archive::Store; +use moq_archive::object_store::memory::InMemory; +use moq_archive::object_store::path::Path; +use moq_archive::object_store::{ + self, CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, + PutMultipartOptions, PutOptions, PutPayload, PutResult, +}; +use moq_json::window; +use moq_net::{Timescale, Timestamp, group, track}; + +use super::*; + +/// In-memory store that records every GET path. +#[derive(Debug, Clone, Default)] +struct Counting { + inner: Arc, + gets: Arc>>, + /// Fail every media GET, so nothing can quietly depend on one. + reject_media: Arc, +} + +impl Counting { + /// Every GET since the last call. + fn take(&self) -> Vec { + std::mem::take(&mut *self.gets.lock().unwrap()) + } + + fn reject_media(&self, reject: bool) { + self.reject_media.store(reject, Ordering::SeqCst); + } +} + +impl std::fmt::Display for Counting { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Counting") + } +} + +#[async_trait::async_trait] +impl ObjectStore for Counting { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + opts: PutOptions, + ) -> object_store::Result { + self.inner.put_opts(location, payload, opts).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(location, opts).await + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> object_store::Result { + self.gets.lock().unwrap().push(location.to_string()); + if self.reject_media.load(Ordering::SeqCst) && is_media(location.as_ref()) { + return Err(object_store::Error::NotImplemented { + operation: "media GET".into(), + implementer: "Counting".into(), + }); + } + self.inner.get_opts(location, options).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} + +/// One rendition of the source: its media track, and its timeline track with the window encoder +/// that writes it. +struct Source { + media: track::Producer, + timeline: track::Producer, + encoder: window::Encoder, + /// The timeline group still open, which a pop may extend. + group: Option, + /// Next timeline group sequence. + next: u64, + /// Next record sequence. + record: u64, +} + +/// A source broadcast recorded into a store as it publishes. +struct Recording { + /// The store's own handle, for its GET log. + counting: Counting, + store: Store, + broadcast: moq_net::broadcast::Producer, + catalog: moq_json::snapshot::Producer, + sources: HashMap, + recorder: tokio::task::JoinHandle>, +} + +impl Recording { + async fn new() -> Self { + let counting = Counting::default(); + let store = Store::new(counting.clone(), "rec"); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track(hang::Catalog::DEFAULT_NAME, hang::Catalog::default_track_info()) + .unwrap(); + let mut json = moq_json::snapshot::Config::default(); + json.deltas = false; + let mut catalog = moq_json::snapshot::Producer::new(track, json); + catalog + .update(moq_net::Timed::from(&self::catalog()).at(Timestamp::now())) + .unwrap(); + + let mut sources = HashMap::new(); + for name in ["360p", "1080p", "audio"] { + // Legacy media is stamped in microseconds. + let media = track::Info::default() + .with_timescale(Timescale::new(1_000_000).unwrap()) + .with_priority(1); + let media = broadcast.create_track(name, media).unwrap(); + let timeline = track::Info::default().with_timescale(Timescale::MILLI); + let timeline = broadcast.create_track(timeline_name(name), timeline).unwrap(); + let config = window::ProducerConfig::default().with_compression(true).with_ops(false); + let source = Source { + media, + timeline, + encoder: window::Encoder::new(config), + group: None, + next: 0, + record: 0, + }; + sources.insert(name.to_string(), source); + } + + // The source publishes in bursts, so the budget covers a whole recording. + let config = moq_archive::producer::Config::default() + .with_flush(Duration::from_millis(100)) + .with_max_delay(Duration::from_secs(3600)); + let recorder = tokio::spawn(moq_archive::Producer::new(store.clone(), broadcast.consume(), config).run()); + // Subscribed before anything is published, so every group is recorded live. + for source in sources.values() { + while source.media.subscription().is_none() || source.timeline.subscription().is_none() { + tokio::time::sleep(Duration::from_millis(1)).await; + } + } + Self { + counting, + store, + broadcast, + catalog, + sources, + recorder, + } + } + + /// Publish `track`'s next record: `groups`, each a list of frame timestamps in micros, + /// spanning `pts..pts + duration` in milliseconds, popping `pop` older records. + fn add(&mut self, track: &str, pts: u64, duration: u64, groups: &[(u64, &[u64])], pop: u64) { + let source = self.sources.get_mut(track).unwrap(); + for &(sequence, frames) in groups { + let mut group = source.media.create_group(group::Info { sequence }).unwrap(); + for (index, µs) in frames.iter().enumerate() { + let timestamp = Timestamp::from_micros(micros).unwrap(); + group.write_frame(timestamp, legacy(track, micros, index == 0)).unwrap(); + } + group.finish().unwrap(); + } + + let sequence = source.record; + source.record += 1; + let first = groups.first().unwrap().0; + let last = groups.last().unwrap().0; + let record = Record::new( + sequence, + pts, + duration, + Position::group(first), + Position::group(last + 1), + ); + let mut payloads = Vec::new(); + let pending = source.encoder.push(&record).unwrap(); + payloads.push((pending.keyframe, pending.payload.clone())); + pending.commit(); + if let Some(pending) = source.encoder.pop(pop).unwrap() { + payloads.push((pending.keyframe, pending.payload.clone())); + pending.commit(); + } + for (keyframe, payload) in payloads { + if keyframe || source.group.is_none() { + if let Some(mut group) = source.group.take() { + group.finish().unwrap(); + } + let group = source + .timeline + .create_group(group::Info { sequence: source.next }) + .unwrap(); + source.group = Some(group); + source.next += 1; + } + let timestamp = Timestamp::from_millis(pts).unwrap(); + source.group.as_mut().unwrap().write_frame(timestamp, payload).unwrap(); + } + } + + /// Wait until the recorder subscribed to every track and stored what was published. + async fn stored(&self) { + tokio::time::sleep(Duration::from_secs(5)).await; + } + + /// End the source cleanly, then wait for the recorder to mark the archive complete. + async fn finish(mut self) -> Store { + self.catalog.finish().unwrap(); + for source in self.sources.values_mut() { + if let Some(mut group) = source.group.take() { + group.finish().unwrap(); + } + source.media.finish().unwrap(); + source.timeline.finish().unwrap(); + } + self.stored().await; + self.broadcast.close(); + self.recorder.await.unwrap().unwrap(); + self.store + } + + /// Every GET since the last call. + fn gets(&self) -> Vec { + self.counting.take() + } +} + +fn timeline_name(track: &str) -> String { + hang::timeline::default_name(track) +} + +/// One Legacy frame: a VP8 keyframe for video (geometry the muxer can parse), filler otherwise. +fn legacy(track: &str, micros: u64, keyframe: bool) -> Bytes { + let payload: &'static [u8] = match (track.starts_with("audio"), keyframe) { + (true, _) => &[0xFC, 0xFF, 0xFE], + (false, true) => &[0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x40, 0x01, 0xf0, 0x00], + (false, false) => &[0x31, 0x00, 0x00], + }; + let frame = hang::container::Frame { + timestamp: Timestamp::from_micros(micros).unwrap(), + payload: Bytes::from_static(payload), + }; + let mut encoded = BytesMut::new(); + frame.encode(&mut encoded).unwrap(); + encoded.freeze() +} + +/// The source's catalog: every rendition's config, plus an `archive` entry naming each +/// rendition's timeline. Out-of-band configs, so no init needs media. +fn catalog() -> hang::Catalog { + let mut archive = hang::catalog::Archive::new(); + for track in ["360p", "1080p", "audio"] { + archive.timelines.insert( + track.to_string(), + hang::catalog::Timeline::new(timeline_name(track), 4_000), + ); + } + let mut catalog = hang::Catalog::default(); + catalog.archive = Some(archive); + for (name, width, height) in [("360p", 640, 360), ("1080p", 1920, 1080)] { + let mut config = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); + config.coded_width = Some(width); + config.coded_height = Some(height); + catalog.video.renditions.insert(name.to_string(), config); + } + let audio = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); + catalog.audio.renditions.insert("audio".to_string(), audio); + catalog +} + +/// An archive served back through an origin by a consumer, and a broadcaster exporting it. +struct Replay { + broadcaster: Arc, + /// Keeps the served broadcast's origin open. + _origin: moq_net::origin::Producer, + consumer: tokio::task::JoinHandle>, + driver: tokio::task::JoinHandle, +} + +impl Replay { + async fn open(store: &Store, cache: u64) -> Self { + let (origin, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::default()); + let driver = tokio::spawn(moq_net::time::run(driver)); + let broadcast = origin.create_broadcast("rec").unwrap(); + let serving = moq_archive::consumer::Config::default().with_cache(cache); + let consumer = moq_archive::Consumer::open(store.clone(), broadcast.clone(), serving) + .await + .unwrap(); + let consumer = tokio::spawn(consumer.run()); + broadcast.announce(Default::default()).unwrap(); + + let broadcaster = Broadcaster::new(crate::resolved(&origin, "rec").await, Config::default()); + tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()) + .await + .expect("a rendition becomes playable or fails"); + + Self { + broadcaster, + _origin: origin, + consumer, + driver, + } + } + + fn rendition(&self, kind: Kind, name: &str) -> Arc { + self.broadcaster + .rendition(kind, name) + .expect("rendition in the catalog") + } + + async fn playlist(&self, kind: Kind, name: &str) -> String { + let rendition = self.rendition(kind, name); + tokio::time::timeout(Duration::from_secs(5), rendition.playlist(None)) + .await + .expect("playlist renders") + .unwrap() + .expect("playlist is servable") + } + + /// Wait until `name`'s playlist satisfies `ready`, then return it. + async fn playlist_until(&self, kind: Kind, name: &str, ready: impl Fn(&str) -> bool) -> String { + for _ in 0..500 { + let playlist = self.playlist(kind, name).await; + if ready(&playlist) { + return playlist; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("{name} playlist never became ready"); + } +} + +impl Drop for Replay { + fn drop(&mut self) { + self.consumer.abort(); + self.driver.abort(); + } +} + +/// A stored media object: neither a timeline, the catalog, nor a marker. +fn is_media(path: &str) -> bool { + let name = path.rsplit('/').next().unwrap_or_default(); + let marker = [".info", "end", ".end"].contains(&name); + !marker && !path.contains("timeline") && !path.contains("catalog") +} + +/// `count` 2s segments: one keyframe group per video record, four audio groups per audio record. +/// The first video rendition by name, 1080p, is the reference. +async fn segments(count: u64) -> Recording { + let mut recording = Recording::new().await; + for segment in 0..count { + add_segment(&mut recording, segment, 0); + } + recording +} + +fn add_segment(recording: &mut Recording, segment: u64, pop: u64) { + let pts = segment * 2_000_000; + for video in ["360p", "1080p"] { + recording.add(video, segment * 2000, 2000, &[(segment, &[pts, pts + 1_000_000])], pop); + } + let audio: Vec<(u64, [u64; 1])> = (0..4).map(|i| (segment * 4 + i, [pts + i * 500_000])).collect(); + let audio: Vec<(u64, &[u64])> = audio + .iter() + .map(|(sequence, frames)| (*sequence, &frames[..])) + .collect(); + recording.add("audio", segment * 2000, 2000, &audio, pop); +} + +#[tokio::test(start_paused = true)] +async fn playlists_read_only_timelines_and_segments_their_own_objects() { + let recording = segments(3).await; + recording.stored().await; + let replay = Replay::open(&recording.store, 64 * 1024 * 1024).await; + + let master = replay.broadcaster.master_playlist(None); + assert!(master.contains("video/360p/media.m3u8") && master.contains("video/1080p/media.m3u8")); + + // Render and reload every playlist with media GETs refused: shared numbering, and not one + // media GET. The reference lists every segment; the others hold back the newest until their + // own timeline passes its end, or ends. + recording.counting.reject_media(true); + for _ in 0..2 { + for (kind, name, listed) in [ + (Kind::Video, "1080p", 3), + (Kind::Video, "360p", 2), + (Kind::Audio, "audio", 3), + ] { + let playlist = replay.playlist(kind, name).await; + for segment in 0..3 { + let line = format!("seg/29b1cd67.{segment}.m4s\n"); + assert_eq!(playlist.contains(&line), segment < listed, "{name}: {playlist}"); + } + assert!(!playlist.contains("#EXT-X-ENDLIST"), "the recording is still growing"); + } + } + let gets = recording.gets(); + for rendition in ["360p", "1080p", "audio"] { + assert!( + gets.contains(&format!("rec/{rendition}/.info")), + "track properties bind before media: {gets:?}" + ); + } + assert!(gets.iter().any(|path| path.contains("timeline")), "{gets:?}"); + recording.counting.reject_media(false); + + // Switching renditions downloads only the selected rendition's objects. + for (kind, name, segment) in [ + (Kind::Video, "360p", 1), + // Not the newest: a growing archive keeps its newest group open, as a live track does. + (Kind::Video, "1080p", 1), + // An audio segment spans four groups. + (Kind::Audio, "audio", 1), + ] { + let media = replay + .rendition(kind, name) + .listed_segment(segment) + .await + .unwrap() + .unwrap(); + assert_eq!(&media[4..8], b"moof"); + let gets = recording.gets(); + assert!(!gets.is_empty(), "{name}"); + assert!( + gets.iter().all(|path| path.starts_with(&format!("rec/{name}/"))), + "{name}: {gets:?}" + ); + } + + // A repeated request hits the consumer's cache. + replay + .rendition(Kind::Video, "360p") + .listed_segment(1) + .await + .unwrap() + .unwrap(); + assert_eq!(recording.gets(), Vec::::new()); +} + +#[tokio::test(start_paused = true)] +async fn a_bounded_cache_rereads_evicted_objects() { + let recording = segments(3).await; + recording.stored().await; + // Too small for any object, so nothing stays cached. + let replay = Replay::open(&recording.store, 1).await; + replay.playlist(Kind::Audio, "audio").await; + recording.gets(); + + // Each of the segment's four groups misses the cache and GETs its object again. + let audio = replay + .rendition(Kind::Audio, "audio") + .listed_segment(1) + .await + .unwrap() + .unwrap(); + assert_eq!(&audio[4..8], b"moof"); + let gets: Vec = recording.gets().into_iter().filter(|path| is_media(path)).collect(); + assert_eq!(gets.len(), 4, "{gets:?}"); +} + +#[tokio::test(start_paused = true)] +async fn missing_rendition_segments_are_gaps_and_time_jumps_are_discontinuities() { + let mut recording = Recording::new().await; + recording.add("1080p", 0, 2000, &[(0, &[0])], 0); + recording.add("360p", 0, 8000, &[(0, &[0])], 0); + recording.add("audio", 0, 2000, &[(0, &[0])], 0); + // 360p stored nothing for segment 1. + recording.add("1080p", 2000, 2000, &[(1, &[2_000_000])], 0); + // Content time jumps from 4s to 10s. + recording.add("1080p", 10_000, 2000, &[(2, &[10_000_000])], 0); + recording.add("360p", 10_000, 2000, &[(1, &[10_000_000])], 0); + // The completed archive resolves each rendition's newest segment. + let counting = recording.counting.clone(); + let store = recording.finish().await; + + let replay = Replay::open(&store, 64 * 1024 * 1024).await; + let low = replay + .playlist_until(Kind::Video, "360p", |playlist| playlist.contains("#EXT-X-ENDLIST")) + .await; + let expected = concat!( + "#EXTINF:2.00000,\nseg/29b1cd67.0.m4s\n", + "#EXT-X-GAP\n#EXTINF:2.00000,\nseg/29b1cd67.1.m4s\n", + "#EXT-X-DISCONTINUITY\n#EXTINF:2.00000,\nseg/29b1cd67.2.m4s\n", + ); + assert!(low.contains(expected), "{low}"); + let high = replay.playlist(Kind::Video, "1080p").await; + assert!(!high.contains("#EXT-X-GAP"), "{high}"); + assert!( + high.contains("#EXT-X-DISCONTINUITY\n#EXTINF:2.00000,\nseg/29b1cd67.2.m4s\n"), + "{high}" + ); + + // A gap is never fetched. + counting.take(); + let rendition = replay.rendition(Kind::Video, "360p"); + assert!(rendition.listed_segment(1).await.unwrap().is_none()); + assert!(!counting.take().iter().any(|path| is_media(path))); + let after = rendition.listed_segment(2).await.unwrap().unwrap(); + assert_eq!(&after[4..8], b"moof"); +} + +/// A growing recording is followed as the archive grows, and ends once its producer marks it +/// complete. +#[tokio::test(start_paused = true)] +async fn a_growing_recording_ends_with_its_markers() { + let mut recording = segments(3).await; + recording.stored().await; + let replay = Replay::open(&recording.store, 64 * 1024 * 1024).await; + replay.playlist(Kind::Video, "1080p").await; + + // A DVR commit: segment 3 arrives and segment 0 expires. + add_segment(&mut recording, 3, 1); + recording.stored().await; + let playlist = replay + .playlist_until(Kind::Video, "1080p", |playlist| { + playlist.contains("seg/29b1cd67.3.m4s\n") + }) + .await; + assert!(playlist.contains("#EXT-X-MEDIA-SEQUENCE:1\n"), "{playlist}"); + assert!(!playlist.contains("seg/29b1cd67.0.m4s"), "{playlist}"); + assert!( + !playlist.contains("#EXT-X-ENDLIST"), + "a recording without markers stays live" + ); + + recording.finish().await; + let playlist = replay + .playlist_until(Kind::Video, "1080p", |playlist| playlist.contains("#EXT-X-ENDLIST")) + .await; + assert!(playlist.contains("seg/29b1cd67.3.m4s\n#EXT-X-ENDLIST\n"), "{playlist}"); +} + +/// A durable timeline still lists only the default 16s window: a live playlist stays bounded +/// however much the store retains. +#[tokio::test(start_paused = true)] +async fn a_durable_timeline_lists_the_window() { + let recording = segments(12).await; + recording.stored().await; + let replay = Replay::open(&recording.store, 64 * 1024 * 1024).await; + let playlist = replay + .playlist_until(Kind::Video, "1080p", |playlist| { + playlist.contains("seg/29b1cd67.11.m4s\n") + }) + .await; + // Eight 2s segments cover the window. + assert!(playlist.contains("#EXT-X-MEDIA-SEQUENCE:4\n"), "{playlist}"); + assert!(!playlist.contains("seg/29b1cd67.3.m4s\n"), "{playlist}"); +} diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 7c73d43fe84..9deac2f0335 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -5,7 +5,7 @@ //! pure function of the timeline records, and media bytes move only when an HTTP client requests //! a segment, which FETCHes exactly the frames that segment covers from the relay cache and //! transmuxes them to CMAF. A broadcast whose catalog advertises no timelines can't be served this -//! way and is skipped, and a rendition is served only once its own timeline is advertised. A `moq-archive` recording replayed onto a broadcast is served the same way: +//! way and is skipped, and a rendition is served only once its own timeline is advertised. A `moq-archive` recording served by its `Consumer` is served the same way: //! its playlists read only the stored timelines, and a segment GETs only its rendition's objects. //! //! Segment boundaries come from one reference rendition's records, numbered by record sequence, @@ -28,6 +28,9 @@ //! yield every rendition and every finalized segment in order, for mirroring a broadcast to //! storage. +#[cfg(test)] +#[expect(clippy::disallowed_methods, reason = "tests read the host clock directly")] +mod archive_tests; pub(crate) mod master; mod mpd; mod playlist;