diff --git a/Cargo.lock b/Cargo.lock index 2ee2b9e51..f5f5a846e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1605,7 +1605,6 @@ dependencies = [ "anyhow", "axum", "axum-server", - "bytes", "clap", "hang", "moq-native", diff --git a/bun.lock b/bun.lock index 99fb7edc5..86d3e6c32 100644 --- a/bun.lock +++ b/bun.lock @@ -39,6 +39,7 @@ "@libav.js/variant-opus-af": "^6.8.8", "@moq/lite": "workspace:^", "@moq/signals": "workspace:^", + "@svta/cml-iso-bmff": "^1.0.0-alpha.9", "async-mutex": "^0.5.0", "comlink": "^4.4.2", "zod": "^4.1.5", @@ -493,6 +494,10 @@ "@speed-highlight/core": ["@speed-highlight/core@1.2.14", "", {}, "sha512-G4ewlBNhUtlLvrJTb88d2mdy2KRijzs4UhnlrOSRT4bmjh/IqNElZa3zkrZ+TC47TwtlDWzVLFADljF1Ijp5hA=="], + "@svta/cml-iso-bmff": ["@svta/cml-iso-bmff@1.0.0-alpha.9", "", { "peerDependencies": { "@svta/cml-utils": "1.1.0" } }, "sha512-m2rrVjOHdZv8vvoKh++xCCDUU7g97HTmlu0SGVf3AXq7qKY8OrNbtQU0mbZrN2s71KbLEc7wfC/ZZIipiPmH3Q=="], + + "@svta/cml-utils": ["@svta/cml-utils@1.1.0", "", {}, "sha512-5RyHD75RYbq0clUkb/L/+JklxAq+PZRAwKZTcmqUt/ciHm79HBq0/IgrDXYvTgIRGRv8gE4GNvUWQbvRZRxZpA=="], + "@tailwindcss/node": ["@tailwindcss/node@4.1.18", "", { "dependencies": { "@jridgewell/remapping": "^2.3.4", "enhanced-resolve": "^5.18.3", "jiti": "^2.6.1", "lightningcss": "1.30.2", "magic-string": "^0.30.21", "source-map-js": "^1.2.1", "tailwindcss": "4.1.18" } }, "sha512-DoR7U1P7iYhw16qJ49fgXUlry1t4CpXeErJHnQ44JgTSKMaZUdf17cfn5mHchfJ4KRBZRFA/Coo+MUF5+gOaCQ=="], "@tailwindcss/oxide": ["@tailwindcss/oxide@4.1.18", "", { "optionalDependencies": { "@tailwindcss/oxide-android-arm64": "4.1.18", "@tailwindcss/oxide-darwin-arm64": "4.1.18", "@tailwindcss/oxide-darwin-x64": "4.1.18", "@tailwindcss/oxide-freebsd-x64": "4.1.18", "@tailwindcss/oxide-linux-arm-gnueabihf": "4.1.18", "@tailwindcss/oxide-linux-arm64-gnu": "4.1.18", "@tailwindcss/oxide-linux-arm64-musl": "4.1.18", "@tailwindcss/oxide-linux-x64-gnu": "4.1.18", "@tailwindcss/oxide-linux-x64-musl": "4.1.18", "@tailwindcss/oxide-wasm32-wasi": "4.1.18", "@tailwindcss/oxide-win32-arm64-msvc": "4.1.18", "@tailwindcss/oxide-win32-x64-msvc": "4.1.18" } }, "sha512-EgCR5tTS5bUSKQgzeMClT6iCY3ToqE1y+ZB0AKldj809QXk1Y+3jB0upOYZrn9aGIzPtUsP7sX4QQ4XtjBB95A=="], diff --git a/deno.lock b/deno.lock index 8e95c842c..ee7053a71 100644 --- a/deno.lock +++ b/deno.lock @@ -37,6 +37,7 @@ "dependencies": [ "npm:@kixelated/libavjs-webcodecs-polyfill@~0.5.5", "npm:@libav.js/variant-opus-af@^6.8.8", + "npm:@svta/cml-iso-bmff@^1.0.0-alpha.9", "npm:@types/audioworklet@^0.0.77", "npm:@types/web@^0.0.241", "npm:async-mutex@0.5", diff --git a/js/hang-demo/src/index.html b/js/hang-demo/src/index.html index adbc0caf7..017cbc76e 100644 --- a/js/hang-demo/src/index.html +++ b/js/hang-demo/src/index.html @@ -24,14 +24,16 @@ The broadcast path is overwritten by the ?path query parameter in index.ts. - TODO: There's a bug with Big Buck Bunny causing audio to stutter, so we need to increase the latency to 100ms. - NOTE: `reload` will detect when the broadcast goes offline/online and automatically reconnect. TODO: Cloudflare doesn't support it yet (SUBSCRIBE_NAMESPACE), so make sure you remove it if you're using Cloudflare. + + NOTE: You can use a
- Using something more niche? There's also a subscribe() method to trigger a callback on change.
+ Using something more niche? There's also a subscribe() method to
+ trigger a callback on change.
const cleanup = hang.volume.subscribe((volume) => {
diff --git a/js/hang-demo/src/meet.html b/js/hang-demo/src/meet.html
index 58d261247..ae77d0133 100644
--- a/js/hang-demo/src/meet.html
+++ b/js/hang-demo/src/meet.html
@@ -36,8 +36,9 @@
Other demos:
- - Watch a single broadcast.
- - Publish a single broadcast.
+ - Watch a broadcast (WebCodecs).
+ - Watch a broadcast (MSE).
+ - Publish a broadcast.
- Check browser support.
diff --git a/js/hang-demo/src/mse.html b/js/hang-demo/src/mse.html
new file mode 100644
index 000000000..75e604762
--- /dev/null
+++ b/js/hang-demo/src/mse.html
@@ -0,0 +1,64 @@
+
+
+
+
+
+
+ MoQ Demo (MSE)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Other demos:
+
+ - Watch a broadcast (WebCodecs).
+ - Publish a broadcast.
+ - Watch a room of broadcasts.
+ - Check browser support.
+
+
+ Tips:
+
+ This demo uses MSE (Media
+ Source Extensions)
+ via a <video> element instead of WebCodecs via <canvas>.
+ MSE has broader device support but higher latency.
+
+
+ To use MSE, provide a <video> element instead of a <canvas> element:
+
<hang-watch url="https://cdn.moq.dev/anon" path="bbb">
+ <video style="max-width: 100%; height: auto;" autoplay muted></video>
+</hang-watch>
+
+
+
+
+
+
diff --git a/js/hang-demo/src/publish.html b/js/hang-demo/src/publish.html
index 2d74b141a..9f4a412e7 100644
--- a/js/hang-demo/src/publish.html
+++ b/js/hang-demo/src/publish.html
@@ -36,6 +36,7 @@ Other demos:
diff --git a/js/hang-demo/src/support.html b/js/hang-demo/src/support.html
index f31267ad3..4af3bf722 100644
--- a/js/hang-demo/src/support.html
+++ b/js/hang-demo/src/support.html
@@ -16,7 +16,8 @@
Other demos:
- - Watch a single broadcast.
+ - Watch a broadcast (WebCodecs).
+ - Watch a broadcast (MSE).
- Publish a broadcast.
- Watch a room of broadcasts.
diff --git a/js/hang-ui/src/shared/components/stats/providers/audio.ts b/js/hang-ui/src/shared/components/stats/providers/audio.ts
index 9a685a513..dffe3d54c 100644
--- a/js/hang-ui/src/shared/components/stats/providers/audio.ts
+++ b/js/hang-ui/src/shared/components/stats/providers/audio.ts
@@ -42,13 +42,11 @@ export class AudioProvider extends BaseProvider {
return;
}
- const active = this.props.audio.source.active.peek();
+ const rendition = this.props.audio.rendition.peek();
+ const config = this.props.audio.config.peek();
+ const stats = this.props.audio.stats.peek();
- const config = this.props.audio.source.config.peek();
-
- const stats = this.props.audio.source.stats.peek();
-
- if (!active || !config) {
+ if (!rendition || !config) {
this.context.setDisplayData("N/A");
return;
}
diff --git a/js/hang-ui/src/shared/components/stats/providers/buffer.ts b/js/hang-ui/src/shared/components/stats/providers/buffer.ts
index b1f29499e..71cb77f17 100644
--- a/js/hang-ui/src/shared/components/stats/providers/buffer.ts
+++ b/js/hang-ui/src/shared/components/stats/providers/buffer.ts
@@ -1,5 +1,5 @@
import type { Getter } from "@moq/signals";
-import type { BufferStatus, ProviderContext, SyncStatus } from "../types";
+import type { ProviderContext } from "../types";
import { BaseProvider } from "./base";
/**
@@ -22,21 +22,7 @@ export class BufferProvider extends BaseProvider {
}
this.signals.effect((effect) => {
- const syncStatus = effect.get(video.source.syncStatus as Getter);
- const bufferStatus = effect.get(video.source.bufferStatus as Getter);
- const latency = effect.get(video.source.latency as Getter);
-
- const isLatencyValid = latency !== null && latency !== undefined && latency > 0;
- const bufferPercentage =
- syncStatus?.state === "wait" && syncStatus?.bufferDuration !== undefined && isLatencyValid
- ? Math.min(100, Math.round((syncStatus.bufferDuration / latency) * 100))
- : bufferStatus?.state === "filled"
- ? 100
- : 0;
-
- const parts = [`${bufferPercentage}%`, isLatencyValid ? `${latency}ms` : "N/A"];
-
- this.context?.setDisplayData(parts.join("\n"));
+ this.context?.setDisplayData("TODO");
});
}
}
diff --git a/js/hang-ui/src/shared/components/stats/providers/video.ts b/js/hang-ui/src/shared/components/stats/providers/video.ts
index 853210060..4e8946568 100644
--- a/js/hang-ui/src/shared/components/stats/providers/video.ts
+++ b/js/hang-ui/src/shared/components/stats/providers/video.ts
@@ -46,8 +46,8 @@ export class VideoProvider extends BaseProvider {
return;
}
- const display = this.props.video.source.display.peek();
- const stats = this.props.video.source.stats.peek();
+ const catalog = this.props.video.catalog.peek();
+ const stats = this.props.video.stats.peek();
const now = performance.now();
// Calculate FPS from frame count delta and timestamp delta
@@ -90,7 +90,7 @@ export class VideoProvider extends BaseProvider {
this.previousWhen = now;
}
- const { width, height } = display ?? {};
+ const { width, height } = catalog?.display ?? {};
const parts = [
width && height ? `${width}x${height}` : "N/A",
diff --git a/js/hang-ui/src/shared/components/stats/types.ts b/js/hang-ui/src/shared/components/stats/types.ts
index bb73c4730..0a0d42e01 100644
--- a/js/hang-ui/src/shared/components/stats/types.ts
+++ b/js/hang-ui/src/shared/components/stats/types.ts
@@ -1,5 +1,8 @@
export type KnownStatsProviders = "network" | "video" | "audio" | "buffer";
+import type * as Hang from "@moq/hang";
+import type * as Moq from "@moq/lite";
+
/**
* Context passed to providers for updating display data
*/
@@ -15,84 +18,15 @@ export interface VideoResolution {
height: number;
}
-/**
- * Stream sync status with buffer information
- */
-export interface SyncStatus {
- state: "ready" | "wait";
- bufferDuration?: number;
-}
-
-/**
- * Stream buffer fill status
- */
-export interface BufferStatus {
- state: "empty" | "filled";
-}
-
-/**
- * Generic reactive signal interface for accessing stream data
- */
-export interface Signal {
- peek(): T | undefined;
- changed?(callback: (value: T | undefined) => void): () => void;
- subscribe?(callback: () => void): () => void;
-}
-
-/**
- * Audio stream statistics
- */
-export type AudioStats = {
- bytesReceived: number;
-};
-
-/**
- * Audio stream source with reactive properties
- */
-export interface AudioSource {
- source: {
- active: Signal;
- config: Signal;
- stats: Signal;
- };
-}
-
-/**
- * Audio stream configuration properties
- */
-export interface AudioConfig {
- sampleRate: number;
- numberOfChannels: number;
- bitrate?: number;
- codec: string;
-}
+// TODO Don't re-export these types?
+export type Signal = Moq.Signals.Getter;
+export type AudioStats = Hang.Watch.Audio.Stats;
+export type AudioSource = Hang.Watch.Audio.Backend;
+export type AudioConfig = Hang.Catalog.AudioConfig;
+export type VideoStats = Hang.Watch.Video.Stats;
-/**
- * Video stream statistics
- */
-export type VideoStats = {
- frameCount: number;
- timestamp: number;
- bytesReceived: number;
+// TODO use Hang.Watch.Backend instead?
+export type ProviderProps = {
+ audio: Hang.Watch.Audio.Backend;
+ video: Hang.Watch.Video.Backend;
};
-
-/**
- * Video stream source with reactive properties
- */
-export interface VideoSource {
- source: {
- display: Signal;
- syncStatus: Signal;
- bufferStatus: Signal;
- latency: Signal;
- stats: Signal;
- };
-}
-
-/**
- * Props passed to metric providers containing stream sources
- */
-export interface ProviderProps {
- audio?: AudioSource;
- video?: VideoSource;
-}
diff --git a/js/hang-ui/src/watch/components/BufferControl.tsx b/js/hang-ui/src/watch/components/BufferControl.tsx
new file mode 100644
index 000000000..ccd5d6cc3
--- /dev/null
+++ b/js/hang-ui/src/watch/components/BufferControl.tsx
@@ -0,0 +1,136 @@
+import { createMemo, createSignal, For, Show } from "solid-js";
+import useWatchUIContext from "../hooks/use-watch-ui";
+
+const MIN_RANGE = 0;
+const RANGE_STEP = 100;
+
+type BufferControlProps = {
+ /** Maximum buffer range in milliseconds (default: 5000ms = 5s) */
+ max?: number;
+};
+
+export default function BufferControl(props: BufferControlProps) {
+ const context = useWatchUIContext();
+ const maxRange = () => props.max ?? 5000;
+ const [isDragging, setIsDragging] = createSignal(false);
+
+ // Compute range style and overflow info relative to current timestamp
+ const computeRange = (range: { start: number; end: number }, timestamp: number, color: string) => {
+ const startMs = (range.start - timestamp) * 1000;
+ const endMs = (range.end - timestamp) * 1000;
+ const visibleStartMs = Math.max(0, startMs);
+ const visibleEndMs = Math.min(endMs, maxRange());
+ const leftPct = (visibleStartMs / maxRange()) * 100;
+ const widthPct = Math.max(0.5, ((visibleEndMs - visibleStartMs) / maxRange()) * 100);
+ const isOverflow = endMs > maxRange();
+ const overflowSec = isOverflow ? ((endMs - visibleStartMs) / 1000).toFixed(1) : null;
+ return {
+ style: `left: ${leftPct}%; width: ${widthPct}%; background: ${color};`,
+ isOverflow,
+ overflowSec,
+ };
+ };
+
+ // Determine color based on gap detection and buffering state
+ const rangeColor = (index: number, isBuffering: boolean) => {
+ if (isBuffering) return "#f87171"; // red
+ if (index > 0) return "#facc15"; // yellow
+ return "#4ade80"; // green
+ };
+
+ const bufferTargetPct = createMemo(() => (context.latency() / maxRange()) * 100);
+
+ // Handle mouse interaction to set buffer via clicking/dragging on the visualization
+ let containerRef: HTMLDivElement | undefined;
+
+ const LABEL_WIDTH = 48; // px reserved for track labels
+
+ const updateBufferFromMouseX = (clientX: number) => {
+ if (!containerRef) return;
+ const rect = containerRef.getBoundingClientRect();
+ const trackWidth = rect.width - LABEL_WIDTH;
+ const x = Math.max(0, Math.min(clientX - rect.left - LABEL_WIDTH, trackWidth));
+ const ms = (x / trackWidth) * maxRange();
+ const snapped = Math.round(ms / RANGE_STEP) * RANGE_STEP;
+ const clamped = Math.max(MIN_RANGE, Math.min(maxRange(), snapped));
+ context.setLatencyValue(clamped);
+ };
+
+ const onMouseDown = (e: MouseEvent) => {
+ setIsDragging(true);
+ updateBufferFromMouseX(e.clientX);
+ document.addEventListener("mousemove", onMouseMove);
+ document.addEventListener("mouseup", onMouseUp);
+ };
+
+ const onMouseMove = (e: MouseEvent) => {
+ if (isDragging()) {
+ updateBufferFromMouseX(e.clientX);
+ }
+ };
+
+ const onMouseUp = () => {
+ setIsDragging(false);
+ document.removeEventListener("mousemove", onMouseMove);
+ document.removeEventListener("mouseup", onMouseUp);
+ };
+
+ return (
+
+ {/* Buffer Visualization - interactive, click/drag to set buffer */}
+
+ {/* Playhead (left edge = current time) */}
+
+
+ {/* Video buffer track */}
+
+ Video
+
+ {(range, i) => {
+ const info = () =>
+ computeRange(range, context.timestamp(), rangeColor(i(), context.buffering()));
+ return (
+
+
+ {info().overflowSec}s
+
+
+ );
+ }}
+
+
+
+ {/* Audio buffer track */}
+
+ Audio
+
+ {(range, i) => {
+ const info = () =>
+ computeRange(range, context.timestamp(), rangeColor(i(), context.buffering()));
+ return (
+
+
+ {info().overflowSec}s
+
+
+ );
+ }}
+
+
+
+ {/* Buffer target line (draggable) - wrapped in track-area container */}
+
+
+ {`${Math.round(context.latency())}ms`}
+
+
+
+
+ );
+}
diff --git a/js/hang-ui/src/watch/components/LatencySlider.tsx b/js/hang-ui/src/watch/components/LatencySlider.tsx
deleted file mode 100644
index 85db1b8de..000000000
--- a/js/hang-ui/src/watch/components/LatencySlider.tsx
+++ /dev/null
@@ -1,33 +0,0 @@
-import useWatchUIContext from "../hooks/use-watch-ui";
-
-const MIN_RANGE = 0;
-const MAX_RANGE = 5_000;
-const RANGE_STEP = 100;
-
-export default function LatencySlider() {
- const context = useWatchUIContext();
- const onInputChange = (event: Event) => {
- const target = event.currentTarget as HTMLInputElement;
- const latency = parseFloat(target.value);
- context.setLatencyValue(latency);
- };
-
- return (
-
-
-
- {typeof context.latency() !== "undefined" ? `${Math.round(context.latency())}ms` : ""}
-
- );
-}
diff --git a/js/hang-ui/src/watch/components/WatchControls.tsx b/js/hang-ui/src/watch/components/WatchControls.tsx
index b718454d8..d77281be1 100644
--- a/js/hang-ui/src/watch/components/WatchControls.tsx
+++ b/js/hang-ui/src/watch/components/WatchControls.tsx
@@ -1,5 +1,5 @@
+import BufferControl from "./BufferControl";
import FullscreenButton from "./FullscreenButton";
-import LatencySlider from "./LatencySlider";
import PlayPauseButton from "./PlayPauseButton";
import QualitySelector from "./QualitySelector";
import StatsButton from "./StatsButton";
@@ -17,7 +17,7 @@ export default function WatchControls() {
-
+
diff --git a/js/hang-ui/src/watch/context.tsx b/js/hang-ui/src/watch/context.tsx
index 8eaa9e8de..5d7651dcf 100644
--- a/js/hang-ui/src/watch/context.tsx
+++ b/js/hang-ui/src/watch/context.tsx
@@ -1,6 +1,7 @@
import { type Moq, Signals } from "@moq/hang";
-import type * as Catalog from "@moq/hang/catalog";
+import type { BufferedRanges } from "@moq/hang/watch";
import type HangWatch from "@moq/hang/watch/element";
+import solid from "@moq/signals/solid";
import type { JSX } from "solid-js";
import { createContext, createSignal, onCleanup } from "solid-js";
@@ -36,6 +37,9 @@ type WatchUIContextValues = {
setIsStatsPanelVisible: (visible: boolean) => void;
isFullscreen: () => boolean;
toggleFullscreen: () => void;
+ timestamp: () => number;
+ videoBuffered: () => BufferedRanges;
+ audioBuffered: () => BufferedRanges;
};
export const WatchUIContext = createContext();
@@ -65,24 +69,29 @@ export default function WatchUIContextProvider(props: WatchUIContextProviderProp
};
const setVolume = (volume: number) => {
- props.hangWatch.volume.set(volume / 100);
+ props.hangWatch.audio.volume.set(volume / 100);
};
const toggleMuted = () => {
- props.hangWatch.muted.update((muted) => !muted);
+ props.hangWatch.audio.muted.update((muted) => !muted);
};
const setLatencyValue = (latency: number) => {
- props.hangWatch.latency.set(latency as Moq.Time.Milli);
+ props.hangWatch.buffer.set(latency as Moq.Time.Milli);
};
const setActiveRenditionValue = (name: string | undefined) => {
- props.hangWatch.video.source.target.update((prev) => ({
+ props.hangWatch.video.target.update((prev) => ({
...prev,
- rendition: name,
+ name: name,
}));
};
+ // Use solid helper for the new signals
+ const timestamp = solid(props.hangWatch.timestamp);
+ const videoBuffered = solid(props.hangWatch.video.buffered);
+ const audioBuffered = solid(props.hangWatch.audio.buffered);
+
const value: WatchUIContextValues = {
hangWatch: props.hangWatch,
watchStatus,
@@ -102,6 +111,9 @@ export default function WatchUIContextProvider(props: WatchUIContextProviderProp
setIsStatsPanelVisible,
isFullscreen,
toggleFullscreen,
+ timestamp,
+ videoBuffered,
+ audioBuffered,
};
const watch = props.hangWatch;
@@ -130,7 +142,7 @@ export default function WatchUIContextProvider(props: WatchUIContextProviderProp
});
signals.effect((effect) => {
- const paused = effect.get(watch.video.paused);
+ const paused = effect.get(watch.paused);
setIsPlaying(!paused);
});
@@ -145,34 +157,30 @@ export default function WatchUIContextProvider(props: WatchUIContextProviderProp
});
signals.effect((effect) => {
- const syncStatus = effect.get(watch.video.source.syncStatus);
- const bufferStatus = effect.get(watch.video.source.bufferStatus);
- const shouldShow = syncStatus.state === "wait" || bufferStatus.state === "empty";
-
- setBuffering(shouldShow);
+ const buffering = effect.get(watch.buffering);
+ setBuffering(buffering);
});
signals.effect((effect) => {
- const latency = effect.get(watch.latency);
+ const latency = effect.get(watch.buffer);
setLatency(latency);
});
signals.effect((effect) => {
- const rootCatalog = effect.get(watch.broadcast.catalog);
- const videoCatalog = rootCatalog?.video;
+ const videoCatalog = effect.get(watch.video.catalog);
const renditions = videoCatalog?.renditions ?? {};
const renditionsList: Rendition[] = Object.entries(renditions).map(([name, config]) => ({
name,
- width: (config as Catalog.VideoConfig).codedWidth,
- height: (config as Catalog.VideoConfig).codedHeight,
+ width: config.codedWidth,
+ height: config.codedHeight,
}));
setAvailableRenditions(renditionsList);
});
signals.effect((effect) => {
- const selected = effect.get(watch.video.source.active);
+ const selected = effect.get(watch.video.rendition);
setActiveRendition(selected);
});
diff --git a/js/hang-ui/src/watch/element.tsx b/js/hang-ui/src/watch/element.tsx
index 978b307f7..30f76a94a 100644
--- a/js/hang-ui/src/watch/element.tsx
+++ b/js/hang-ui/src/watch/element.tsx
@@ -2,7 +2,7 @@ import type HangWatch from "@moq/hang/watch/element";
import { useContext } from "solid-js";
import { Show } from "solid-js/web";
import { Stats } from "../shared/components/stats";
-import type { ProviderProps } from "../shared/components/stats/types";
+
import BufferingIndicator from "./components/BufferingIndicator";
import WatchControls from "./components/WatchControls";
import WatchUIContextProvider, { WatchUIContext } from "./context";
@@ -21,12 +21,8 @@ export function WatchUI(props: { watch: HangWatch }) {
{
- if (!ctx?.hangWatch) return undefined;
- return {
- audio: { source: ctx.hangWatch.audio.source },
- video: { source: ctx.hangWatch.video.source },
- };
+ getElement={(ctx): HangWatch | undefined => {
+ return ctx?.hangWatch;
}}
/>
diff --git a/js/hang-ui/src/watch/styles/index.css b/js/hang-ui/src/watch/styles/index.css
index cab710734..5ad901831 100644
--- a/js/hang-ui/src/watch/styles/index.css
+++ b/js/hang-ui/src/watch/styles/index.css
@@ -3,6 +3,13 @@
@import "../../shared/components/button/button.css";
@import "../../shared/components/stats/styles/index.css";
+/* Color variables for buffer states */
+:root {
+ --buffer-green: #4ade80;
+ --buffer-yellow: #facc15;
+ --buffer-red: #f87171;
+}
+
.watchVideoContainer {
display: block;
position: relative;
@@ -202,3 +209,123 @@
background: #1a1a1a;
color: #fff;
}
+
+/* Buffer Control styles */
+.bufferControlContainer {
+ display: flex;
+ flex-direction: column;
+ gap: 8px;
+ padding: 8px 12px;
+}
+
+.bufferVisualization {
+ position: relative;
+ width: 100%;
+ height: 52px;
+ background: rgba(255, 255, 255, 0.1);
+ border-radius: 4px;
+ cursor: pointer;
+ user-select: none;
+ padding-left: 48px;
+ box-sizing: border-box;
+}
+
+.bufferPlayhead {
+ position: absolute;
+ left: 48px;
+ top: 0;
+ bottom: 0;
+ width: 2px;
+ background: #fff;
+ z-index: 3;
+}
+
+.bufferTrack {
+ position: absolute;
+ left: 48px;
+ right: 0;
+ height: 20px;
+ display: flex;
+ align-items: center;
+}
+
+.bufferTrack--video {
+ top: 4px;
+}
+
+.bufferTrack--audio {
+ top: 28px;
+}
+
+.bufferTrackLabel {
+ position: absolute;
+ left: -46px;
+ width: 40px;
+ font-size: 10px;
+ color: rgba(255, 255, 255, 0.6);
+ text-align: right;
+ padding-right: 4px;
+ pointer-events: none;
+ box-sizing: border-box;
+}
+
+.bufferRange {
+ position: absolute;
+ top: 2px;
+ height: calc(100% - 4px);
+ border-radius: 2px;
+ min-width: 2px;
+ opacity: 0.85;
+ display: flex;
+ align-items: center;
+ justify-content: flex-end;
+ overflow: hidden;
+}
+
+.bufferOverflowLabel {
+ font-size: 9px;
+ color: rgba(0, 0, 0, 0.7);
+ padding-right: 4px;
+ font-weight: 500;
+}
+
+.bufferTargetArea {
+ position: absolute;
+ top: 0;
+ bottom: 0;
+ left: 48px;
+ right: 0;
+ pointer-events: none;
+}
+
+.bufferTargetLine {
+ position: absolute;
+ top: 0;
+ bottom: 0;
+ width: 3px;
+ background: var(--color-orange-400, #fb923c);
+ z-index: 2;
+ cursor: ew-resize;
+ border-radius: 1px;
+ pointer-events: auto;
+}
+
+.bufferTargetLabel {
+ position: absolute;
+ bottom: 100%;
+ left: 50%;
+ transform: translateX(-50%);
+ font-size: 11px;
+ font-weight: 500;
+ color: var(--color-orange-400, #fb923c);
+ white-space: nowrap;
+ margin-bottom: 2px;
+ opacity: 0;
+ transition: opacity 0.15s ease;
+ pointer-events: none;
+}
+
+.bufferVisualization:hover .bufferTargetLabel,
+.bufferVisualization.dragging .bufferTargetLabel {
+ opacity: 1;
+}
diff --git a/js/hang/package.json b/js/hang/package.json
index c1e162c1b..09a6856df 100644
--- a/js/hang/package.json
+++ b/js/hang/package.json
@@ -31,9 +31,10 @@
},
"dependencies": {
"@kixelated/libavjs-webcodecs-polyfill": "^0.5.5",
+ "@libav.js/variant-opus-af": "^6.8.8",
"@moq/lite": "workspace:^",
"@moq/signals": "workspace:^",
- "@libav.js/variant-opus-af": "^6.8.8",
+ "@svta/cml-iso-bmff": "^1.0.0-alpha.9",
"async-mutex": "^0.5.0",
"comlink": "^4.4.2",
"zod": "^4.1.5"
diff --git a/js/hang/src/catalog/audio.ts b/js/hang/src/catalog/audio.ts
index e57710bf2..2711d9c8a 100644
--- a/js/hang/src/catalog/audio.ts
+++ b/js/hang/src/catalog/audio.ts
@@ -1,5 +1,5 @@
import { z } from "zod";
-import { ContainerSchema, DEFAULT_CONTAINER } from "./container";
+import { ContainerSchema } from "./container";
import { u53Schema } from "./integers";
// Backwards compatibility: old track schema
@@ -14,9 +14,8 @@ export const AudioConfigSchema = z.object({
// See: https://w3c.github.io/webcodecs/codec_registry.html
codec: z.string(),
- // Container format for timestamp encoding
- // Defaults to "legacy" when not specified in catalog (backward compatibility)
- container: ContainerSchema.default(DEFAULT_CONTAINER),
+ // The container format, used to decode the timestamp and more.
+ container: ContainerSchema,
// The description is used for some codecs.
// If provided, we can initialize the decoder based on the catalog alone.
@@ -32,6 +31,11 @@ export const AudioConfigSchema = z.object({
// The bitrate of the audio in bits per second
// TODO: Support up to Number.MAX_SAFE_INTEGER
bitrate: u53Schema.optional(),
+
+ // Minimum buffer size in milliseconds required for smooth playback.
+ // This represents the minimum time the player should buffer before starting playback.
+ // The player should add additional buffer on top of this value.
+ minBuffer: u53Schema.optional(),
});
export const AudioSchema = z
diff --git a/js/hang/src/catalog/container.ts b/js/hang/src/catalog/container.ts
index 05b0e81db..3d289f318 100644
--- a/js/hang/src/catalog/container.ts
+++ b/js/hang/src/catalog/container.ts
@@ -1,18 +1,27 @@
import { z } from "zod";
+import { u53Schema } from "./integers";
/**
- * Container format for frame timestamp encoding.
+ * Container format for frame timestamp encoding and frame payload structure.
*
- * - "legacy": Uses QUIC VarInt encoding (1-8 bytes, variable length)
- * - "raw": Uses fixed u64 encoding (8 bytes, big-endian)
- * - "fmp4": Fragmented MP4 container (future)
+ * - "legacy": Uses QUIC VarInt encoding (1-8 bytes, variable length), raw frame payloads.
+ * Timestamps are in microseconds.
+ * - "cmaf": Fragmented MP4 container - frames contain complete moof+mdat fragments.
+ * Timestamps are in timescale units.
*/
-export const ContainerSchema = z.enum(["legacy", "raw", "fmp4"]);
+export const ContainerSchema = z
+ .discriminatedUnion("kind", [
+ // The default hang container
+ z.object({ kind: z.literal("legacy") }),
+ // CMAF container with timescale for timestamp conversion
+ z.object({
+ kind: z.literal("cmaf"),
+ // Time units per second
+ timescale: u53Schema,
+ // Track ID used in the moof/mdat fragments
+ trackId: u53Schema,
+ }),
+ ])
+ .default({ kind: "legacy" });
export type Container = z.infer;
-
-/**
- * Default container format when not specified.
- * Set to legacy for backward compatibility.
- */
-export const DEFAULT_CONTAINER: Container = "legacy";
diff --git a/js/hang/src/catalog/index.ts b/js/hang/src/catalog/index.ts
index cfe27be11..ecf4e9daf 100644
--- a/js/hang/src/catalog/index.ts
+++ b/js/hang/src/catalog/index.ts
@@ -2,10 +2,10 @@ export * from "./audio";
export * from "./capabilities";
export * from "./chat";
export * from "./container";
-export { DEFAULT_CONTAINER } from "./container";
export * from "./integers";
export * from "./location";
export * from "./preview";
+export * from "./priority";
export * from "./root";
export * from "./track";
export * from "./user";
diff --git a/js/hang/src/publish/priority.ts b/js/hang/src/catalog/priority.ts
similarity index 100%
rename from js/hang/src/publish/priority.ts
rename to js/hang/src/catalog/priority.ts
diff --git a/js/hang/src/catalog/video.ts b/js/hang/src/catalog/video.ts
index b8b77883f..28b942d58 100644
--- a/js/hang/src/catalog/video.ts
+++ b/js/hang/src/catalog/video.ts
@@ -1,5 +1,5 @@
import { z } from "zod";
-import { ContainerSchema, DEFAULT_CONTAINER } from "./container";
+import { ContainerSchema } from "./container";
import { u53Schema } from "./integers";
// Backwards compatibility: old track schema
@@ -13,9 +13,8 @@ export const VideoConfigSchema = z.object({
// See: https://w3c.github.io/webcodecs/codec_registry.html
codec: z.string(),
- // Container format for timestamp encoding
- // Defaults to "legacy" when not specified in catalog (backward compatibility)
- container: ContainerSchema.default(DEFAULT_CONTAINER),
+ // The container format, used to decode the timestamp and more.
+ container: ContainerSchema,
// The description is used for some codecs.
// If provided, we can initialize the decoder based on the catalog alone.
@@ -43,6 +42,11 @@ export const VideoConfigSchema = z.object({
// If true, the decoder will optimize for latency.
// Default: true
optimizeForLatency: z.boolean().optional(),
+
+ // Minimum buffer size in milliseconds required for smooth playback.
+ // This represents the minimum time the player should buffer before starting playback.
+ // The player should add additional buffer on top of this value.
+ minBuffer: u53Schema.optional(),
});
// Mirrors VideoDecoderConfig
diff --git a/js/hang/src/container/codec.ts b/js/hang/src/container/codec.ts
deleted file mode 100644
index ca60754a4..000000000
--- a/js/hang/src/container/codec.ts
+++ /dev/null
@@ -1,155 +0,0 @@
-import type { Time } from "@moq/lite";
-import type * as Catalog from "../catalog";
-import { DEFAULT_CONTAINER } from "../catalog";
-
-/**
- * Encodes a timestamp according to the specified container format.
- *
- * @param timestamp - The timestamp in microseconds
- * @param container - The container format to use
- * @returns The encoded timestamp as a Uint8Array
- */
-export function encodeTimestamp(timestamp: Time.Micro, container: Catalog.Container = DEFAULT_CONTAINER): Uint8Array {
- switch (container) {
- case "legacy":
- return encodeVarInt(timestamp);
- case "raw":
- return encodeU64(timestamp);
- case "fmp4":
- throw new Error("fmp4 container not yet implemented");
- }
-}
-
-/**
- * Decodes a timestamp from a buffer according to the specified container format.
- *
- * @param buffer - The buffer containing the encoded timestamp
- * @param container - The container format to use
- * @returns [timestamp in microseconds, remaining buffer after timestamp]
- */
-export function decodeTimestamp(
- buffer: Uint8Array,
- container: Catalog.Container = DEFAULT_CONTAINER,
-): [Time.Micro, Uint8Array] {
- switch (container) {
- case "legacy": {
- const [value, remaining] = decodeVarInt(buffer);
- return [value as Time.Micro, remaining];
- }
- case "raw": {
- const [value, remaining] = decodeU64(buffer);
- return [value as Time.Micro, remaining];
- }
- case "fmp4":
- throw new Error("fmp4 container not yet implemented");
- }
-}
-
-/**
- * Gets the size in bytes of an encoded timestamp for the given container format.
- * For variable-length formats, returns the maximum size.
- *
- * @param container - The container format
- * @returns Size in bytes
- */
-export function getTimestampSize(container: Catalog.Container = DEFAULT_CONTAINER): number {
- switch (container) {
- case "legacy":
- return 8; // VarInt maximum size
- case "raw":
- return 8; // u64 fixed size
- case "fmp4":
- throw new Error("fmp4 container not yet implemented");
- }
-}
-
-// ============================================================================
-// LEGACY VARINT IMPLEMENTATION
-// ============================================================================
-
-const MAX_U6 = 2 ** 6 - 1;
-const MAX_U14 = 2 ** 14 - 1;
-const MAX_U30 = 2 ** 30 - 1;
-const MAX_U53 = Number.MAX_SAFE_INTEGER;
-
-function decodeVarInt(buf: Uint8Array): [number, Uint8Array] {
- const size = 1 << ((buf[0] & 0xc0) >> 6);
-
- const view = new DataView(buf.buffer, buf.byteOffset, size);
- const remain = new Uint8Array(buf.buffer, buf.byteOffset + size, buf.byteLength - size);
- let v: number;
-
- if (size === 1) {
- v = buf[0] & 0x3f;
- } else if (size === 2) {
- v = view.getUint16(0) & 0x3fff;
- } else if (size === 4) {
- v = view.getUint32(0) & 0x3fffffff;
- } else if (size === 8) {
- // NOTE: Precision loss above 2^52
- v = Number(view.getBigUint64(0) & 0x3fffffffffffffffn);
- } else {
- throw new Error("impossible");
- }
-
- return [v, remain];
-}
-
-function encodeVarInt(v: number): Uint8Array {
- const dst = new Uint8Array(8);
-
- if (v <= MAX_U6) {
- dst[0] = v;
- return new Uint8Array(dst.buffer, dst.byteOffset, 1);
- }
-
- if (v <= MAX_U14) {
- const view = new DataView(dst.buffer, dst.byteOffset, 2);
- view.setUint16(0, v | 0x4000);
- return new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
- }
-
- if (v <= MAX_U30) {
- const view = new DataView(dst.buffer, dst.byteOffset, 4);
- view.setUint32(0, v | 0x80000000);
- return new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
- }
-
- if (v <= MAX_U53) {
- const view = new DataView(dst.buffer, dst.byteOffset, 8);
- view.setBigUint64(0, BigInt(v) | 0xc000000000000000n);
- return new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
- }
-
- throw new Error(`overflow, value larger than 53-bits: ${v}`);
-}
-
-// ============================================================================
-// RAW U64 IMPLEMENTATION
-// ============================================================================
-
-/**
- * Decodes a fixed 8-byte big-endian unsigned 64-bit integer.
- */
-function decodeU64(buf: Uint8Array): [number, Uint8Array] {
- if (buf.byteLength < 8) {
- throw new Error("Buffer too short for u64 decode");
- }
-
- const view = new DataView(buf.buffer, buf.byteOffset, 8);
- const value = Number(view.getBigUint64(0));
- const remain = new Uint8Array(buf.buffer, buf.byteOffset + 8, buf.byteLength - 8);
-
- return [value, remain];
-}
-
-/**
- * Encodes a number as a fixed 8-byte big-endian unsigned 64-bit integer.
- * Much simpler than VarInt!
- */
-function encodeU64(v: number): Uint8Array {
- const dst = new Uint8Array(8);
- const view = new DataView(dst.buffer, dst.byteOffset, 8);
- view.setBigUint64(0, BigInt(v));
- return dst;
-}
diff --git a/js/hang/src/container/index.ts b/js/hang/src/container/index.ts
deleted file mode 100644
index 2e87cf323..000000000
--- a/js/hang/src/container/index.ts
+++ /dev/null
@@ -1 +0,0 @@
-export * from "./codec";
diff --git a/js/hang/src/frame.ts b/js/hang/src/frame.ts
index a0fd86219..7e21d2278 100644
--- a/js/hang/src/frame.ts
+++ b/js/hang/src/frame.ts
@@ -1,8 +1,6 @@
import type * as Moq from "@moq/lite";
import { Time } from "@moq/lite";
import { Effect, Signal } from "@moq/signals";
-import type * as Catalog from "./catalog";
-import * as Container from "./container";
export interface Source {
byteLength: number;
@@ -16,9 +14,9 @@ export interface Frame {
group: number;
}
-export function encode(source: Uint8Array | Source, timestamp: Time.Micro, container?: Catalog.Container): Uint8Array {
+export function encode(source: Uint8Array | Source, timestamp: Time.Micro): Uint8Array {
// Encode timestamp using the specified container format
- const timestampBytes = Container.encodeTimestamp(timestamp, container);
+ const timestampBytes = encodeTimestamp(timestamp);
// Allocate buffer for timestamp + payload
const payloadSize = source instanceof Uint8Array ? source.byteLength : source.byteLength;
@@ -38,20 +36,18 @@ export function encode(source: Uint8Array | Source, timestamp: Time.Micro, conta
}
// NOTE: A keyframe is always the first frame in a group, so it's not encoded on the wire.
-export function decode(buffer: Uint8Array, container?: Catalog.Container): { data: Uint8Array; timestamp: Time.Micro } {
+export function decode(buffer: Uint8Array): { data: Uint8Array; timestamp: Time.Micro } {
// Decode timestamp using the specified container format
- const [timestamp, data] = Container.decodeTimestamp(buffer, container);
+ const [timestamp, data] = decodeTimestamp(buffer);
return { timestamp: timestamp as Time.Micro, data };
}
export class Producer {
#track: Moq.Track;
#group?: Moq.Group;
- #container?: Catalog.Container;
- constructor(track: Moq.Track, container?: Catalog.Container) {
+ constructor(track: Moq.Track) {
this.#track = track;
- this.#container = container;
}
encode(data: Uint8Array | Source, timestamp: Time.Micro, keyframe: boolean) {
@@ -62,7 +58,7 @@ export class Producer {
throw new Error("must start with a keyframe");
}
- this.#group?.writeFrame(encode(data, timestamp, this.#container));
+ this.#group?.writeFrame(encode(data, timestamp));
}
close() {
@@ -74,7 +70,6 @@ export class Producer {
export interface ConsumerProps {
// Target latency in milliseconds (default: 0)
latency?: Signal | Time.Milli;
- container?: Catalog.Container;
}
interface Group {
@@ -86,7 +81,6 @@ interface Group {
export class Consumer {
#track: Moq.Track;
#latency: Signal;
- #container?: Catalog.Container;
#groups: Group[] = [];
#active?: number; // the active group sequence number
@@ -98,7 +92,6 @@ export class Consumer {
constructor(track: Moq.Track, props?: ConsumerProps) {
this.#track = track;
this.#latency = Signal.from(props?.latency ?? Time.Milli.zero);
- this.#container = props?.container;
this.#signals.spawn(this.#run.bind(this));
this.#signals.cleanup(() => {
@@ -152,7 +145,7 @@ export class Consumer {
const next = await group.consumer.readFrame();
if (!next) break;
- const { data, timestamp } = decode(next, this.#container);
+ const { data, timestamp } = decode(next);
const frame = {
data,
timestamp,
@@ -282,3 +275,86 @@ export class Consumer {
this.#groups.length = 0;
}
}
+
+/**
+ * Encodes a varint timestamp
+ *
+ * @param timestamp - The timestamp in microseconds
+ * @returns The encoded timestamp as a Uint8Array
+ */
+function encodeTimestamp(timestamp: Time.Micro): Uint8Array {
+ return encodeVarInt(timestamp);
+}
+
+/**
+ * Decodes a timestamp from a buffer according to the specified container format.
+ *
+ * @param buffer - The buffer containing the encoded timestamp
+ * @param container - The container format to use
+ * @returns [timestamp in microseconds, remaining buffer after timestamp]
+ */
+function decodeTimestamp(buffer: Uint8Array): [Time.Micro, Uint8Array] {
+ const [value, remaining] = decodeVarInt(buffer);
+ return [value as Time.Micro, remaining];
+}
+
+// ============================================================================
+// LEGACY VARINT IMPLEMENTATION
+// ============================================================================
+
+const MAX_U6 = 2 ** 6 - 1;
+const MAX_U14 = 2 ** 14 - 1;
+const MAX_U30 = 2 ** 30 - 1;
+const MAX_U53 = Number.MAX_SAFE_INTEGER;
+
+function decodeVarInt(buf: Uint8Array): [number, Uint8Array] {
+ const size = 1 << ((buf[0] & 0xc0) >> 6);
+
+ const view = new DataView(buf.buffer, buf.byteOffset, size);
+ const remain = new Uint8Array(buf.buffer, buf.byteOffset + size, buf.byteLength - size);
+ let v: number;
+
+ if (size === 1) {
+ v = buf[0] & 0x3f;
+ } else if (size === 2) {
+ v = view.getUint16(0) & 0x3fff;
+ } else if (size === 4) {
+ v = view.getUint32(0) & 0x3fffffff;
+ } else if (size === 8) {
+ // NOTE: Precision loss above 2^52
+ v = Number(view.getBigUint64(0) & 0x3fffffffffffffffn);
+ } else {
+ throw new Error("impossible");
+ }
+
+ return [v, remain];
+}
+
+function encodeVarInt(v: number): Uint8Array {
+ const dst = new Uint8Array(8);
+
+ if (v <= MAX_U6) {
+ dst[0] = v;
+ return new Uint8Array(dst.buffer, dst.byteOffset, 1);
+ }
+
+ if (v <= MAX_U14) {
+ const view = new DataView(dst.buffer, dst.byteOffset, 2);
+ view.setUint16(0, v | 0x4000);
+ return new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
+ }
+
+ if (v <= MAX_U30) {
+ const view = new DataView(dst.buffer, dst.byteOffset, 4);
+ view.setUint32(0, v | 0x80000000);
+ return new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
+ }
+
+ if (v <= MAX_U53) {
+ const view = new DataView(dst.buffer, dst.byteOffset, 8);
+ view.setBigUint64(0, BigInt(v) | 0xc000000000000000n);
+ return new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
+ }
+
+ throw new Error(`overflow, value larger than 53-bits: ${v}`);
+}
diff --git a/js/hang/src/meet/element.ts b/js/hang/src/meet/element.ts
index b1572fecc..68d193bd0 100644
--- a/js/hang/src/meet/element.ts
+++ b/js/hang/src/meet/element.ts
@@ -180,8 +180,11 @@ export default class HangMeet extends HTMLElement {
},
});
- const renderer = new Watch.Video.Renderer(broadcast.video, { canvas });
- const emitter = new Watch.Audio.Emitter(broadcast.audio);
+ const videoSource = new Watch.Video.Source({ broadcast });
+ const audioSource = new Watch.Audio.Source({ broadcast });
+
+ const renderer = new Watch.Video.Renderer(videoSource, { canvas });
+ const emitter = new Watch.Audio.Emitter(audioSource);
this.#remotes.set(name, { canvas, renderer, emitter });
diff --git a/js/hang/src/mp4/decode.ts b/js/hang/src/mp4/decode.ts
new file mode 100644
index 000000000..25c896db4
--- /dev/null
+++ b/js/hang/src/mp4/decode.ts
@@ -0,0 +1,327 @@
+/**
+ * MP4 decoding utilities for parsing fMP4 init and data segments.
+ * Used by WebCodecs to extract raw frames from CMAF container.
+ */
+
+import {
+ type MediaHeaderBox,
+ type ParsedIsoBox,
+ readAvc1,
+ readHev1,
+ readHvc1,
+ readIsoBoxes,
+ readMdat,
+ readMdhd,
+ readMfhd,
+ readMp4a,
+ readStsd,
+ readTfdt,
+ readTfhd,
+ readTkhd,
+ readTrun,
+ type SampleDescriptionBox,
+ type TrackFragmentBaseMediaDecodeTimeBox,
+ type TrackFragmentHeaderBox,
+ type TrackRunBox,
+ type TrackRunSample,
+} from "@svta/cml-iso-bmff";
+
+// Configure readers for specific box types we need to parse
+const INIT_READERS = {
+ avc1: readAvc1,
+ avc3: readAvc1, // avc3 has same structure
+ hvc1: readHvc1,
+ hev1: readHev1,
+ mp4a: readMp4a,
+ stsd: readStsd,
+ mdhd: readMdhd,
+ tkhd: readTkhd,
+};
+
+const DATA_READERS = {
+ mfhd: readMfhd,
+ tfhd: readTfhd,
+ tfdt: readTfdt,
+ trun: readTrun,
+ mdat: readMdat,
+};
+
+/**
+ * Recursively find a box by type in the box tree.
+ * This is more reliable than the library's findIsoBox which may not traverse all children.
+ */
+function findBox(
+ boxes: ParsedIsoBox[],
+ predicate: (box: ParsedIsoBox) => box is T,
+): T | undefined {
+ for (const box of boxes) {
+ if (predicate(box)) {
+ return box;
+ }
+ // Recursively search children - boxes may have a 'boxes' property with children
+ // biome-ignore lint/suspicious/noExplicitAny: ISO box structure varies
+ const children = (box as any).boxes;
+ if (children && Array.isArray(children)) {
+ const found = findBox(children, predicate);
+ if (found) return found;
+ }
+ }
+ return undefined;
+}
+
+/**
+ * Result of parsing an init segment.
+ */
+export interface InitSegment {
+ /** Codec-specific description (avcC, hvcC, esds, dOps, etc.) */
+ description?: Uint8Array;
+ /** Time units per second */
+ timescale: number;
+ /** Track ID from the init segment */
+ trackId: number;
+}
+
+/**
+ * A decoded sample from a data segment.
+ */
+export interface Sample {
+ /** Raw sample data */
+ data: Uint8Array;
+ /** Timestamp in microseconds */
+ timestamp: number;
+ /** Whether this is a keyframe (sync sample) */
+ keyframe: boolean;
+}
+
+// Helper to convert Uint8Array to ArrayBuffer for the library
+function toArrayBuffer(data: Uint8Array): ArrayBuffer {
+ // Create a new ArrayBuffer and copy data to avoid SharedArrayBuffer issues
+ const buffer = new ArrayBuffer(data.byteLength);
+ new Uint8Array(buffer).set(data);
+ return buffer;
+}
+
+// Type guard for finding boxes by type
+function isBoxType(type: string) {
+ return (box: ParsedIsoBox): box is T => box.type === type;
+}
+
+/**
+ * Parse an init segment (ftyp + moov) to extract codec description and timescale.
+ *
+ * @param init - The init segment data
+ * @returns Parsed init segment information
+ */
+export function decodeInitSegment(init: Uint8Array): InitSegment {
+ // Cast to ParsedIsoBox[] since the library's return type changes with readers
+ const boxes = readIsoBoxes(toArrayBuffer(init), { readers: INIT_READERS }) as ParsedIsoBox[];
+
+ // Find moov > trak > mdia > mdhd for timescale
+ const mdhd = findBox(boxes, isBoxType("mdhd"));
+ if (!mdhd) {
+ throw new Error("No mdhd box found in init segment");
+ }
+
+ // Find moov > trak > tkhd for track ID
+ // biome-ignore lint/suspicious/noExplicitAny: ISO box traversal
+ const tkhd = findBox(boxes, isBoxType("tkhd"));
+ const trackId = tkhd?.trackId ?? 1;
+
+ // Find moov > trak > mdia > minf > stbl > stsd for sample description
+ const stsd = findBox(boxes, isBoxType("stsd"));
+ if (!stsd || !stsd.entries || stsd.entries.length === 0) {
+ throw new Error("No stsd box found in init segment");
+ }
+
+ // Extract codec-specific description from the first sample entry
+ const entry = stsd.entries[0];
+ const description = extractDescription(entry);
+
+ return {
+ description,
+ timescale: mdhd.timescale,
+ trackId,
+ };
+}
+
+/**
+ * Extract codec-specific description from a sample entry.
+ * The description is codec-specific: avcC for H.264, hvcC for H.265, esds for AAC, dOps for Opus.
+ */
+// biome-ignore lint/suspicious/noExplicitAny: ISO box types vary
+function extractDescription(entry: any): Uint8Array | undefined {
+ if (!entry.boxes || !Array.isArray(entry.boxes)) {
+ return undefined;
+ }
+
+ // Look for codec config boxes in the sample entry
+ for (const box of entry.boxes) {
+ // Handle raw Uint8Array boxes (already serialized)
+ if (box instanceof Uint8Array) {
+ // Extract the payload without the box header (8 bytes: 4 size + 4 type)
+ if (box.length > 8) {
+ // Check if this looks like a codec config box by reading the type
+ const typeBytes = String.fromCharCode(box[4], box[5], box[6], box[7]);
+ if (typeBytes === "avcC" || typeBytes === "hvcC" || typeBytes === "esds" || typeBytes === "dOps") {
+ return new Uint8Array(box.slice(8));
+ }
+ }
+ continue;
+ }
+
+ // Check for known codec config box types
+ const boxType = box.type;
+ if (boxType === "avcC" || boxType === "hvcC" || boxType === "esds" || boxType === "dOps") {
+ // The library stores parsed boxes with a 'view' property containing IsoBoxReadView
+ // which has access to the raw buffer. Extract the box payload (without header).
+ if (box.view) {
+ const view = box.view;
+ // IsoBoxReadView has buffer, byteOffset, and byteLength properties
+ // The box payload starts after the 8-byte header (size + type)
+ const headerSize = 8;
+ const payloadOffset = view.byteOffset + headerSize;
+ const payloadLength = box.size - headerSize;
+ return new Uint8Array(view.buffer, payloadOffset, payloadLength);
+ }
+ // Fallback: try data or raw properties
+ if (box.data instanceof Uint8Array) {
+ return new Uint8Array(box.data);
+ }
+ if (box.raw instanceof Uint8Array) {
+ return new Uint8Array(box.raw.slice(8));
+ }
+ }
+ }
+
+ return undefined;
+}
+
+/**
+ * Parse a data segment (moof + mdat) to extract raw samples.
+ *
+ * @param segment - The moof + mdat data
+ * @param timescale - Time units per second (from init segment)
+ * @returns Array of decoded samples
+ */
+export function decodeDataSegment(segment: Uint8Array, timescale: number): Sample[] {
+ // Cast to ParsedIsoBox[] since the library's return type changes with readers
+ const boxes = readIsoBoxes(toArrayBuffer(segment), { readers: DATA_READERS }) as ParsedIsoBox[];
+
+ // Find moof > traf > tfdt for base media decode time
+ const tfdt = findBox(boxes, isBoxType("tfdt"));
+ const baseDecodeTime = tfdt?.baseMediaDecodeTime ?? 0;
+
+ // Find moof > traf > tfhd for default sample values
+ const tfhd = findBox(boxes, isBoxType("tfhd"));
+ const defaultDuration = tfhd?.defaultSampleDuration ?? 0;
+ const defaultSize = tfhd?.defaultSampleSize ?? 0;
+ const defaultFlags = tfhd?.defaultSampleFlags ?? 0;
+
+ // Find moof > traf > trun for sample info
+ const trun = findBox(boxes, isBoxType("trun"));
+ if (!trun) {
+ throw new Error("No trun box found in data segment");
+ }
+
+ // Find mdat for sample data
+ // biome-ignore lint/suspicious/noExplicitAny: mdat box type
+ const mdat = findBox(boxes, isBoxType("mdat"));
+ if (!mdat) {
+ throw new Error("No mdat box found in data segment");
+ }
+
+ // mdat.data contains the raw sample data
+ const mdatData = mdat.data as Uint8Array;
+ if (!mdatData) {
+ throw new Error("No data in mdat box");
+ }
+
+ const samples: Sample[] = [];
+ let dataOffset = 0;
+ let decodeTime = baseDecodeTime;
+
+ for (let i = 0; i < trun.sampleCount; i++) {
+ const sample: TrackRunSample = trun.samples[i] ?? {};
+
+ const sampleSize = sample.sampleSize ?? defaultSize;
+ const sampleDuration = sample.sampleDuration ?? defaultDuration;
+ const sampleFlags =
+ i === 0 && trun.firstSampleFlags !== undefined
+ ? trun.firstSampleFlags
+ : (sample.sampleFlags ?? defaultFlags);
+ const compositionOffset = sample.sampleCompositionTimeOffset ?? 0;
+
+ // Extract sample data
+ const data = new Uint8Array(mdatData.slice(dataOffset, dataOffset + sampleSize));
+ dataOffset += sampleSize;
+
+ // Calculate presentation timestamp in microseconds
+ // PTS = (decode_time + composition_offset) * 1_000_000 / timescale
+ const pts = decodeTime + compositionOffset;
+ const timestamp = Math.round((pts * 1_000_000) / timescale);
+
+ // Check if keyframe (sample_is_non_sync_sample flag is bit 16)
+ // If flag is 0, treat as keyframe for safety
+ const keyframe = sampleFlags === 0 || (sampleFlags & 0x00010000) === 0;
+
+ samples.push({
+ data,
+ timestamp,
+ keyframe,
+ });
+
+ decodeTime += sampleDuration;
+ }
+
+ return samples;
+}
+
+/**
+ * Rewrite the track_id in a moof+mdat segment to 1.
+ * This allows us to always generate init segments with track_id=1.
+ *
+ * Modifies the buffer in place and returns it.
+ */
+export function rewriteTrackId(segment: Uint8Array): Uint8Array {
+ const tfhdOffset = findBoxOffset(segment, 0, segment.length, "tfhd");
+ if (tfhdOffset === -1) {
+ return segment; // No tfhd found, return as-is
+ }
+
+ // tfhd structure: size(4) + type(4) + version/flags(4) + track_id(4)
+ // track_id is at offset 12 within the box
+ const view = new DataView(segment.buffer, segment.byteOffset + tfhdOffset);
+ view.setUint32(12, 1, false); // Set track_id to 1 (big-endian)
+
+ return segment;
+}
+
+/**
+ * Find the byte offset of a box by type within a range.
+ * Recursively searches container boxes (moof, traf, etc.)
+ */
+function findBoxOffset(data: Uint8Array, start: number, end: number, targetType: string): number {
+ const view = new DataView(data.buffer, data.byteOffset);
+ let offset = start;
+
+ while (offset + 8 <= end) {
+ const size = view.getUint32(offset, false);
+ if (size < 8 || offset + size > end) break;
+
+ const type = String.fromCharCode(data[offset + 4], data[offset + 5], data[offset + 6], data[offset + 7]);
+
+ if (type === targetType) {
+ return offset;
+ }
+
+ // Recursively search container boxes
+ if (type === "moof" || type === "traf" || type === "mdia" || type === "minf" || type === "stbl") {
+ const found = findBoxOffset(data, offset + 8, offset + size, targetType);
+ if (found !== -1) return found;
+ }
+
+ offset += size;
+ }
+
+ return -1;
+}
diff --git a/js/hang/src/mp4/encode.ts b/js/hang/src/mp4/encode.ts
new file mode 100644
index 000000000..186ce3b65
--- /dev/null
+++ b/js/hang/src/mp4/encode.ts
@@ -0,0 +1,1062 @@
+/**
+ * MP4 encoding utilities for creating fMP4 init and data segments.
+ * Used by MSE to create init segments (ftyp+moov) and data segments (moof+mdat).
+ */
+
+import {
+ type DataEntryUrlBox,
+ type DataInformationBox,
+ type DataReferenceBox,
+ type DecodingTimeToSampleBox,
+ type FileTypeBox,
+ type HandlerReferenceBox,
+ type IsoBoxStreamable,
+ type IsoBoxWriterMap,
+ type MediaBox,
+ type MediaDataBox,
+ type MediaHeaderBox,
+ type MediaInformationBox,
+ type MovieBox,
+ type MovieExtendsBox,
+ type MovieFragmentBox,
+ type MovieFragmentHeaderBox,
+ type MovieHeaderBox,
+ type SampleDescriptionBox,
+ type SampleTableBox,
+ type SoundMediaHeaderBox,
+ type TrackBox,
+ type TrackExtendsBox,
+ type TrackFragmentBaseMediaDecodeTimeBox,
+ type TrackFragmentBox,
+ type TrackFragmentHeaderBox,
+ type TrackHeaderBox,
+ type TrackRunBox,
+ type VideoMediaHeaderBox,
+ writeDref,
+ writeFtyp,
+ writeHdlr,
+ writeIsoBoxes,
+ writeMdat,
+ writeMdhd,
+ writeMfhd,
+ writeMvhd,
+ writeSmhd,
+ writeStsd,
+ writeStts,
+ writeTfdt,
+ writeTfhd,
+ writeTkhd,
+ writeTrex,
+ writeTrun,
+ writeUrl,
+ writeVmhd,
+} from "@svta/cml-iso-bmff";
+
+import type * as Catalog from "../catalog";
+import * as Hex from "../util/hex";
+
+// Identity matrix for tkhd/mvhd (stored as 16.16 fixed point)
+const IDENTITY_MATRIX = [0x00010000, 0, 0, 0, 0x00010000, 0, 0, 0, 0x40000000];
+
+// Writers config - maps box types to their writer functions
+const WRITERS: IsoBoxWriterMap = {
+ // Init segment boxes
+ ftyp: writeFtyp,
+ mvhd: writeMvhd,
+ tkhd: writeTkhd,
+ mdhd: writeMdhd,
+ hdlr: writeHdlr,
+ vmhd: writeVmhd,
+ smhd: writeSmhd,
+ "url ": writeUrl,
+ dref: writeDref,
+ stsd: writeStsd,
+ stts: writeStts,
+ trex: writeTrex,
+ // Data segment boxes
+ mfhd: writeMfhd,
+ tfhd: writeTfhd,
+ tfdt: writeTfdt,
+ trun: writeTrun,
+ mdat: writeMdat,
+ // For boxes without library writers, we create them manually as Uint8Arrays
+};
+
+/** Write boxes using our WRITERS config. */
+function writeBoxes(boxes: Iterable): Uint8Array[] {
+ return writeIsoBoxes(boxes, { writers: WRITERS });
+}
+
+/**
+ * Helper to create a simple full box (version + flags + content) as raw bytes.
+ * Used for boxes that don't have writers in the library.
+ */
+function createFullBox(type: string, version: number, flags: number, content: Uint8Array): Uint8Array {
+ const size = 8 + 4 + content.length; // header + version/flags + content
+ const box = new Uint8Array(size);
+ const view = new DataView(box.buffer);
+
+ view.setUint32(0, size, false);
+ box[4] = type.charCodeAt(0);
+ box[5] = type.charCodeAt(1);
+ box[6] = type.charCodeAt(2);
+ box[7] = type.charCodeAt(3);
+ view.setUint32(8, (version << 24) | flags, false);
+ box.set(content, 12);
+
+ return box;
+}
+
+/**
+ * Create an empty stsc (Sample to Chunk) box for fragmented MP4.
+ */
+function createEmptyStsc(): Uint8Array {
+ // stsc: version(1) + flags(3) + entry_count(4) = 4 bytes content
+ const content = new Uint8Array(4); // entry_count = 0
+ return createFullBox("stsc", 0, 0, content);
+}
+
+/**
+ * Create an empty stsz (Sample Size) box for fragmented MP4.
+ */
+function createEmptyStsz(): Uint8Array {
+ // stsz: version(1) + flags(3) + sample_size(4) + sample_count(4) = 8 bytes content
+ const content = new Uint8Array(8); // sample_size = 0, sample_count = 0
+ return createFullBox("stsz", 0, 0, content);
+}
+
+/**
+ * Create an empty stco (Chunk Offset) box for fragmented MP4.
+ */
+function createEmptyStco(): Uint8Array {
+ // stco: version(1) + flags(3) + entry_count(4) = 4 bytes content
+ const content = new Uint8Array(4); // entry_count = 0
+ return createFullBox("stco", 0, 0, content);
+}
+
+/**
+ * Create an avc1 (H.264 Visual Sample Entry) box with embedded avcC.
+ * Built manually because the library doesn't properly serialize Uint8Array child boxes.
+ */
+function createAvc1Box(width: number, height: number, avcC: Uint8Array): Uint8Array {
+ // avc1 box structure:
+ // - 6 bytes reserved (0)
+ // - 2 bytes data_reference_index (1)
+ // - 2 bytes pre_defined (0)
+ // - 2 bytes reserved (0)
+ // - 12 bytes pre_defined (0)
+ // - 2 bytes width
+ // - 2 bytes height
+ // - 4 bytes horizresolution (0x00480000 = 72 dpi)
+ // - 4 bytes vertresolution (0x00480000 = 72 dpi)
+ // - 4 bytes reserved (0)
+ // - 2 bytes frame_count (1)
+ // - 32 bytes compressorname (null-padded string)
+ // - 2 bytes depth (0x0018 = 24)
+ // - 2 bytes pre_defined (-1 = 0xFFFF)
+ // - child boxes (avcC)
+
+ const avcCSize = 8 + avcC.length; // box header + payload
+ const avc1ContentSize = 6 + 2 + 2 + 2 + 12 + 2 + 2 + 4 + 4 + 4 + 2 + 32 + 2 + 2 + avcCSize;
+ const avc1Size = 8 + avc1ContentSize; // box header + content
+
+ const box = new Uint8Array(avc1Size);
+ const view = new DataView(box.buffer);
+ let offset = 0;
+
+ // Box header
+ view.setUint32(offset, avc1Size, false);
+ offset += 4;
+ box[offset++] = 0x61; // 'a'
+ box[offset++] = 0x76; // 'v'
+ box[offset++] = 0x63; // 'c'
+ box[offset++] = 0x31; // '1'
+
+ // SampleEntry fields
+ offset += 6; // reserved (6 bytes of 0)
+ view.setUint16(offset, 1, false);
+ offset += 2; // data_reference_index = 1
+
+ // VisualSampleEntry fields
+ view.setUint16(offset, 0, false);
+ offset += 2; // pre_defined
+ view.setUint16(offset, 0, false);
+ offset += 2; // reserved
+ offset += 12; // pre_defined (12 bytes of 0)
+ view.setUint16(offset, width, false);
+ offset += 2;
+ view.setUint16(offset, height, false);
+ offset += 2;
+ view.setUint32(offset, 0x00480000, false);
+ offset += 4; // horizresolution (72 dpi)
+ view.setUint32(offset, 0x00480000, false);
+ offset += 4; // vertresolution (72 dpi)
+ view.setUint32(offset, 0, false);
+ offset += 4; // reserved
+ view.setUint16(offset, 1, false);
+ offset += 2; // frame_count = 1
+ offset += 32; // compressorname (32 bytes of 0)
+ view.setUint16(offset, 0x0018, false);
+ offset += 2; // depth = 24
+ view.setUint16(offset, 0xffff, false);
+ offset += 2; // pre_defined = -1
+
+ // avcC child box
+ view.setUint32(offset, avcCSize, false);
+ offset += 4;
+ box[offset++] = 0x61; // 'a'
+ box[offset++] = 0x76; // 'v'
+ box[offset++] = 0x63; // 'c'
+ box[offset++] = 0x43; // 'C'
+ box.set(avcC, offset);
+
+ return box;
+}
+
+/**
+ * Creates an MSE-compatible initialization segment (ftyp + moov) for H.264 video.
+ *
+ * @example
+ * ```ts
+ * // From WebCodecs EncodedVideoChunkMetadata
+ * const config = await encoder.encode(frame);
+ * const metadata = config.decoderConfig;
+ *
+ * const initSegment = createVideoInitSegment({
+ * width: metadata.codedWidth,
+ * height: metadata.codedHeight,
+ * avcC: new Uint8Array(metadata.description),
+ * });
+ *
+ * sourceBuffer.appendBuffer(initSegment);
+ * ```
+ */
+export function createVideoInitSegment(config: Catalog.VideoConfig): Uint8Array {
+ const { codedWidth, codedHeight, description, container } = config;
+ if (!codedWidth || !codedHeight || !description) {
+ // TODO: We could
+ throw new Error("Missing required fields to create video init segment");
+ }
+
+ // Use timescale from CMAF container, or microseconds for legacy
+ const timescale = container.kind === "cmaf" ? container.timescale : 1_000_000;
+
+ // Use track_id from CMAF container, or default to 1 for legacy
+ const trackId = container.kind === "cmaf" ? container.trackId : 1;
+
+ // ftyp - File Type Box
+ const ftyp: FileTypeBox = {
+ type: "ftyp",
+ majorBrand: "isom",
+ minorVersion: 0x200,
+ compatibleBrands: ["isom", "iso6", "mp41"],
+ };
+
+ // mvhd - Movie Header Box
+ const mvhd: MovieHeaderBox = {
+ type: "mvhd",
+ version: 0,
+ flags: 0,
+ creationTime: 0,
+ modificationTime: 0,
+ timescale: timescale,
+ duration: 0, // Unknown/fragmented
+ rate: 0x00010000, // 1.0 in 16.16 fixed point
+ volume: 0x0100, // 1.0 in 8.8 fixed point
+ reserved1: 0,
+ reserved2: [0, 0],
+ matrix: IDENTITY_MATRIX,
+ preDefined: [0, 0, 0, 0, 0, 0],
+ nextTrackId: trackId + 1,
+ };
+
+ // tkhd - Track Header Box
+ const tkhd: TrackHeaderBox = {
+ type: "tkhd",
+ version: 0,
+ flags: 0x000003, // Track enabled + in movie
+ creationTime: 0,
+ modificationTime: 0,
+ trackId: trackId,
+ reserved1: 0,
+ duration: 0,
+ reserved2: [0, 0],
+ layer: 0,
+ alternateGroup: 0,
+ volume: 0, // Video tracks have 0 volume
+ reserved3: 0,
+ matrix: IDENTITY_MATRIX,
+ width: codedWidth * 0x10000, // 16.16 fixed point (avoid << which produces signed int)
+ height: codedHeight * 0x10000,
+ };
+
+ // mdhd - Media Header Box
+ const mdhd: MediaHeaderBox = {
+ type: "mdhd",
+ version: 0,
+ flags: 0,
+ creationTime: 0,
+ modificationTime: 0,
+ timescale: timescale,
+ duration: 0,
+ language: "und",
+ preDefined: 0,
+ };
+
+ // hdlr - Handler Reference Box
+ const hdlr: HandlerReferenceBox = {
+ type: "hdlr",
+ version: 0,
+ flags: 0,
+ preDefined: 0,
+ handlerType: "vide",
+ reserved: [0, 0, 0],
+ name: "VideoHandler",
+ };
+
+ // vmhd - Video Media Header Box
+ const vmhd: VideoMediaHeaderBox = {
+ type: "vmhd",
+ version: 0,
+ flags: 1, // Required to be 1
+ graphicsmode: 0,
+ opcolor: [0, 0, 0],
+ };
+
+ // url - Data Entry URL Box (self-contained)
+ const urlBox: DataEntryUrlBox = {
+ type: "url ",
+ version: 0,
+ flags: 0x000001, // Self-contained flag
+ location: "",
+ };
+
+ // dref - Data Reference Box
+ const dref: DataReferenceBox = {
+ type: "dref",
+ version: 0,
+ flags: 0,
+ entryCount: 1,
+ entries: [urlBox],
+ };
+
+ // dinf - Data Information Box
+ const dinf: DataInformationBox = {
+ type: "dinf",
+ boxes: [dref],
+ };
+
+ // Build the avc1 box manually since the library doesn't properly serialize Uint8Array children
+ const avc1Box = createAvc1Box(codedWidth, codedHeight, Hex.toBytes(description));
+
+ // stsd - Sample Description Box
+ const stsd: SampleDescriptionBox = {
+ type: "stsd",
+ version: 0,
+ flags: 0,
+ entryCount: 1,
+ // biome-ignore lint/suspicious/noExplicitAny: Raw avc1 box since library doesn't handle avcC children
+ entries: [avc1Box] as any[],
+ };
+
+ // stts - Decoding Time to Sample (empty for fragmented)
+ const stts: DecodingTimeToSampleBox = {
+ type: "stts",
+ version: 0,
+ flags: 0,
+ entryCount: 0,
+ entries: [],
+ };
+
+ // Create raw boxes for types without library writers
+ const stsc = createEmptyStsc();
+ const stsz = createEmptyStsz();
+ const stco = createEmptyStco();
+
+ // stbl - Sample Table Box
+ // Note: stsc, stsz, stco are raw Uint8Arrays since the library doesn't have writers for them
+ const stbl: SampleTableBox = {
+ type: "stbl",
+ // biome-ignore lint/suspicious/noExplicitAny: Raw boxes for types without library writers
+ boxes: [stsd, stts, stsc, stsz, stco] as any[],
+ };
+
+ // minf - Media Information Box
+ const minf: MediaInformationBox = {
+ type: "minf",
+ boxes: [vmhd, dinf, stbl],
+ };
+
+ // mdia - Media Box
+ const mdia: MediaBox = {
+ type: "mdia",
+ boxes: [mdhd, hdlr, minf],
+ };
+
+ // trak - Track Box
+ const trak: TrackBox = {
+ type: "trak",
+ boxes: [tkhd, mdia],
+ };
+
+ // trex - Track Extends Box (required for fragmented MP4)
+ const trex: TrackExtendsBox = {
+ type: "trex",
+ version: 0,
+ flags: 0,
+ trackId: trackId,
+ defaultSampleDescriptionIndex: 1,
+ defaultSampleDuration: 0,
+ defaultSampleSize: 0,
+ defaultSampleFlags: 0,
+ };
+
+ // mvex - Movie Extends Box (signals fragmented MP4)
+ const mvex: MovieExtendsBox = {
+ type: "mvex",
+ boxes: [trex],
+ };
+
+ // moov - Movie Box
+ const moov: MovieBox = {
+ type: "moov",
+ boxes: [mvhd, trak, mvex],
+ };
+
+ // Write all boxes and concatenate
+ const buffers = writeBoxes([ftyp, moov]);
+ const totalLength = buffers.reduce((sum, buf) => sum + buf.byteLength, 0);
+ const result = new Uint8Array(totalLength);
+
+ let offset = 0;
+ for (const buf of buffers) {
+ result.set(new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength), offset);
+ offset += buf.byteLength;
+ }
+
+ return result;
+}
+
+/**
+ * Creates an MSE-compatible initialization segment (ftyp + moov) for audio.
+ * Supports AAC (mp4a) and Opus codecs.
+ */
+export function createAudioInitSegment(config: Catalog.AudioConfig): Uint8Array {
+ const { sampleRate, numberOfChannels, description, codec, container } = config;
+
+ // Use timescale from CMAF container, or microseconds for legacy
+ const timescale = container.kind === "cmaf" ? container.timescale : 1_000_000;
+
+ // Use track_id from CMAF container, or default to 1 for legacy
+ const trackId = container.kind === "cmaf" ? container.trackId : 1;
+
+ // ftyp - File Type Box
+ const ftyp: FileTypeBox = {
+ type: "ftyp",
+ majorBrand: "isom",
+ minorVersion: 0x200,
+ compatibleBrands: ["isom", "iso6", "mp41"],
+ };
+
+ // mvhd - Movie Header Box
+ const mvhd: MovieHeaderBox = {
+ type: "mvhd",
+ version: 0,
+ flags: 0,
+ creationTime: 0,
+ modificationTime: 0,
+ timescale: timescale,
+ duration: 0,
+ rate: 0x00010000,
+ volume: 0x0100,
+ reserved1: 0,
+ reserved2: [0, 0],
+ matrix: IDENTITY_MATRIX,
+ preDefined: [0, 0, 0, 0, 0, 0],
+ nextTrackId: trackId + 1,
+ };
+
+ // tkhd - Track Header Box
+ const tkhd: TrackHeaderBox = {
+ type: "tkhd",
+ version: 0,
+ flags: 0x000003,
+ creationTime: 0,
+ modificationTime: 0,
+ trackId: trackId,
+ reserved1: 0,
+ duration: 0,
+ reserved2: [0, 0],
+ layer: 0,
+ alternateGroup: 0,
+ volume: 0x0100, // Audio tracks have volume (1.0 in 8.8 fixed point)
+ reserved3: 0,
+ matrix: IDENTITY_MATRIX,
+ width: 0,
+ height: 0,
+ };
+
+ // mdhd - Media Header Box
+ const mdhd: MediaHeaderBox = {
+ type: "mdhd",
+ version: 0,
+ flags: 0,
+ creationTime: 0,
+ modificationTime: 0,
+ timescale: timescale,
+ duration: 0,
+ language: "und",
+ preDefined: 0,
+ };
+
+ // hdlr - Handler Reference Box
+ const hdlr: HandlerReferenceBox = {
+ type: "hdlr",
+ version: 0,
+ flags: 0,
+ preDefined: 0,
+ handlerType: "soun",
+ reserved: [0, 0, 0],
+ name: "SoundHandler",
+ };
+
+ // smhd - Sound Media Header Box
+ const smhd: SoundMediaHeaderBox = {
+ type: "smhd",
+ version: 0,
+ flags: 0,
+ balance: 0,
+ reserved: 0,
+ };
+
+ // url - Data Entry URL Box (self-contained)
+ const urlBox: DataEntryUrlBox = {
+ type: "url ",
+ version: 0,
+ flags: 0x000001,
+ location: "",
+ };
+
+ // dref - Data Reference Box
+ const dref: DataReferenceBox = {
+ type: "dref",
+ version: 0,
+ flags: 0,
+ entryCount: 1,
+ entries: [urlBox],
+ };
+
+ // dinf - Data Information Box
+ const dinf: DataInformationBox = {
+ type: "dinf",
+ boxes: [dref],
+ };
+
+ // Build codec-specific sample entry (manually to ensure child boxes are properly serialized)
+ const sampleEntry = createAudioSampleEntry(codec, sampleRate, numberOfChannels, description);
+
+ // stsd - Sample Description Box
+ const stsd: SampleDescriptionBox = {
+ type: "stsd",
+ version: 0,
+ flags: 0,
+ entryCount: 1,
+ // biome-ignore lint/suspicious/noExplicitAny: Raw sample entry box
+ entries: [sampleEntry] as any[],
+ };
+
+ // stts - Decoding Time to Sample (empty for fragmented)
+ const stts: DecodingTimeToSampleBox = {
+ type: "stts",
+ version: 0,
+ flags: 0,
+ entryCount: 0,
+ entries: [],
+ };
+
+ // Create raw boxes for types without library writers
+ const stsc = createEmptyStsc();
+ const stsz = createEmptyStsz();
+ const stco = createEmptyStco();
+
+ // stbl - Sample Table Box
+ // Note: stsc, stsz, stco are raw Uint8Arrays since the library doesn't have writers for them
+ const stbl: SampleTableBox = {
+ type: "stbl",
+ // biome-ignore lint/suspicious/noExplicitAny: Raw boxes for types without library writers
+ boxes: [stsd, stts, stsc, stsz, stco] as any[],
+ };
+
+ // minf - Media Information Box
+ const minf: MediaInformationBox = {
+ type: "minf",
+ boxes: [smhd, dinf, stbl],
+ };
+
+ // mdia - Media Box
+ const mdia: MediaBox = {
+ type: "mdia",
+ boxes: [mdhd, hdlr, minf],
+ };
+
+ // trak - Track Box
+ const trak: TrackBox = {
+ type: "trak",
+ boxes: [tkhd, mdia],
+ };
+
+ // trex - Track Extends Box
+ const trex: TrackExtendsBox = {
+ type: "trex",
+ version: 0,
+ flags: 0,
+ trackId: trackId,
+ defaultSampleDescriptionIndex: 1,
+ defaultSampleDuration: 0,
+ defaultSampleSize: 0,
+ defaultSampleFlags: 0,
+ };
+
+ // mvex - Movie Extends Box
+ const mvex: MovieExtendsBox = {
+ type: "mvex",
+ boxes: [trex],
+ };
+
+ // moov - Movie Box
+ const moov: MovieBox = {
+ type: "moov",
+ boxes: [mvhd, trak, mvex],
+ };
+
+ // Write all boxes and concatenate
+ const buffers = writeBoxes([ftyp, moov]);
+ const totalLength = buffers.reduce((sum, buf) => sum + buf.byteLength, 0);
+ const result = new Uint8Array(totalLength);
+
+ let offset = 0;
+ for (const buf of buffers) {
+ result.set(new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength), offset);
+ offset += buf.byteLength;
+ }
+
+ return result;
+}
+
+/**
+ * Create an audio sample entry box (mp4a or Opus) with embedded codec config.
+ * Built manually because the library doesn't properly serialize Uint8Array child boxes.
+ */
+function createAudioSampleEntry(
+ codec: string,
+ sampleRate: number,
+ channelCount: number,
+ description?: string,
+): Uint8Array {
+ if (codec.startsWith("mp4a")) {
+ return createMp4aBox(sampleRate, channelCount, description);
+ } else if (codec === "opus") {
+ return createOpusBox(sampleRate, channelCount, description);
+ }
+ throw new Error(`Unsupported audio codec: ${codec}`);
+}
+
+/**
+ * Create an mp4a (AAC Audio Sample Entry) box with embedded esds.
+ */
+function createMp4aBox(sampleRate: number, channelCount: number, description?: string): Uint8Array {
+ const esds = createEsdsBox(sampleRate, channelCount, description);
+
+ // mp4a box structure (AudioSampleEntry):
+ // - 6 bytes reserved (0)
+ // - 2 bytes data_reference_index (1)
+ // - 8 bytes reserved (0) - includes reserved2[2] and pre_defined fields
+ // - 2 bytes channelcount
+ // - 2 bytes samplesize (16)
+ // - 2 bytes pre_defined (0)
+ // - 2 bytes reserved (0)
+ // - 4 bytes samplerate (16.16 fixed point)
+ // - child boxes (esds)
+
+ const mp4aContentSize = 6 + 2 + 8 + 2 + 2 + 2 + 2 + 4 + esds.length;
+ const mp4aSize = 8 + mp4aContentSize;
+
+ const box = new Uint8Array(mp4aSize);
+ const view = new DataView(box.buffer);
+ let offset = 0;
+
+ // Box header
+ view.setUint32(offset, mp4aSize, false);
+ offset += 4;
+ box[offset++] = 0x6d; // 'm'
+ box[offset++] = 0x70; // 'p'
+ box[offset++] = 0x34; // '4'
+ box[offset++] = 0x61; // 'a'
+
+ // SampleEntry fields
+ offset += 6; // reserved (6 bytes of 0)
+ view.setUint16(offset, 1, false);
+ offset += 2; // data_reference_index = 1
+
+ // AudioSampleEntry fields
+ offset += 8; // reserved (8 bytes of 0)
+ view.setUint16(offset, channelCount, false);
+ offset += 2;
+ view.setUint16(offset, 16, false);
+ offset += 2; // samplesize = 16
+ view.setUint16(offset, 0, false);
+ offset += 2; // pre_defined
+ view.setUint16(offset, 0, false);
+ offset += 2; // reserved
+ view.setUint32(offset, sampleRate * 0x10000, false);
+ offset += 4; // samplerate (16.16 fixed point)
+
+ // esds child box (already includes box header)
+ box.set(esds, offset);
+
+ return box;
+}
+
+/**
+ * Create an Opus (Opus Audio Sample Entry) box with embedded dOps.
+ */
+function createOpusBox(sampleRate: number, channelCount: number, description?: string): Uint8Array {
+ const dOps = createDOpsBox(channelCount, sampleRate, description);
+
+ // Opus box structure (AudioSampleEntry):
+ // Same structure as mp4a
+ const opusContentSize = 6 + 2 + 8 + 2 + 2 + 2 + 2 + 4 + dOps.length;
+ const opusSize = 8 + opusContentSize;
+
+ const box = new Uint8Array(opusSize);
+ const view = new DataView(box.buffer);
+ let offset = 0;
+
+ // Box header
+ view.setUint32(offset, opusSize, false);
+ offset += 4;
+ box[offset++] = 0x4f; // 'O'
+ box[offset++] = 0x70; // 'p'
+ box[offset++] = 0x75; // 'u'
+ box[offset++] = 0x73; // 's'
+
+ // SampleEntry fields
+ offset += 6; // reserved (6 bytes of 0)
+ view.setUint16(offset, 1, false);
+ offset += 2; // data_reference_index = 1
+
+ // AudioSampleEntry fields
+ offset += 8; // reserved (8 bytes of 0)
+ view.setUint16(offset, channelCount, false);
+ offset += 2;
+ view.setUint16(offset, 16, false);
+ offset += 2; // samplesize = 16
+ view.setUint16(offset, 0, false);
+ offset += 2; // pre_defined
+ view.setUint16(offset, 0, false);
+ offset += 2; // reserved
+ view.setUint32(offset, sampleRate * 0x10000, false);
+ offset += 4; // samplerate (16.16 fixed point)
+
+ // dOps child box (already includes box header)
+ box.set(dOps, offset);
+
+ return box;
+}
+
+/**
+ * Generate an AudioSpecificConfig for AAC-LC.
+ * Format: 5 bits audioObjectType + 4 bits samplingFrequencyIndex + 4 bits channelConfiguration + 3 bits (zeros)
+ */
+function generateAudioSpecificConfig(sampleRate: number, channelCount: number): Uint8Array {
+ // Sampling frequency index lookup
+ const sampleRateIndex: Record = {
+ 96000: 0,
+ 88200: 1,
+ 64000: 2,
+ 48000: 3,
+ 44100: 4,
+ 32000: 5,
+ 24000: 6,
+ 22050: 7,
+ 16000: 8,
+ 12000: 9,
+ 11025: 10,
+ 8000: 11,
+ 7350: 12,
+ };
+
+ const freqIndex = sampleRateIndex[sampleRate] ?? 4; // Default to 44100 if not found
+ const audioObjectType = 2; // AAC-LC
+
+ // AudioSpecificConfig is 2 bytes for AAC-LC:
+ // 5 bits: audioObjectType (2 for AAC-LC)
+ // 4 bits: samplingFrequencyIndex
+ // 4 bits: channelConfiguration
+ // 3 bits: GASpecificConfig (frameLengthFlag=0, dependsOnCoreCoder=0, extensionFlag=0)
+ const byte0 = (audioObjectType << 3) | (freqIndex >> 1);
+ const byte1 = ((freqIndex & 1) << 7) | (channelCount << 3);
+
+ return new Uint8Array([byte0, byte1]);
+}
+
+/**
+ * Creates an esds (Elementary Stream Descriptor) box for AAC.
+ * The description from WebCodecs is the AudioSpecificConfig.
+ * If no description is provided, generates one from sampleRate and channelCount.
+ */
+function createEsdsBox(sampleRate: number, channelCount: number, description?: string): Uint8Array {
+ const audioSpecificConfig = description
+ ? Hex.toBytes(description)
+ : generateAudioSpecificConfig(sampleRate, channelCount);
+
+ // ES_Descriptor structure:
+ // - tag (0x03) + size + ES_ID (2) + flags (1)
+ // - DecoderConfigDescriptor: tag (0x04) + size + objectTypeIndication (1) + streamType (1) + bufferSizeDB (3) + maxBitrate (4) + avgBitrate (4)
+ // - DecoderSpecificInfo: tag (0x05) + size + AudioSpecificConfig
+ // - SLConfigDescriptor: tag (0x06) + size + predefined (1)
+
+ const decSpecificInfoSize = audioSpecificConfig.length;
+ const decConfigDescSize = 13 + 2 + decSpecificInfoSize; // 13 fixed + tag/size + ASC
+ const esDescSize = 3 + 2 + decConfigDescSize + 3; // 3 fixed + tag/size + DCD + SLC (3 bytes)
+
+ const esdsSize = 12 + 2 + esDescSize; // 4 (size) + 4 (type) + 4 (version/flags) + tag/size + ESD
+ const esds = new Uint8Array(esdsSize);
+ const view = new DataView(esds.buffer);
+
+ let offset = 0;
+
+ // Box header
+ view.setUint32(offset, esdsSize, false);
+ offset += 4;
+ esds[offset++] = 0x65; // 'e'
+ esds[offset++] = 0x73; // 's'
+ esds[offset++] = 0x64; // 'd'
+ esds[offset++] = 0x73; // 's'
+
+ // Version and flags (full box)
+ view.setUint32(offset, 0, false);
+ offset += 4;
+
+ // ES_Descriptor
+ esds[offset++] = 0x03; // tag
+ esds[offset++] = esDescSize; // size (assuming < 128)
+
+ view.setUint16(offset, 0, false);
+ offset += 2; // ES_ID
+ esds[offset++] = 0; // flags
+
+ // DecoderConfigDescriptor
+ esds[offset++] = 0x04; // tag
+ esds[offset++] = decConfigDescSize; // size
+
+ esds[offset++] = 0x40; // objectTypeIndication: Audio ISO/IEC 14496-3 (AAC)
+ esds[offset++] = 0x15; // streamType (5 = audio) << 2 | upstream (0) << 1 | reserved (1)
+ esds[offset++] = 0x00; // bufferSizeDB (3 bytes)
+ esds[offset++] = 0x00;
+ esds[offset++] = 0x00;
+ view.setUint32(offset, 0, false);
+ offset += 4; // maxBitrate
+ view.setUint32(offset, 0, false);
+ offset += 4; // avgBitrate
+
+ // DecoderSpecificInfo (AudioSpecificConfig)
+ esds[offset++] = 0x05; // tag
+ esds[offset++] = decSpecificInfoSize; // size
+ esds.set(audioSpecificConfig, offset);
+ offset += decSpecificInfoSize;
+
+ // SLConfigDescriptor
+ esds[offset++] = 0x06; // tag
+ esds[offset++] = 0x01; // size
+ esds[offset++] = 0x02; // predefined = MP4
+
+ return esds;
+}
+
+/**
+ * Creates a dOps (Opus Specific) box.
+ * See https://opus-codec.org/docs/opus_in_isobmff.html
+ */
+function createDOpsBox(channelCount: number, sampleRate: number, description?: string): Uint8Array {
+ // If description is provided, it's the OpusHead without the magic signature
+ if (description) {
+ const opusHead = Hex.toBytes(description);
+ const dOpsSize = 8 + opusHead.length;
+ const dOps = new Uint8Array(dOpsSize);
+ const view = new DataView(dOps.buffer);
+
+ view.setUint32(0, dOpsSize, false);
+ dOps[4] = 0x64; // 'd'
+ dOps[5] = 0x4f; // 'O'
+ dOps[6] = 0x70; // 'p'
+ dOps[7] = 0x73; // 's'
+ dOps.set(opusHead, 8);
+
+ return dOps;
+ }
+
+ // Build minimal dOps box
+ // dOps structure: Version (1) + OutputChannelCount (1) + PreSkip (2) +
+ // InputSampleRate (4) + OutputGain (2) + ChannelMappingFamily (1)
+ const dOpsSize = 8 + 11; // box header + content
+ const dOps = new Uint8Array(dOpsSize);
+ const view = new DataView(dOps.buffer);
+
+ let offset = 0;
+ view.setUint32(offset, dOpsSize, false);
+ offset += 4;
+ dOps[offset++] = 0x64; // 'd'
+ dOps[offset++] = 0x4f; // 'O'
+ dOps[offset++] = 0x70; // 'p'
+ dOps[offset++] = 0x73; // 's'
+
+ dOps[offset++] = 0; // Version
+ dOps[offset++] = channelCount;
+ view.setUint16(offset, 312, false);
+ offset += 2; // PreSkip (typical value)
+ view.setUint32(offset, sampleRate, false);
+ offset += 4; // InputSampleRate
+ view.setInt16(offset, 0, false);
+ offset += 2; // OutputGain
+ dOps[offset++] = 0; // ChannelMappingFamily (0 = mono/stereo)
+
+ return dOps;
+}
+
+export interface DataSegmentOptions {
+ /** Raw frame data */
+ data: Uint8Array;
+ /** Timestamp in timescale units */
+ timestamp: number;
+ /** Duration in timescale units */
+ duration: number;
+ /** Whether this is a keyframe */
+ keyframe: boolean;
+ /** Sequence number for this fragment */
+ sequence: number;
+ /** Track ID (default: 1) */
+ trackId?: number;
+}
+
+/**
+ * Encode a raw frame into a moof+mdat segment for MSE.
+ *
+ * @param opts - Options for the data segment
+ * @returns The encoded moof+mdat segment
+ */
+export function encodeDataSegment(opts: DataSegmentOptions): Uint8Array {
+ const { data, timestamp, duration, keyframe, sequence, trackId = 1 } = opts;
+
+ // Sample flags:
+ // - sample_depends_on: bits 25-24 (2 = does not depend on others for IDR, 1 = depends on others)
+ // - sample_is_non_sync_sample: bit 16 (0 = sync/keyframe, 1 = non-sync)
+ // For keyframe: depends_on=2 (0x02000000), non_sync=0
+ // For non-keyframe: depends_on=1 (0x01000000), non_sync=1 (0x00010000)
+ const sampleFlags = keyframe ? 0x02000000 : 0x01010000;
+
+ // mfhd - Movie Fragment Header
+ const mfhd: MovieFragmentHeaderBox = {
+ type: "mfhd",
+ version: 0,
+ flags: 0,
+ sequenceNumber: sequence,
+ };
+
+ // tfhd - Track Fragment Header
+ // Flags: default-base-is-moof (0x020000)
+ const tfhd: TrackFragmentHeaderBox = {
+ type: "tfhd",
+ version: 0,
+ flags: 0x020000,
+ trackId,
+ };
+
+ // tfdt - Track Fragment Base Media Decode Time
+ const tfdt: TrackFragmentBaseMediaDecodeTimeBox = {
+ type: "tfdt",
+ version: 1, // version 1 for 64-bit baseMediaDecodeTime
+ flags: 0,
+ baseMediaDecodeTime: timestamp,
+ };
+
+ // trun - Track Run
+ // Flags: data-offset-present (0x000001) | sample-duration-present (0x000100) |
+ // sample-size-present (0x000200) | sample-flags-present (0x000400)
+ const trun: TrackRunBox = {
+ type: "trun",
+ version: 0,
+ flags: 0x000001 | 0x000100 | 0x000200 | 0x000400,
+ sampleCount: 1,
+ dataOffset: 0, // Will be calculated after we know moof size
+ samples: [
+ {
+ sampleDuration: duration,
+ sampleSize: data.byteLength,
+ sampleFlags,
+ },
+ ],
+ };
+
+ // traf - Track Fragment
+ const traf: TrackFragmentBox = {
+ type: "traf",
+ boxes: [tfhd, tfdt, trun],
+ };
+
+ // moof - Movie Fragment
+ const moof: MovieFragmentBox = {
+ type: "moof",
+ boxes: [mfhd, traf],
+ };
+
+ // Write moof to calculate its size
+ const moofBuffers = writeBoxes([moof]);
+ let moofSize = 0;
+ for (const buf of moofBuffers) {
+ moofSize += buf.byteLength;
+ }
+
+ // Update trun.dataOffset to point to mdat data
+ // dataOffset is relative to moof start, so it's moofSize + 8 (mdat header)
+ trun.dataOffset = moofSize + 8;
+
+ // Re-write moof with correct dataOffset
+ const moofBuffersFinal = writeBoxes([moof]);
+ moofSize = 0;
+ for (const buf of moofBuffersFinal) {
+ moofSize += buf.byteLength;
+ }
+
+ // mdat - Media Data
+ // Need to ensure the data is a proper ArrayBuffer-backed Uint8Array for the library
+ const mdatBuffer = new ArrayBuffer(data.byteLength);
+ const mdatData = new Uint8Array(mdatBuffer);
+ mdatData.set(data);
+ const mdat: MediaDataBox = {
+ type: "mdat",
+ data: mdatData,
+ };
+
+ const mdatBuffers = writeBoxes([mdat]);
+ let mdatSize = 0;
+ for (const buf of mdatBuffers) {
+ mdatSize += buf.byteLength;
+ }
+
+ // Concatenate all buffers
+ const result = new Uint8Array(moofSize + mdatSize);
+ let offset = 0;
+
+ for (const buf of moofBuffersFinal) {
+ result.set(new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength), offset);
+ offset += buf.byteLength;
+ }
+
+ for (const buf of mdatBuffers) {
+ result.set(new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength), offset);
+ offset += buf.byteLength;
+ }
+
+ return result;
+}
diff --git a/js/hang/src/mp4/index.ts b/js/hang/src/mp4/index.ts
new file mode 100644
index 000000000..0cdce1adb
--- /dev/null
+++ b/js/hang/src/mp4/index.ts
@@ -0,0 +1,6 @@
+/**
+ * MP4 utilities for encoding and decoding fMP4 segments.
+ */
+
+export * from "./decode";
+export * from "./encode";
diff --git a/js/hang/src/publish/audio/encoder.ts b/js/hang/src/publish/audio/encoder.ts
index a19e096f0..4325dcab7 100644
--- a/js/hang/src/publish/audio/encoder.ts
+++ b/js/hang/src/publish/audio/encoder.ts
@@ -1,12 +1,9 @@
import type * as Moq from "@moq/lite";
import { Time } from "@moq/lite";
import { Effect, type Getter, Signal } from "@moq/signals";
-import type * as Catalog from "../../catalog";
-import { DEFAULT_CONTAINER } from "../../catalog";
-import { u53 } from "../../catalog/integers";
+import * as Catalog from "../../catalog";
import * as Frame from "../../frame";
import * as libav from "../../util/libav";
-import { PRIORITY } from "../priority";
import type * as Capture from "./capture";
import type { Source } from "./types";
@@ -33,14 +30,13 @@ export type EncoderProps = {
export class Encoder {
static readonly TRACK = "audio/data";
- static readonly PRIORITY = PRIORITY.audio;
+ static readonly PRIORITY = Catalog.PRIORITY.audio;
enabled: Signal;
muted: Signal;
volume: Signal;
maxLatency: Time.Milli;
- #container: Catalog.Container;
source: Signal;
@@ -65,7 +61,6 @@ export class Encoder {
this.muted = Signal.from(props?.muted ?? false);
this.volume = Signal.from(props?.volume ?? 1);
this.maxLatency = props?.maxLatency ?? (100 as Time.Milli); // Default is a group every 100ms
- this.#container = props?.container ?? DEFAULT_CONTAINER;
this.#signals.effect(this.#runSource.bind(this));
this.#signals.effect(this.#runConfig.bind(this));
@@ -129,10 +124,10 @@ export class Encoder {
const config = {
codec: "opus",
- sampleRate: u53(worklet.context.sampleRate),
- numberOfChannels: u53(worklet.channelCount),
- bitrate: u53(worklet.channelCount * 32_000),
- container: this.#container,
+ sampleRate: Catalog.u53(worklet.context.sampleRate),
+ numberOfChannels: Catalog.u53(worklet.channelCount),
+ bitrate: Catalog.u53(worklet.channelCount * 32_000),
+ container: { kind: "legacy" } as const,
};
effect.set(this.#config, config);
@@ -176,8 +171,6 @@ export class Encoder {
// We're using an async polyfill temporarily for Safari support.
await libav.polyfill();
- console.log(`[Audio Publisher] Using container format: ${this.#container}`);
-
const encoder = new AudioEncoder({
output: (frame) => {
if (frame.type !== "key") {
@@ -192,7 +185,7 @@ export class Encoder {
groupTimestamp = frame.timestamp as Time.Micro;
}
- const buffer = Frame.encode(frame, frame.timestamp as Time.Micro, this.#container);
+ const buffer = Frame.encode(frame, frame.timestamp as Time.Micro);
group.writeFrame(buffer);
},
error: (err) => {
diff --git a/js/hang/src/publish/chat/message.ts b/js/hang/src/publish/chat/message.ts
index 343f715ef..b8ef2f810 100644
--- a/js/hang/src/publish/chat/message.ts
+++ b/js/hang/src/publish/chat/message.ts
@@ -1,7 +1,6 @@
import type * as Moq from "@moq/lite";
import { Effect, Signal } from "@moq/signals";
-import type * as Catalog from "../../catalog";
-import { PRIORITY } from "../priority";
+import * as Catalog from "../../catalog";
export type MessageProps = {
enabled?: boolean | Signal;
@@ -9,7 +8,7 @@ export type MessageProps = {
export class Message {
static readonly TRACK = "chat/message.txt";
- static readonly PRIORITY = PRIORITY.chat;
+ static readonly PRIORITY = Catalog.PRIORITY.chat;
enabled: Signal;
diff --git a/js/hang/src/publish/chat/typing.ts b/js/hang/src/publish/chat/typing.ts
index 419ad56cf..0064f30eb 100644
--- a/js/hang/src/publish/chat/typing.ts
+++ b/js/hang/src/publish/chat/typing.ts
@@ -1,7 +1,6 @@
import type * as Moq from "@moq/lite";
import { Effect, Signal } from "@moq/signals";
-import type * as Catalog from "../../catalog";
-import { PRIORITY } from "../priority";
+import * as Catalog from "../../catalog";
export type TypingProps = {
enabled?: boolean | Signal;
@@ -9,7 +8,7 @@ export type TypingProps = {
export class Typing {
static readonly TRACK = "chat/typing.bool";
- static readonly PRIORITY = PRIORITY.typing;
+ static readonly PRIORITY = Catalog.PRIORITY.typing;
enabled: Signal;
diff --git a/js/hang/src/publish/location/peers.ts b/js/hang/src/publish/location/peers.ts
index 9811431ac..e2952f575 100644
--- a/js/hang/src/publish/location/peers.ts
+++ b/js/hang/src/publish/location/peers.ts
@@ -2,7 +2,6 @@ import type * as Moq from "@moq/lite";
import * as Zod from "@moq/lite/zod";
import { Effect, Signal } from "@moq/signals";
import * as Catalog from "../../catalog";
-import { PRIORITY } from "../priority";
export interface PeersProps {
enabled?: boolean | Signal;
@@ -11,7 +10,7 @@ export interface PeersProps {
export class Peers {
static readonly TRACK = "location/peers.json";
- static readonly PRIORITY = PRIORITY.location;
+ static readonly PRIORITY = Catalog.PRIORITY.location;
enabled: Signal;
positions = new Signal>({});
diff --git a/js/hang/src/publish/location/window.ts b/js/hang/src/publish/location/window.ts
index dd72bbe02..d6d3a762a 100644
--- a/js/hang/src/publish/location/window.ts
+++ b/js/hang/src/publish/location/window.ts
@@ -2,7 +2,6 @@ import type * as Moq from "@moq/lite";
import * as Zod from "@moq/lite/zod";
import { Effect, Signal } from "@moq/signals";
import * as Catalog from "../../catalog";
-import { PRIORITY } from "../priority";
export type WindowProps = {
// If true, then we'll publish our position to the broadcast.
@@ -17,7 +16,7 @@ export type WindowProps = {
export class Window {
static readonly TRACK = "location/window.json";
- static readonly PRIORITY = PRIORITY.location;
+ static readonly PRIORITY = Catalog.PRIORITY.location;
enabled: Signal;
position: Signal;
diff --git a/js/hang/src/publish/preview.ts b/js/hang/src/publish/preview.ts
index 91df96d1f..3752dd7cc 100644
--- a/js/hang/src/publish/preview.ts
+++ b/js/hang/src/publish/preview.ts
@@ -1,7 +1,7 @@
import type * as Moq from "@moq/lite";
import { Effect, Signal } from "@moq/signals";
import type * as Catalog from "../catalog";
-import { PRIORITY } from "./priority";
+import { PRIORITY } from "../catalog/priority";
export type PreviewProps = {
enabled?: boolean | Signal;
diff --git a/js/hang/src/publish/video/encoder.ts b/js/hang/src/publish/video/encoder.ts
index 1aed44cda..9dbb444e3 100644
--- a/js/hang/src/publish/video/encoder.ts
+++ b/js/hang/src/publish/video/encoder.ts
@@ -1,8 +1,7 @@
import type * as Moq from "@moq/lite";
import { Time } from "@moq/lite";
import { Effect, type Getter, Signal } from "@moq/signals";
-import type * as Catalog from "../../catalog";
-import { DEFAULT_CONTAINER, u53 } from "../../catalog";
+import * as Catalog from "../../catalog";
import * as Frame from "../../frame";
import { isFirefox } from "../../util/hacks";
import type { Source } from "./types";
@@ -40,7 +39,6 @@ export class Encoder {
enabled: Signal;
source: Signal;
frame: Getter;
- #container: Catalog.Container;
#catalog = new Signal(undefined);
readonly catalog: Getter = this.#catalog;
@@ -64,7 +62,6 @@ export class Encoder {
this.source = source;
this.enabled = Signal.from(props?.enabled ?? false);
this.config = Signal.from(props?.config);
- this.#container = props?.container ?? DEFAULT_CONTAINER;
this.#signals.effect(this.#runCatalog.bind(this));
this.#signals.effect(this.#runConfig.bind(this));
@@ -78,8 +75,6 @@ export class Encoder {
effect.set(this.active, true, false);
effect.spawn(async () => {
- console.log(`[Video Publisher] Using container format: ${this.#container}`);
-
let group: Moq.Group | undefined;
effect.cleanup(() => group?.close());
@@ -95,7 +90,7 @@ export class Encoder {
throw new Error("no keyframe");
}
- const buffer = Frame.encode(frame, frame.timestamp as Time.Micro, this.#container);
+ const buffer = Frame.encode(frame, frame.timestamp as Time.Micro);
group?.writeFrame(buffer);
},
error: (err: Error) => {
@@ -143,12 +138,12 @@ export class Encoder {
const catalog: Catalog.VideoConfig = {
codec: config.codec,
- bitrate: config.bitrate ? u53(config.bitrate) : undefined,
+ bitrate: config.bitrate ? Catalog.u53(config.bitrate) : undefined,
framerate: config.framerate,
- codedWidth: u53(config.width),
- codedHeight: u53(config.height),
+ codedWidth: Catalog.u53(config.width),
+ codedHeight: Catalog.u53(config.height),
optimizeForLatency: true,
- container: this.#container,
+ container: { kind: "legacy" } as const,
};
effect.set(this.#catalog, catalog);
diff --git a/js/hang/src/publish/video/index.ts b/js/hang/src/publish/video/index.ts
index 18c838891..97bfaf22c 100644
--- a/js/hang/src/publish/video/index.ts
+++ b/js/hang/src/publish/video/index.ts
@@ -1,6 +1,5 @@
import { Effect, Signal } from "@moq/signals";
import * as Catalog from "../../catalog";
-import { PRIORITY } from "../priority";
import { Encoder, type EncoderProps } from "./encoder";
import { TrackProcessor } from "./polyfill";
import type { Source } from "./types";
@@ -18,7 +17,7 @@ export type Props = {
export class Root {
static readonly TRACK_HD = "video/hd";
static readonly TRACK_SD = "video/sd";
- static readonly PRIORITY = PRIORITY.video;
+ static readonly PRIORITY = Catalog.PRIORITY.video;
source: Signal;
hd: Encoder;
diff --git a/js/hang/src/util/latency.ts b/js/hang/src/util/latency.ts
new file mode 100644
index 000000000..b3da28558
--- /dev/null
+++ b/js/hang/src/util/latency.ts
@@ -0,0 +1,48 @@
+import type * as Moq from "@moq/lite";
+import { Effect, Signal } from "@moq/signals";
+import type * as Catalog from "../catalog";
+
+export interface LatencyProps {
+ buffer: Signal;
+ config: Signal | Signal;
+}
+
+// A helper class that computes the final latency based on the catalog's minBuffer and the user's buffer.
+// If the minBuffer is not present, then we use frame timings to compute the frame rate as the default.
+export class Latency {
+ buffer: Signal;
+ config: Signal | Signal;
+
+ signals = new Effect();
+
+ combined = new Signal(0 as Moq.Time.Milli);
+
+ constructor(props: LatencyProps) {
+ this.buffer = props.buffer;
+ this.config = props.config;
+
+ this.signals.effect(this.#run.bind(this));
+ }
+
+ #run(effect: Effect): void {
+ const buffer = effect.get(this.buffer);
+
+ // Compute the latency based on the catalog's minBuffer and the user's buffer.
+ const config = effect.get(this.config);
+
+ // TODO use the audio frequency + sample_rate?
+ // TODO or compute the duration between frames if neither minBuffer nor framerate is set
+ let minBuffer: number | undefined = config?.minBuffer;
+ if (!minBuffer && config && "framerate" in config) {
+ minBuffer = config.framerate ? 1000 / config.framerate : 0;
+ }
+ minBuffer ??= 0;
+
+ const latency = (minBuffer + buffer) as Moq.Time.Milli;
+ this.combined.set(latency);
+ }
+
+ close(): void {
+ this.signals.close();
+ }
+}
diff --git a/js/hang/src/watch/audio/backend.ts b/js/hang/src/watch/audio/backend.ts
new file mode 100644
index 000000000..39f8d82cd
--- /dev/null
+++ b/js/hang/src/watch/audio/backend.ts
@@ -0,0 +1,40 @@
+import type { Getter, Signal } from "@moq/signals";
+import type * as Catalog from "../../catalog";
+import type { BufferedRanges } from "../backend";
+
+// Audio specific signals that work regardless of the backend source (mse vs webcodecs).
+export interface Backend {
+ // The catalog of the audio.
+ catalog: Getter;
+
+ // The volume of the audio, between 0 and 1.
+ volume: Signal;
+
+ // Whether the audio is muted.
+ muted: Signal;
+
+ // The desired rendition/bitrate of the audio.
+ target: Signal;
+
+ // The name of the active rendition.
+ rendition: Signal;
+
+ // The config of the active rendition.
+ config: Getter;
+
+ // The stats of the audio.
+ stats: Getter;
+
+ // Buffered time ranges (for MSE backend).
+ buffered: Getter;
+}
+
+export interface Stats {
+ sampleCount: number;
+ bytesReceived: number;
+}
+
+export type Target = {
+ // Optional manual override for the selected rendition name.
+ name?: string;
+};
diff --git a/js/hang/src/watch/audio/index.ts b/js/hang/src/watch/audio/index.ts
index 7962a77a1..997d9b28c 100644
--- a/js/hang/src/watch/audio/index.ts
+++ b/js/hang/src/watch/audio/index.ts
@@ -1,2 +1,3 @@
+export type * from "./backend";
export * from "./emitter";
export * from "./source";
diff --git a/js/hang/src/watch/audio/source.ts b/js/hang/src/watch/audio/source.ts
index fad996840..c88490728 100644
--- a/js/hang/src/watch/audio/source.ts
+++ b/js/hang/src/watch/audio/source.ts
@@ -1,37 +1,52 @@
import type * as Moq from "@moq/lite";
import type { Time } from "@moq/lite";
import { Effect, type Getter, Signal } from "@moq/signals";
-import type * as Catalog from "../../catalog";
+import * as Catalog from "../../catalog";
import * as Frame from "../../frame";
+import * as Mp4 from "../../mp4";
import * as Hex from "../../util/hex";
+import { Latency } from "../../util/latency";
import * as libav from "../../util/libav";
+import type { BufferedRanges } from "../backend";
+import type { Broadcast } from "../broadcast";
+import type { Target } from "../video/backend";
import type * as Render from "./render";
+// Unfortunately, we need to use a Vite-exclusive import for now.
+import RenderWorklet from "./render-worklet.ts?worker&url";
-// We want some extra overhead to avoid starving the render worklet.
-// The default Opus frame duration is 20ms.
-// TODO: Put it in the catalog so we don't have to guess.
-const JITTER_UNDERHEAD = 25 as Time.Milli;
+// Extra overhead subtracted from effective latency to avoid starving the render worklet.
+// The audio worklet needs some buffer to avoid underruns.
+const BUFFER_UNDERHEAD = 25 as Time.Milli;
export type SourceProps = {
+ broadcast: Broadcast | Signal;
+
// Enable to download the audio track.
enabled?: boolean | Signal;
- // Jitter buffer size in milliseconds (default: 100ms)
- latency?: Time.Milli | Signal;
+ // Additional buffer in milliseconds on top of the catalog's minBuffer (default: 100ms).
+ // The effective latency = catalog.minBuffer + buffer
+ // Increase this if experiencing stuttering due to network jitter.
+ buffer?: Time.Milli | Signal;
+
+ // The desired rendition/bitrate of the audio.
+ // TODO finish implementing this
+ target?: Target | Signal;
};
export interface AudioStats {
bytesReceived: number;
}
-// Unfortunately, we need to use a Vite-exclusive import for now.
-import RenderWorklet from "./render-worklet.ts?worker&url";
-
// Downloads audio from a track and emits it to an AudioContext.
// The user is responsible for hooking up audio to speakers, an analyzer, etc.
export class Source {
- broadcast: Getter;
+ broadcast: Signal;
enabled: Signal;
+ target: Signal;
+
+ #catalog = new Signal(undefined);
+ readonly catalog: Getter = this.#catalog;
#context = new Signal(undefined);
readonly context: Getter = this.#context;
@@ -47,44 +62,76 @@ export class Source {
#stats = new Signal(undefined);
readonly stats: Getter = this.#stats;
- catalog = new Signal(undefined);
+ // Empty stub for WebCodecs (no traditional buffering)
+ #buffered = new Signal([]);
+ readonly buffered: Getter = this.#buffered;
+
config = new Signal(undefined);
- // Not a signal because I'm lazy.
- readonly latency: Signal;
+ // Additional buffer in milliseconds (on top of catalog's minBuffer).
+ buffer: Signal;
+
+ #latency: Latency;
+ readonly latency: Getter;
// The name of the active rendition.
- active = new Signal(undefined);
+ rendition = new Signal(undefined);
#signals = new Effect();
- constructor(
- broadcast: Getter,
- catalog: Getter,
- props?: SourceProps,
- ) {
- this.broadcast = broadcast;
+ constructor(props?: SourceProps) {
+ this.broadcast = Signal.from(props?.broadcast);
this.enabled = Signal.from(props?.enabled ?? false);
- this.latency = Signal.from(props?.latency ?? (100 as Time.Milli)); // TODO Reduce this once fMP4 stuttering is fixed.
-
- this.#signals.effect((effect) => {
- const audio = effect.get(catalog)?.audio;
- this.catalog.set(audio);
-
- if (audio?.renditions) {
- const first = Object.entries(audio.renditions).at(0);
- if (first) {
- effect.set(this.active, first[0]);
- effect.set(this.config, first[1]);
- }
- }
+ this.buffer = Signal.from(props?.buffer ?? (100 as Time.Milli));
+ this.target = Signal.from(props?.target);
+
+ this.#latency = new Latency({
+ buffer: this.buffer,
+ config: this.config,
});
+ this.latency = this.#latency.combined;
+ this.#signals.effect(this.#runCatalog.bind(this));
this.#signals.effect(this.#runWorklet.bind(this));
this.#signals.effect(this.#runEnabled.bind(this));
this.#signals.effect(this.#runDecoder.bind(this));
}
+ #runCatalog(effect: Effect): void {
+ const broadcast = effect.get(this.broadcast);
+ if (!broadcast) return;
+
+ const catalog = effect.get(broadcast.catalog);
+ if (!catalog) return;
+
+ const audio = catalog.audio;
+ if (!audio) return;
+
+ effect.set(this.#catalog, audio);
+
+ const rendition = this.#selectRendition(audio);
+ if (!rendition) return;
+
+ effect.set(this.rendition, rendition.track);
+ effect.set(this.config, rendition.config);
+ }
+
+ #selectRendition(audio: Catalog.Audio): { track: string; config: Catalog.AudioConfig } | undefined {
+ // Prefer legacy container if available, but support CMAF as well
+ let cmafRendition: { track: string; config: Catalog.AudioConfig } | undefined;
+
+ for (const [track, config] of Object.entries(audio.renditions)) {
+ if (config.container.kind === "legacy") {
+ return { track, config };
+ }
+ if (config.container.kind === "cmaf" && !cmafRendition) {
+ cmafRendition = { track, config };
+ }
+ }
+
+ return cmafRendition;
+ }
+
#runWorklet(effect: Effect): void {
// It takes a second or so to initialize the AudioContext/AudioWorklet, so do it even if disabled.
// This is less efficient for video-only playback but makes muting/unmuting instant.
@@ -102,7 +149,7 @@ export class Source {
// This way we can process the audio for visualizations.
const context = new AudioContext({
- latencyHint: "interactive", // We don't use real-time because of the jitter buffer.
+ latencyHint: "interactive", // We don't use real-time because of the buffer.
sampleRate,
});
effect.set(this.#context, context);
@@ -157,21 +204,32 @@ export class Source {
const broadcast = effect.get(this.broadcast);
if (!broadcast) return;
+ const active = broadcast ? effect.get(broadcast.active) : undefined;
+ if (!active) return;
+
const config = effect.get(this.config);
if (!config) return;
- const active = effect.get(this.active);
- if (!active) return;
+ const rendition = effect.get(this.rendition);
+ if (!rendition) return;
- const sub = broadcast.subscribe(active, catalog.priority);
+ const sub = active.subscribe(rendition, Catalog.PRIORITY.audio);
effect.cleanup(() => sub.close());
- // Create consumer with slightly less latency than the render worklet to avoid underflowing.
- // Container defaults to "legacy" via Zod schema for backward compatibility
- console.log(`[Audio Subscriber] Using container format: ${config.container}`);
+ if (config.container.kind === "cmaf") {
+ this.#runCmafDecoder(effect, active, sub, config);
+ } else {
+ this.#runLegacyDecoder(effect, sub, config);
+ }
+ }
+
+ #runLegacyDecoder(effect: Effect, sub: Moq.Track, config: Catalog.AudioConfig): void {
+ // Subtract audio worklet overhead from latency to avoid starving the render worklet
+ // TODO Make latency reactive
+ const latency = Math.max(this.latency.peek() - BUFFER_UNDERHEAD, 0) as Time.Milli;
+
const consumer = new Frame.Consumer(sub, {
- latency: Math.max(this.latency.peek() - JITTER_UNDERHEAD, 0) as Time.Milli,
- container: config.container,
+ latency,
});
effect.cleanup(() => consumer.close());
@@ -210,6 +268,66 @@ export class Source {
});
}
+ #runCmafDecoder(effect: Effect, _broadcast: Moq.Broadcast, sub: Moq.Track, config: Catalog.AudioConfig): void {
+ if (config.container.kind !== "cmaf") return;
+
+ const { timescale } = config.container;
+ const description = config.description ? Hex.toBytes(config.description) : undefined;
+
+ effect.spawn(async () => {
+ const loaded = await libav.polyfill();
+ if (!loaded) return; // cancelled
+
+ const decoder = new AudioDecoder({
+ output: (data) => this.#emit(data),
+ error: (error) => console.error(error),
+ });
+ effect.cleanup(() => decoder.close());
+
+ // Configure decoder with description from catalog
+ decoder.configure({
+ codec: config.codec,
+ sampleRate: config.sampleRate,
+ numberOfChannels: config.numberOfChannels,
+ description,
+ });
+
+ // Process data segments
+ // TODO: Use a consumer wrapper for CMAF to support latency control
+ for (;;) {
+ const group = await sub.nextGroup();
+ if (!group) break;
+
+ effect.spawn(async () => {
+ try {
+ for (;;) {
+ const segment = await group.readFrame();
+ if (!segment) break;
+
+ const samples = Mp4.decodeDataSegment(segment, timescale);
+
+ for (const sample of samples) {
+ this.#stats.update((stats) => ({
+ bytesReceived: (stats?.bytesReceived ?? 0) + sample.data.byteLength,
+ }));
+
+ const chunk = new EncodedAudioChunk({
+ type: sample.keyframe ? "key" : "delta",
+ data: sample.data,
+ timestamp: sample.timestamp,
+ });
+
+ decoder.decode(chunk);
+ }
+ }
+ } finally {
+ group.close();
+ }
+ });
+ }
+ });
+ }
+
#emit(sample: AudioData) {
const timestamp = sample.timestamp as Time.Micro;
@@ -245,5 +363,6 @@ export class Source {
close() {
this.#signals.close();
+ this.#latency.close();
}
}
diff --git a/js/hang/src/watch/backend.ts b/js/hang/src/watch/backend.ts
new file mode 100644
index 000000000..37fdca1c0
--- /dev/null
+++ b/js/hang/src/watch/backend.ts
@@ -0,0 +1,219 @@
+import type * as Moq from "@moq/lite";
+import { Effect, type Getter, Signal } from "@moq/signals";
+import type * as Catalog from "../catalog";
+import * as Audio from "./audio";
+import type { Broadcast } from "./broadcast";
+import * as MSE from "./mse";
+import * as Video from "./video";
+
+// Serializable representation of TimeRanges
+export interface BufferedRange {
+ start: number; // seconds
+ end: number; // seconds
+}
+export type BufferedRanges = BufferedRange[];
+
+// Helper to convert DOM TimeRanges
+export function timeRangesToArray(ranges: TimeRanges): BufferedRanges {
+ const result: BufferedRange[] = [];
+ for (let i = 0; i < ranges.length; i++) {
+ result.push({ start: ranges.start(i), end: ranges.end(i) });
+ }
+ return result;
+}
+
+export interface Backend {
+ // Additional buffer in milliseconds on top of the catalog's minBuffer.
+ // The effective latency = catalog.minBuffer + buffer.
+ buffer: Signal;
+
+ // Whether audio/video playback is paused.
+ paused: Signal;
+
+ // Whether the video is currently buffering, false when paused.
+ buffering: Getter;
+
+ // Current playback position in seconds.
+ timestamp: Getter;
+
+ // Video specific signals.
+ video: Video.Backend;
+
+ // Audio specific signals.
+ audio: Audio.Backend;
+}
+
+export interface CombinedProps {
+ element?: HTMLCanvasElement | HTMLVideoElement | Signal;
+ broadcast?: Broadcast | Signal;
+
+ // Additional buffer in milliseconds on top of the catalog's minBuffer.
+ buffer?: Moq.Time.Milli | Signal;
+ paused?: boolean | Signal;
+}
+
+// We have to proxy some of these signals because we support both the MSE and WebCodecs.
+class VideoSignals implements Video.Backend {
+ // The desired size/rendition/bitrate of the video.
+ target = new Signal(undefined);
+
+ // The catalog of the video.
+ catalog = new Signal(undefined);
+
+ // The name of the active rendition.
+ rendition = new Signal(undefined);
+
+ // The stats of the video.
+ stats = new Signal(undefined);
+
+ // The config of the active rendition.
+ config = new Signal(undefined);
+
+ // Buffered time ranges for MSE backend.
+ buffered = new Signal([]);
+}
+
+// Audio specific signals that work regardless of the backend source (mse vs webcodecs).
+class AudioSignals implements Audio.Backend {
+ // The volume of the audio, between 0 and 1.
+ volume = new Signal(0.5);
+
+ // Whether the audio is muted.
+ muted = new Signal(false);
+
+ // The desired rendition/bitrate of the audio.
+ target = new Signal(undefined);
+
+ // The catalog of the audio.
+ catalog = new Signal(undefined);
+
+ // The name of the active rendition.
+ rendition = new Signal(undefined);
+
+ // The config of the active rendition.
+ config = new Signal(undefined);
+
+ // The stats of the audio.
+ stats = new Signal(undefined);
+
+ // Buffered time ranges for MSE backend.
+ buffered = new Signal([]);
+}
+
+/// A generic backend that supports either MSE or WebCodecs based on the provided element.
+///
+/// This is primarily what backs the web component, but it's useful as a standalone for other use cases.
+export class Combined implements Backend {
+ element = new Signal(undefined);
+ broadcast: Signal;
+ buffer: Signal;
+ paused: Signal;
+
+ video = new VideoSignals();
+ audio = new AudioSignals();
+
+ #buffering = new Signal(false);
+ readonly buffering: Getter = this.#buffering;
+
+ #timestamp = new Signal(0);
+ readonly timestamp: Getter = this.#timestamp;
+
+ signals = new Effect();
+
+ constructor(props?: CombinedProps) {
+ this.element = Signal.from(props?.element);
+ this.broadcast = Signal.from(props?.broadcast);
+
+ this.buffer = Signal.from(props?.buffer ?? (100 as Moq.Time.Milli));
+ this.paused = Signal.from(props?.paused ?? false);
+
+ this.signals.effect(this.#runElement.bind(this));
+ }
+
+ #runElement(effect: Effect): void {
+ const element = effect.get(this.element);
+ if (!element) return;
+
+ if (element instanceof HTMLCanvasElement) {
+ this.#runWebcodecs(effect, element);
+ } else if (element instanceof HTMLVideoElement) {
+ this.#runMse(effect, element);
+ }
+ }
+
+ #runWebcodecs(effect: Effect, element: HTMLCanvasElement): void {
+ const videoSource = new Video.Source({
+ broadcast: this.broadcast,
+ buffer: this.buffer,
+ target: this.video.target,
+ });
+ const audioSource = new Audio.Source({
+ broadcast: this.broadcast,
+ buffer: this.buffer,
+ target: this.audio.target,
+ });
+
+ const audioEmitter = new Audio.Emitter(audioSource, {
+ volume: this.audio.volume,
+ muted: this.audio.muted,
+ paused: this.paused,
+ });
+
+ const videoRenderer = new Video.Renderer(videoSource, { canvas: element, paused: this.paused });
+
+ effect.cleanup(() => {
+ videoSource.close();
+ audioSource.close();
+ audioEmitter.close();
+ videoRenderer.close();
+ });
+
+ // Proxy the read only signals to the backend.
+ effect.proxy(this.video.catalog, videoSource.catalog);
+ effect.proxy(this.video.rendition, videoSource.rendition);
+ effect.proxy(this.video.config, videoSource.config);
+ effect.proxy(this.video.stats, videoSource.stats);
+ effect.proxy(this.video.buffered, videoSource.buffered);
+
+ effect.proxy(this.audio.catalog, audioSource.catalog);
+ effect.proxy(this.audio.rendition, audioSource.rendition);
+ effect.proxy(this.audio.config, audioSource.config);
+ effect.proxy(this.audio.stats, audioSource.stats);
+ effect.proxy(this.audio.buffered, audioSource.buffered);
+
+ // Derive timestamp from video stats (in lock-step with frame signal)
+ effect.effect((e) => {
+ const stats = e.get(videoSource.stats);
+ if (stats) {
+ this.#timestamp.set(stats.timestamp / 1_000_000); // microseconds to seconds
+ }
+ });
+ }
+
+ #runMse(effect: Effect, element: HTMLVideoElement): void {
+ const source = new MSE.Source({
+ broadcast: this.broadcast,
+ buffer: this.buffer,
+ element,
+ paused: this.paused,
+ video: { target: this.video.target },
+ audio: { volume: this.audio.volume, muted: this.audio.muted },
+ });
+ effect.cleanup(() => source.close());
+
+ // Proxy the read only signals to the backend.
+ effect.proxy(this.video.catalog, source.video.catalog);
+ effect.proxy(this.video.rendition, source.video.rendition);
+ effect.proxy(this.video.config, source.video.config);
+ effect.proxy(this.video.stats, source.video.stats);
+ effect.proxy(this.video.buffered, source.video.buffered);
+
+ effect.proxy(this.audio.catalog, source.audio.catalog);
+ effect.proxy(this.audio.rendition, source.audio.rendition);
+ effect.proxy(this.audio.config, source.audio.config);
+ effect.proxy(this.audio.stats, source.audio.stats);
+ effect.proxy(this.audio.buffered, source.audio.buffered);
+
+ effect.proxy(this.#timestamp, source.timestamp);
+ }
+}
diff --git a/js/hang/src/watch/broadcast.ts b/js/hang/src/watch/broadcast.ts
index c8fe30a1a..fcaa7a13e 100644
--- a/js/hang/src/watch/broadcast.ts
+++ b/js/hang/src/watch/broadcast.ts
@@ -1,13 +1,7 @@
import type * as Moq from "@moq/lite";
import { Effect, type Getter, Signal } from "@moq/signals";
import * as Catalog from "../catalog";
-import { PRIORITY } from "../publish/priority";
-import * as Audio from "./audio";
-import { Chat, type ChatProps } from "./chat";
-import * as Location from "./location";
-import { Preview, type PreviewProps } from "./preview";
-import * as User from "./user";
-import * as Video from "./video";
+import { PRIORITY } from "../catalog/priority";
export interface BroadcastProps {
connection?: Moq.Connection.Established | Signal;
@@ -21,16 +15,9 @@ export interface BroadcastProps {
// You can disable reloading if you don't want to wait for an announcement.
reload?: boolean | Signal;
-
- video?: Video.SourceProps;
- audio?: Audio.SourceProps;
- location?: Location.Props;
- chat?: ChatProps;
- preview?: PreviewProps;
- user?: User.Props;
}
-// A broadcast that (optionally) reloads automatically when live/offline.
+// A catalog source that (optionally) reloads automatically when live/offline.
export class Broadcast {
connection: Signal;
@@ -39,21 +26,14 @@ export class Broadcast {
status = new Signal<"offline" | "loading" | "live">("offline");
reload: Signal;
- audio: Audio.Source;
- video: Video.Source;
- location: Location.Root;
- chat: Chat;
- preview: Preview;
- user: User.Info;
-
- #broadcast = new Signal(undefined);
+ #active = new Signal(undefined);
+ readonly active: Getter = this.#active;
#catalog = new Signal(undefined);
readonly catalog: Getter = this.#catalog;
// This signal is true when the broadcast has been announced, unless reloading is disabled.
- #active = new Signal(false);
- readonly active: Getter = this.#active;
+ #announced = new Signal(false);
signals = new Effect();
@@ -62,12 +42,6 @@ export class Broadcast {
this.path = Signal.from(props?.path);
this.enabled = Signal.from(props?.enabled ?? false);
this.reload = Signal.from(props?.reload ?? true);
- this.audio = new Audio.Source(this.#broadcast, this.#catalog, props?.audio);
- this.video = new Video.Source(this.#broadcast, this.#catalog, props?.video);
- this.location = new Location.Root(this.#broadcast, this.#catalog, props?.location);
- this.chat = new Chat(this.#broadcast, this.#catalog, props?.chat);
- this.preview = new Preview(this.#broadcast, this.#catalog, props?.preview);
- this.user = new User.Info(this.#catalog, props?.user);
this.signals.effect(this.#runReload.bind(this));
this.signals.effect(this.#runBroadcast.bind(this));
@@ -81,7 +55,7 @@ export class Broadcast {
const reload = effect.get(this.reload);
if (!reload) {
// Mark as active without waiting for an announcement.
- effect.set(this.#active, true, false);
+ effect.set(this.#announced, true, false);
return;
}
@@ -105,7 +79,7 @@ export class Broadcast {
continue;
}
- effect.set(this.#active, update.active, false);
+ effect.set(this.#announced, update.active, false);
}
});
}
@@ -114,19 +88,19 @@ export class Broadcast {
const conn = effect.get(this.connection);
const enabled = effect.get(this.enabled);
const path = effect.get(this.path);
- const active = effect.get(this.#active);
- if (!conn || !enabled || path === undefined || !active) return;
+ const announced = effect.get(this.#announced);
+ if (!conn || !enabled || path === undefined || !announced) return;
const broadcast = conn.consume(path);
effect.cleanup(() => broadcast.close());
- effect.set(this.#broadcast, broadcast);
+ effect.set(this.#active, broadcast);
}
#runCatalog(effect: Effect): void {
if (!effect.get(this.enabled)) return;
- const broadcast = effect.get(this.#broadcast);
+ const broadcast = effect.get(this.active);
if (!broadcast) return;
this.status.set("loading");
@@ -158,12 +132,5 @@ export class Broadcast {
close() {
this.signals.close();
-
- this.audio.close();
- this.video.close();
- this.location.close();
- this.chat.close();
- this.preview.close();
- this.user.close();
}
}
diff --git a/js/hang/src/watch/element.ts b/js/hang/src/watch/element.ts
index 47e45d831..386fd84b6 100644
--- a/js/hang/src/watch/element.ts
+++ b/js/hang/src/watch/element.ts
@@ -1,12 +1,14 @@
import type { Time } from "@moq/lite";
import * as Moq from "@moq/lite";
-import { Effect, Signal } from "@moq/signals";
-import * as Audio from "./audio";
+import { Effect, type Getter, Signal } from "@moq/signals";
+import type * as Audio from "./audio";
+import { type Backend, Combined } from "./backend";
import { Broadcast } from "./broadcast";
-import * as Video from "./video";
+import type * as Video from "./video";
// TODO remove name; replaced with path
-const OBSERVED = ["url", "name", "path", "paused", "volume", "muted", "reload", "latency"] as const;
+// TODO remove latency; replaced with buffer
+const OBSERVED = ["url", "name", "path", "paused", "volume", "muted", "reload", "buffer", "latency"] as const;
type Observed = (typeof OBSERVED)[number];
// Close everything when this element is garbage collected.
@@ -15,7 +17,7 @@ type Observed = (typeof OBSERVED)[number];
const cleanup = new FinalizationRegistry((signals) => signals.close());
// An optional web component that wraps a
-export default class HangWatch extends HTMLElement {
+export default class HangWatch extends HTMLElement implements Backend {
static observedAttributes = OBSERVED;
// The connection to the moq-relay server.
@@ -24,43 +26,12 @@ export default class HangWatch extends HTMLElement {
// The broadcast being watched.
broadcast: Broadcast;
- // Responsible for rendering the video.
- video: Video.Renderer;
-
- // Responsible for emitting the audio.
- audio: Audio.Emitter;
-
- // The URL of the moq-relay server
- url = new Signal(undefined);
-
- // The path of the broadcast relative to the URL (may be empty).
- path = new Signal(undefined);
-
- // Whether audio/video playback is paused.
- paused = new Signal(false);
-
- // The volume of the audio, between 0 and 1.
- volume = new Signal(0.5);
-
- // Whether the audio is muted.
- muted = new Signal(false);
-
- // Whether the controls are shown.
- controls = new Signal(false);
-
- // Don't automatically reload the broadcast.
- // TODO: Temporarily defaults to false because Cloudflare doesn't support it yet.
- reload = new Signal(false);
-
- // Delay playing audio and video for up to 100ms
- latency = new Signal(100 as Time.Milli);
+ // The backend that powers this element.
+ #backend: Combined;
// Set when the element is connected to the DOM.
#enabled = new Signal(false);
- // The canvas element to render the video to.
- canvas = new Signal(undefined);
-
// Expose the Effect class, so users can easily create effects scoped to this element.
signals = new Effect();
@@ -70,43 +41,34 @@ export default class HangWatch extends HTMLElement {
cleanup.register(this, this.signals);
this.connection = new Moq.Connection.Reload({
- url: this.url,
enabled: this.#enabled,
});
this.signals.cleanup(() => this.connection.close());
this.broadcast = new Broadcast({
connection: this.connection.established,
- path: this.path,
enabled: this.#enabled,
- reload: this.reload,
- audio: {
- latency: this.latency,
- },
- video: {
- latency: this.latency,
- },
});
this.signals.cleanup(() => this.broadcast.close());
- this.video = new Video.Renderer(this.broadcast.video, { canvas: this.canvas, paused: this.paused });
- this.signals.cleanup(() => this.video.close());
-
- this.audio = new Audio.Emitter(this.broadcast.audio, {
- volume: this.volume,
- muted: this.muted,
- paused: this.paused,
+ this.#backend = new Combined({
+ broadcast: this.broadcast,
});
- this.signals.cleanup(() => this.audio.close());
// Watch to see if the canvas element is added or removed.
- const setCanvas = () => {
- this.canvas.set(this.querySelector("canvas") as HTMLCanvasElement | undefined);
+ const setElement = () => {
+ const canvas = this.querySelector("canvas") as HTMLCanvasElement | undefined;
+ const video = this.querySelector("video") as HTMLVideoElement | undefined;
+ if (canvas && video) {
+ throw new Error("Cannot have both canvas and video elements");
+ }
+ this.#backend.element.set(canvas ?? video);
};
- const observer = new MutationObserver(setCanvas);
+
+ const observer = new MutationObserver(setElement);
observer.observe(this, { childList: true, subtree: true });
this.signals.cleanup(() => observer.disconnect());
- setCanvas();
+ setElement();
// Optionally update attributes to match the library state.
// This is kind of dangerous because it can create loops.
@@ -131,7 +93,7 @@ export default class HangWatch extends HTMLElement {
});
this.signals.effect((effect) => {
- const muted = effect.get(this.muted);
+ const muted = effect.get(this.audio.muted);
if (muted) {
this.setAttribute("muted", "");
} else {
@@ -149,22 +111,13 @@ export default class HangWatch extends HTMLElement {
});
this.signals.effect((effect) => {
- const volume = effect.get(this.volume);
+ const volume = effect.get(this.audio.volume);
this.setAttribute("volume", volume.toString());
});
this.signals.effect((effect) => {
- const controls = effect.get(this.controls);
- if (controls) {
- this.setAttribute("controls", "");
- } else {
- this.removeAttribute("controls");
- }
- });
-
- this.signals.effect((effect) => {
- const latency = Math.floor(effect.get(this.latency));
- this.setAttribute("latency", latency.toString());
+ const buffer = Math.floor(effect.get(this.buffer));
+ this.setAttribute("buffer", buffer.toString());
});
}
@@ -194,18 +147,51 @@ export default class HangWatch extends HTMLElement {
this.paused.set(newValue !== null);
} else if (name === "volume") {
const volume = newValue ? Number.parseFloat(newValue) : 0.5;
- this.volume.set(volume);
+ this.audio.volume.set(volume);
} else if (name === "muted") {
- this.muted.set(newValue !== null);
+ this.audio.muted.set(newValue !== null);
} else if (name === "reload") {
- this.reload.set(newValue !== null);
- } else if (name === "latency") {
- this.latency.set((newValue ? Number.parseFloat(newValue) : 100) as Time.Milli);
+ this.broadcast.reload.set(newValue !== null);
+ } else if (name === "buffer" || name === "latency") {
+ // "latency" is a legacy alias for "buffer"
+ this.buffer.set((newValue ? Number.parseFloat(newValue) : 0) as Time.Milli);
} else {
const exhaustive: never = name;
throw new Error(`Invalid attribute: ${exhaustive}`);
}
}
+
+ get url(): Signal {
+ return this.connection.url;
+ }
+
+ get path(): Signal {
+ return this.broadcast.path;
+ }
+
+ get buffer(): Signal {
+ return this.#backend.buffer;
+ }
+
+ get paused(): Signal {
+ return this.#backend.paused;
+ }
+
+ get audio(): Audio.Backend {
+ return this.#backend.audio;
+ }
+
+ get video(): Video.Backend {
+ return this.#backend.video;
+ }
+
+ get buffering(): Getter {
+ return this.#backend.buffering;
+ }
+
+ get timestamp(): Getter {
+ return this.#backend.timestamp;
+ }
}
customElements.define("hang-watch", HangWatch);
diff --git a/js/hang/src/watch/index.ts b/js/hang/src/watch/index.ts
index 8531ce218..e485e26a2 100644
--- a/js/hang/src/watch/index.ts
+++ b/js/hang/src/watch/index.ts
@@ -1,4 +1,5 @@
export * as Audio from "./audio";
+export * from "./backend";
export * from "./broadcast";
export * as Chat from "./chat";
export * as Location from "./location";
diff --git a/js/hang/src/watch/mse/audio.ts b/js/hang/src/watch/mse/audio.ts
new file mode 100644
index 000000000..8e8f4334d
--- /dev/null
+++ b/js/hang/src/watch/mse/audio.ts
@@ -0,0 +1,274 @@
+import type * as Moq from "@moq/lite";
+import { Effect, type Getter, Signal } from "@moq/signals";
+import * as Catalog from "../../catalog";
+import * as Frame from "../../frame";
+import * as Mp4 from "../../mp4";
+import { Latency } from "../../util/latency";
+import type { Backend, Stats, Target } from "../audio/backend";
+import { type BufferedRanges, timeRangesToArray } from "../backend";
+import type { Broadcast } from "../broadcast";
+
+export type AudioProps = {
+ broadcast?: Broadcast | Signal;
+ element?: HTMLMediaElement | Signal;
+ mediaSource?: MediaSource | Signal;
+
+ // Additional buffer in milliseconds on top of the catalog's minBuffer.
+ buffer?: Moq.Time.Milli | Signal;
+ volume?: number | Signal;
+ muted?: boolean | Signal;
+ target?: Target | Signal;
+};
+
+export class Audio implements Backend {
+ broadcast: Signal;
+ element: Signal;
+ mediaSource: Signal;
+
+ volume: Signal;
+ muted: Signal;
+ target: Signal;
+ buffer: Signal;
+
+ #catalog = new Signal(undefined);
+ readonly catalog: Getter = this.#catalog;
+
+ #rendition = new Signal(undefined);
+ readonly rendition: Signal = this.#rendition;
+
+ #stats = new Signal(undefined);
+ readonly stats: Signal = this.#stats;
+
+ #config = new Signal(undefined);
+ readonly config: Signal = this.#config;
+
+ #buffered = new Signal([]);
+ readonly buffered: Getter = this.#buffered;
+
+ #selected = new Signal<{ track: string; mime: string; config: Catalog.AudioConfig } | undefined>(undefined);
+
+ #latency: Latency;
+ readonly latency: Getter;
+
+ #signals = new Effect();
+
+ constructor(props?: AudioProps) {
+ this.broadcast = Signal.from(props?.broadcast);
+ this.element = Signal.from(props?.element);
+ this.mediaSource = Signal.from(props?.mediaSource);
+
+ this.buffer = Signal.from(props?.buffer ?? (100 as Moq.Time.Milli));
+ this.volume = Signal.from(props?.volume ?? 0.5);
+ this.muted = Signal.from(props?.muted ?? false);
+ this.target = Signal.from(props?.target);
+
+ this.#latency = new Latency({
+ buffer: this.buffer,
+ config: this.config,
+ });
+ this.latency = this.#latency.combined;
+
+ this.#signals.effect(this.#runCatalog.bind(this));
+ this.#signals.effect(this.#runSelected.bind(this));
+ this.#signals.effect(this.#runMedia.bind(this));
+ this.#signals.effect(this.#runVolume.bind(this));
+ }
+
+ #runCatalog(effect: Effect): void {
+ const broadcast = effect.get(this.broadcast);
+ if (!broadcast) return;
+
+ const active = effect.get(broadcast.active);
+ if (!active) return;
+
+ const catalog = effect.get(broadcast.catalog)?.audio;
+ if (!catalog) return;
+
+ effect.set(this.#catalog, catalog);
+ }
+
+ #runSelected(effect: Effect): void {
+ const catalog = effect.get(this.#catalog);
+ if (!catalog) return;
+
+ const target = effect.get(this.target);
+
+ for (const [track, config] of Object.entries(catalog.renditions)) {
+ const mime = `audio/mp4; codecs="${config.codec}"`;
+ if (!MediaSource.isTypeSupported(mime)) continue;
+
+ // Support both CMAF and legacy containers
+ if (target?.name && track !== target.name) continue;
+
+ effect.set(this.#selected, { track, mime, config });
+ return;
+ }
+
+ console.warn(`[MSE] No supported audio rendition found:`, catalog.renditions);
+ }
+
+ #runMedia(effect: Effect): void {
+ const element = effect.get(this.element);
+ if (!element) return;
+
+ const mediaSource = effect.get(this.mediaSource);
+ if (!mediaSource) return;
+
+ const broadcast = effect.get(this.broadcast);
+ if (!broadcast) return;
+
+ const active = effect.get(broadcast.active);
+ if (!active) return;
+
+ const selected = effect.get(this.#selected);
+ if (!selected) return;
+
+ const sourceBuffer = mediaSource.addSourceBuffer(selected.mime);
+ effect.cleanup(() => {
+ mediaSource.removeSourceBuffer(sourceBuffer);
+ sourceBuffer.abort();
+ });
+
+ effect.event(sourceBuffer, "error", (e) => {
+ console.error("[MSE] SourceBuffer error:", e);
+ });
+
+ effect.event(sourceBuffer, "updateend", () => {
+ this.#buffered.set(timeRangesToArray(sourceBuffer.buffered));
+ });
+
+ if (selected.config.container.kind === "cmaf") {
+ this.#runCmafMedia(effect, active, selected, sourceBuffer, element);
+ } else {
+ this.#runLegacyMedia(effect, active, selected, sourceBuffer, element);
+ }
+ }
+
+ async #appendBuffer(sourceBuffer: SourceBuffer, buffer: Uint8Array): Promise {
+ while (sourceBuffer.updating) {
+ await new Promise((resolve) => sourceBuffer.addEventListener("updateend", resolve, { once: true }));
+ }
+ sourceBuffer.appendBuffer(buffer as BufferSource);
+ while (sourceBuffer.updating) {
+ await new Promise((resolve) => sourceBuffer.addEventListener("updateend", resolve, { once: true }));
+ }
+ }
+
+ #runCmafMedia(
+ effect: Effect,
+ active: Moq.Broadcast,
+ selected: { track: string; mime: string; config: Catalog.AudioConfig },
+ sourceBuffer: SourceBuffer,
+ element: HTMLMediaElement,
+ ): void {
+ if (selected.config.container.kind !== "cmaf") return;
+
+ const data = active.subscribe(selected.track, Catalog.PRIORITY.audio);
+ effect.cleanup(() => data.close());
+
+ effect.spawn(async () => {
+ // Generate init segment from catalog config (uses track_id from container)
+ const initSegment = Mp4.createAudioInitSegment(selected.config);
+ await this.#appendBuffer(sourceBuffer, initSegment);
+
+ for (;;) {
+ // TODO: Use Frame.Consumer for CMAF so we can support higher latencies.
+ // It requires extracting the timestamp from the frame payload.
+ const frame = await data.readFrame();
+ if (!frame) return;
+
+ await this.#appendBuffer(sourceBuffer, frame);
+
+ // Seek to the start of the buffer if we're behind it (for startup).
+ if (element.buffered.length > 0 && element.currentTime < element.buffered.start(0)) {
+ element.currentTime = element.buffered.start(0);
+ }
+ }
+ });
+ }
+
+ #runLegacyMedia(
+ effect: Effect,
+ active: Moq.Broadcast,
+ selected: { track: string; mime: string; config: Catalog.AudioConfig },
+ sourceBuffer: SourceBuffer,
+ element: HTMLMediaElement,
+ ): void {
+ const data = active.subscribe(selected.track, Catalog.PRIORITY.audio);
+ effect.cleanup(() => data.close());
+
+ // Create consumer that reorders groups/frames up to the provided latency.
+ // Legacy container uses microsecond timescale implicitly.
+ const consumer = new Frame.Consumer(data, {
+ latency: this.#latency.combined,
+ });
+ effect.cleanup(() => consumer.close());
+
+ effect.spawn(async () => {
+ // Generate init segment from catalog config (timescale = 1,000,000 = microseconds)
+ const initSegment = Mp4.createAudioInitSegment(selected.config);
+ await this.#appendBuffer(sourceBuffer, initSegment);
+
+ let sequence = 1;
+ let duration: Moq.Time.Micro | undefined;
+
+ // Buffer one frame so we can compute accurate duration from the next frame's timestamp
+ let pending = await consumer.decode();
+ if (!pending) return;
+
+ for (;;) {
+ const next = await consumer.decode();
+
+ // Compute duration from next frame's timestamp, or use last known duration if stream ended
+ if (next) {
+ duration = (next.timestamp - pending.timestamp) as Moq.Time.Micro;
+ }
+
+ // Wrap raw frame in moof+mdat
+ const segment = Mp4.encodeDataSegment({
+ data: pending.data,
+ timestamp: pending.timestamp,
+ duration: duration ?? 0, // Default to 0 duration if there's literally one frame then stream FIN.
+ keyframe: pending.keyframe,
+ sequence: sequence++,
+ });
+
+ await this.#appendBuffer(sourceBuffer, segment);
+
+ // Seek to the start of the buffer if we're behind it (for startup).
+ if (element.buffered.length > 0 && element.currentTime < element.buffered.start(0)) {
+ element.currentTime = element.buffered.start(0);
+ }
+
+ if (!next) return;
+ pending = next;
+ }
+ });
+ }
+
+ #runVolume(effect: Effect): void {
+ const element = effect.get(this.element);
+ if (!element) return;
+
+ const volume = effect.get(this.volume);
+ const muted = effect.get(this.muted);
+
+ if (muted && !element.muted) {
+ element.muted = true;
+ } else if (!muted && element.muted) {
+ element.muted = false;
+ }
+
+ if (volume !== element.volume) {
+ element.volume = volume;
+ }
+
+ effect.event(element, "volumechange", () => {
+ this.volume.set(element.volume);
+ });
+ }
+
+ close(): void {
+ this.#signals.close();
+ }
+}
diff --git a/js/hang/src/watch/mse/index.ts b/js/hang/src/watch/mse/index.ts
new file mode 100644
index 000000000..5959b4783
--- /dev/null
+++ b/js/hang/src/watch/mse/index.ts
@@ -0,0 +1,3 @@
+export * from "./audio";
+export * from "./source";
+export * from "./video";
diff --git a/js/hang/src/watch/mse/source.ts b/js/hang/src/watch/mse/source.ts
new file mode 100644
index 000000000..93fb352da
--- /dev/null
+++ b/js/hang/src/watch/mse/source.ts
@@ -0,0 +1,198 @@
+import type * as Moq from "@moq/lite";
+import { Effect, type Getter, Signal } from "@moq/signals";
+import type { Backend } from "../backend";
+import type { Broadcast } from "../broadcast";
+import { Audio, type AudioProps } from "./audio";
+import { Video, type VideoProps } from "./video";
+
+export type SourceProps = {
+ broadcast?: Broadcast | Signal;
+ // Additional buffer in milliseconds on top of the catalog's minBuffer.
+ buffer?: Moq.Time.Milli | Signal;
+ element?: HTMLMediaElement | Signal;
+ paused?: boolean | Signal;
+
+ video?: VideoProps;
+ audio?: AudioProps;
+};
+
+/**
+ * MSE-based video source for CMAF/fMP4 fragments.
+ * Uses Media Source Extensions to handle complete moof+mdat fragments.
+ */
+export class Source implements Backend {
+ broadcast: Signal;
+
+ #mediaSource = new Signal(undefined);
+
+ element: Signal;
+ buffer: Signal;
+ paused: Signal;
+
+ video: Video;
+ audio: Audio;
+
+ #buffering = new Signal(false);
+ readonly buffering: Getter = this.#buffering;
+
+ #timestamp = new Signal(0);
+ readonly timestamp: Getter = this.#timestamp;
+
+ #signals = new Effect();
+
+ constructor(props?: SourceProps) {
+ this.broadcast = Signal.from(props?.broadcast);
+ this.buffer = Signal.from(props?.buffer ?? (100 as Moq.Time.Milli));
+ this.element = Signal.from(props?.element);
+ this.paused = Signal.from(props?.paused ?? false);
+
+ this.video = new Video({
+ broadcast: this.broadcast,
+ element: this.element,
+ mediaSource: this.#mediaSource,
+ buffer: this.buffer,
+ ...props?.video,
+ });
+ this.audio = new Audio({
+ broadcast: this.broadcast,
+ element: this.element,
+ mediaSource: this.#mediaSource,
+ buffer: this.buffer,
+ ...props?.audio,
+ });
+
+ this.#signals.effect(this.#runMediaSource.bind(this));
+ this.#signals.effect(this.#runSkip.bind(this));
+ this.#signals.effect(this.#runTrim.bind(this));
+ this.#signals.effect(this.#runBuffering.bind(this));
+ this.#signals.effect(this.#runPaused.bind(this));
+ this.#signals.effect(this.#runTimestamp.bind(this));
+ }
+
+ #runMediaSource(effect: Effect): void {
+ const element = effect.get(this.element);
+ if (!element) return;
+
+ const mediaSource = new MediaSource();
+
+ element.src = URL.createObjectURL(mediaSource);
+ effect.cleanup(() => URL.revokeObjectURL(element.src));
+
+ effect.event(
+ mediaSource,
+ "sourceopen",
+ () => {
+ effect.set(this.#mediaSource, mediaSource);
+ },
+ { once: true },
+ );
+
+ effect.event(mediaSource, "error", (e) => {
+ console.error("[MSE] MediaSource error event:", e);
+ });
+ }
+
+ #runSkip(effect: Effect): void {
+ const element = effect.get(this.element);
+ if (!element) return;
+
+ // Don't skip when paused, otherwise we'll keep jerking forward.
+ const paused = effect.get(this.paused);
+ if (paused) return;
+
+ // Use video's computed latency (catalog minBuffer + user buffer)
+ const latency = effect.get(this.video.latency);
+
+ effect.interval(() => {
+ // Skip over gaps based on the effective latency.
+ const buffered = element.buffered;
+ if (buffered.length === 0) return;
+
+ const last = buffered.end(buffered.length - 1);
+ const diff = last - element.currentTime;
+
+ // We seek an extra 100ms because seeking isn't free/instant.
+ if (diff > latency && diff > 0.1) {
+ console.warn("skipping ahead", diff, "seconds");
+ element.currentTime += diff + 0.1;
+ }
+ }, 100);
+ }
+
+ #runTrim(effect: Effect): void {
+ const element = effect.get(this.element);
+ if (!element) return;
+
+ const mediaSource = effect.get(this.#mediaSource);
+ if (!mediaSource) return;
+
+ // Periodically clean up old buffered data.
+ effect.interval(async () => {
+ for (const sourceBuffer of mediaSource.sourceBuffers) {
+ while (sourceBuffer.updating) {
+ await new Promise((resolve) => sourceBuffer.addEventListener("updateend", resolve, { once: true }));
+ }
+
+ // Keep at least 10 seconds of buffered data to avoid removing I-frames.
+ if (element.currentTime > 10) {
+ sourceBuffer.remove(0, element.currentTime - 10);
+ }
+ }
+ }, 1000);
+ }
+
+ #runBuffering(effect: Effect): void {
+ const element = effect.get(this.element);
+ if (!element) return;
+
+ const update = () => {
+ this.#buffering.set(element.readyState <= HTMLMediaElement.HAVE_CURRENT_DATA);
+ };
+
+ // TODO Are these the correct events to use?
+ effect.event(element, "waiting", update);
+ effect.event(element, "playing", update);
+ effect.event(element, "seeking", update);
+ }
+
+ #runPaused(effect: Effect): void {
+ const element = effect.get(this.element);
+ if (!element) return;
+
+ const paused = effect.get(this.paused);
+ if (paused && !element.paused) {
+ element.pause();
+ } else if (!paused && element.paused) {
+ element.play().catch((e) => {
+ console.error("[MSE] MediaElement play error:", e);
+ this.paused.set(false);
+ });
+ }
+ }
+
+ #runTimestamp(effect: Effect): void {
+ const element = effect.get(this.element);
+ if (!element) return;
+
+ // Use requestVideoFrameCallback if available (frame-accurate)
+ if ("requestVideoFrameCallback" in element) {
+ const video = element as HTMLVideoElement;
+ let handle: number;
+ const onFrame = () => {
+ this.#timestamp.set(video.currentTime);
+ handle = video.requestVideoFrameCallback(onFrame);
+ };
+ handle = video.requestVideoFrameCallback(onFrame);
+ effect.cleanup(() => video.cancelVideoFrameCallback(handle));
+ } else {
+ // Fallback to timeupdate event
+ effect.event(element, "timeupdate", () => {
+ this.#timestamp.set(element.currentTime);
+ });
+ }
+ }
+
+ close(): void {
+ this.#signals.close();
+ }
+}
diff --git a/js/hang/src/watch/mse/video.ts b/js/hang/src/watch/mse/video.ts
new file mode 100644
index 000000000..32c66c672
--- /dev/null
+++ b/js/hang/src/watch/mse/video.ts
@@ -0,0 +1,251 @@
+import type * as Moq from "@moq/lite";
+import { Effect, type Getter, Signal } from "@moq/signals";
+import * as Catalog from "../../catalog";
+import * as Frame from "../../frame";
+import * as Mp4 from "../../mp4";
+import { Latency } from "../../util/latency";
+import { type BufferedRanges, timeRangesToArray } from "../backend";
+import type { Broadcast } from "../broadcast";
+import type { Backend, Stats, Target } from "../video/backend";
+
+export type VideoProps = {
+ broadcast?: Broadcast | Signal;
+ mediaSource?: MediaSource | Signal;
+ element?: HTMLMediaElement | Signal;
+
+ // Additional buffer in milliseconds on top of the catalog's minBuffer.
+ buffer?: Moq.Time.Milli | Signal;
+ target?: Target | Signal;
+};
+
+/**
+ * MSE-based video source for CMAF/fMP4 fragments.
+ * Uses Media Source Extensions to handle complete moof+mdat fragments.
+ */
+export class Video implements Backend {
+ broadcast: Signal;
+ element: Signal;
+ mediaSource: Signal;
+
+ // TODO Modify #select to use this signal.
+ target: Signal;
+ buffer: Signal;
+
+ #catalog = new Signal(undefined);
+ readonly catalog: Getter = this.#catalog;
+
+ #rendition = new Signal(undefined);
+ readonly rendition: Getter = this.#rendition;
+
+ // TODO implement stats
+ #stats = new Signal(undefined);
+ readonly stats: Getter = this.#stats;
+
+ #config = new Signal(undefined);
+ readonly config: Signal = this.#config;
+
+ #buffered = new Signal([]);
+ readonly buffered: Getter = this.#buffered;
+
+ // The selected rendition as a separate signal so we don't resubscribe until it changes.
+ #selected = new Signal<{ track: string; mime: string; config: Catalog.VideoConfig } | undefined>(undefined);
+
+ #latency: Latency;
+ readonly latency: Getter;
+
+ signals = new Effect();
+
+ constructor(props?: VideoProps) {
+ this.broadcast = Signal.from(props?.broadcast);
+ this.mediaSource = Signal.from(props?.mediaSource);
+ this.target = Signal.from(props?.target);
+ this.element = Signal.from(props?.element);
+ this.buffer = Signal.from(props?.buffer ?? (100 as Moq.Time.Milli));
+
+ this.#latency = new Latency({
+ buffer: this.buffer,
+ config: this.config,
+ });
+ this.latency = this.#latency.combined;
+
+ this.signals.effect(this.#runCatalog.bind(this));
+ this.signals.effect(this.#runSelected.bind(this));
+ this.signals.effect(this.#runMedia.bind(this));
+ }
+
+ #runCatalog(effect: Effect): void {
+ const broadcast = effect.get(this.broadcast);
+ if (!broadcast) return;
+
+ const catalog = effect.get(broadcast.catalog)?.video;
+ if (!catalog) return;
+
+ effect.set(this.#catalog, catalog);
+ }
+
+ #runSelected(effect: Effect): void {
+ const catalog = effect.get(this.#catalog);
+ if (!catalog) return;
+
+ const target = effect.get(this.target);
+
+ for (const [track, config] of Object.entries(catalog.renditions)) {
+ const mime = `video/mp4; codecs="${config.codec}"`;
+ if (!MediaSource.isTypeSupported(mime)) continue;
+
+ // Support both CMAF and legacy containers
+ if (target?.name && track !== target.name) continue;
+
+ effect.set(this.#selected, { track, mime, config });
+ return;
+ }
+
+ console.warn(`[MSE] No supported video rendition found:`, catalog.renditions);
+ }
+
+ #runMedia(effect: Effect): void {
+ const element = effect.get(this.element);
+ if (!element) return;
+
+ const mediaSource = effect.get(this.mediaSource);
+ if (!mediaSource) return;
+
+ const broadcast = effect.get(this.broadcast);
+ if (!broadcast) return;
+
+ const active = effect.get(broadcast.active);
+ if (!active) return;
+
+ // TODO Don't do a hard effect reload when this doesn't change the outcome.
+ const selected = effect.get(this.#selected);
+ if (!selected) return;
+
+ const sourceBuffer = mediaSource.addSourceBuffer(selected.mime);
+ effect.cleanup(() => {
+ mediaSource.removeSourceBuffer(sourceBuffer);
+ sourceBuffer.abort();
+ });
+
+ effect.event(sourceBuffer, "error", (e) => {
+ console.error("[MSE] SourceBuffer error:", e);
+ });
+
+ effect.event(sourceBuffer, "updateend", () => {
+ this.#buffered.set(timeRangesToArray(sourceBuffer.buffered));
+ });
+
+ if (selected.config.container.kind === "cmaf") {
+ this.#runCmafMedia(effect, active, selected, sourceBuffer, element);
+ } else {
+ this.#runLegacyMedia(effect, active, selected, sourceBuffer, element);
+ }
+ }
+
+ async #appendBuffer(sourceBuffer: SourceBuffer, buffer: Uint8Array): Promise {
+ while (sourceBuffer.updating) {
+ await new Promise((resolve) => sourceBuffer.addEventListener("updateend", resolve, { once: true }));
+ }
+
+ sourceBuffer.appendBuffer(buffer as BufferSource);
+
+ while (sourceBuffer.updating) {
+ await new Promise((resolve) => sourceBuffer.addEventListener("updateend", resolve, { once: true }));
+ }
+ }
+
+ #runCmafMedia(
+ effect: Effect,
+ active: Moq.Broadcast,
+ selected: { track: string; mime: string; config: Catalog.VideoConfig },
+ sourceBuffer: SourceBuffer,
+ element: HTMLMediaElement,
+ ): void {
+ if (selected.config.container.kind !== "cmaf") return;
+
+ const data = active.subscribe(selected.track, Catalog.PRIORITY.video);
+ effect.cleanup(() => data.close());
+
+ effect.spawn(async () => {
+ // Generate init segment from catalog config (uses track_id from container)
+ const initSegment = Mp4.createVideoInitSegment(selected.config);
+ await this.#appendBuffer(sourceBuffer, initSegment);
+
+ for (;;) {
+ // TODO: Use Frame.Consumer for CMAF so we can support higher latencies.
+ // It requires extracting the timestamp from the frame payload.
+ const frame = await data.readFrame();
+ if (!frame) return;
+
+ await this.#appendBuffer(sourceBuffer, frame);
+
+ // Seek to the start of the buffer if we're behind it (for startup).
+ if (element.buffered.length > 0 && element.currentTime < element.buffered.start(0)) {
+ element.currentTime = element.buffered.start(0);
+ }
+ }
+ });
+ }
+
+ #runLegacyMedia(
+ effect: Effect,
+ active: Moq.Broadcast,
+ selected: { track: string; mime: string; config: Catalog.VideoConfig },
+ sourceBuffer: SourceBuffer,
+ element: HTMLMediaElement,
+ ): void {
+ const data = active.subscribe(selected.track, Catalog.PRIORITY.video);
+ effect.cleanup(() => data.close());
+
+ // Create consumer that reorders groups/frames up to the provided latency.
+ // Legacy container uses microsecond timescale implicitly.
+ const consumer = new Frame.Consumer(data, {
+ latency: this.#latency.combined,
+ });
+ effect.cleanup(() => consumer.close());
+
+ effect.spawn(async () => {
+ // Generate init segment from catalog config (timescale = 1,000,000 = microseconds)
+ const initSegment = Mp4.createVideoInitSegment(selected.config);
+ await this.#appendBuffer(sourceBuffer, initSegment);
+
+ let sequence = 1;
+ let duration: Moq.Time.Micro | undefined;
+
+ // Buffer one frame so we can compute accurate duration from the next frame's timestamp
+ let pending = await consumer.decode();
+ if (!pending) return;
+
+ for (;;) {
+ const next = await consumer.decode();
+
+ // Compute duration from next frame's timestamp, or use last known duration if stream ended
+ if (next) {
+ duration = (next.timestamp - pending.timestamp) as Moq.Time.Micro;
+ }
+
+ // Wrap raw frame in moof+mdat
+ const segment = Mp4.encodeDataSegment({
+ data: pending.data,
+ timestamp: pending.timestamp,
+ duration: duration ?? 0, // Default to 0 duration if there's literally one frame then stream FIN.
+ keyframe: pending.keyframe,
+ sequence: sequence++,
+ });
+
+ await this.#appendBuffer(sourceBuffer, segment);
+
+ // Seek to the start of the buffer if we're behind it (for startup).
+ if (element.buffered.length > 0 && element.currentTime < element.buffered.start(0)) {
+ element.currentTime = element.buffered.start(0);
+ }
+
+ if (!next) return;
+ pending = next;
+ }
+ });
+ }
+
+ close(): void {
+ this.signals.close();
+ }
+}
diff --git a/js/hang/src/watch/video/backend.ts b/js/hang/src/watch/video/backend.ts
new file mode 100644
index 000000000..a3e5e6c21
--- /dev/null
+++ b/js/hang/src/watch/video/backend.ts
@@ -0,0 +1,40 @@
+import type { Getter, Signal } from "@moq/signals";
+import type * as Catalog from "../../catalog";
+import type { BufferedRanges } from "../backend";
+
+// Video specific signals that work regardless of the backend source (mse vs webcodecs).
+export interface Backend {
+ // The catalog of the video.
+ catalog: Getter;
+
+ // The desired size/rendition/bitrate of the video.
+ target: Signal;
+
+ // The name of the active rendition.
+ rendition: Getter;
+
+ // The stats of the video.
+ stats: Getter;
+
+ // The config of the active rendition.
+ config: Getter;
+
+ // Buffered time ranges (for MSE backend).
+ buffered: Getter;
+}
+
+export type Target = {
+ // Optional manual override for the selected rendition name.
+ name?: string;
+
+ // The desired size of the video in pixels.
+ pixels?: number;
+
+ // TODO bitrate
+};
+
+export interface Stats {
+ frameCount: number;
+ timestamp: number;
+ bytesReceived: number;
+}
diff --git a/js/hang/src/watch/video/index.ts b/js/hang/src/watch/video/index.ts
index 25cebd0e4..fccd6917a 100644
--- a/js/hang/src/watch/video/index.ts
+++ b/js/hang/src/watch/video/index.ts
@@ -1,2 +1,3 @@
+export type * from "./backend";
export * from "./renderer";
export * from "./source";
diff --git a/js/hang/src/watch/video/source.ts b/js/hang/src/watch/video/source.ts
index 406a600bb..44055e062 100644
--- a/js/hang/src/watch/video/source.ts
+++ b/js/hang/src/watch/video/source.ts
@@ -1,27 +1,31 @@
import type * as Moq from "@moq/lite";
import type { Time } from "@moq/lite";
import { Effect, type Getter, Signal } from "@moq/signals";
-import type * as Catalog from "../../catalog";
+import * as Catalog from "../../catalog";
import * as Frame from "../../frame";
-import { PRIORITY } from "../../publish/priority";
+import * as Mp4 from "../../mp4";
import * as Hex from "../../util/hex";
+import { Latency } from "../../util/latency";
+import type { BufferedRanges } from "../backend";
+import type { Broadcast } from "../broadcast";
+import type { Backend, Stats, Target } from "./backend";
-export type SourceProps = {
- enabled?: boolean | Signal;
+// Only count it as buffering if we had to sleep for 200ms or more before rendering the next frame.
+// Unfortunately, this has to be quite high because of b-frames.
+// TODO Maybe we need to detect b-frames and make this dynamic?
+const BUFFERING_MS = 200 as Time.Milli;
- // Jitter buffer size in milliseconds (default: 100ms)
- // When using b-frames, this should to be larger than the frame duration.
- latency?: Time.Milli | Signal;
-};
+export type SourceProps = {
+ broadcast: Broadcast | Signal;
-export type Target = {
- // The desired size of the video in pixels.
- pixels?: number;
+ enabled?: boolean | Signal;
- // Optional manual override for the selected rendition name.
- rendition?: string;
+ // Additional buffer in milliseconds on top of the catalog's minBuffer (default: 100ms).
+ // The effective latency = catalog.minBuffer + buffer
+ // Increase this if experiencing stuttering due to network jitter.
+ buffer?: Time.Milli | Signal;
- // TODO bitrate
+ target?: Target | Signal;
};
// The types in VideoDecoderConfig that cause a hard reload.
@@ -29,33 +33,16 @@ export type Target = {
// This way we can keep the current subscription active.
type RequiredDecoderConfig = Omit;
-type BufferStatus = { state: "empty" | "filled" };
-
-type SyncStatus = {
- state: "ready" | "wait";
- bufferDuration?: number;
-};
-
-export interface VideoStats {
- frameCount: number;
- timestamp: number;
- bytesReceived: number;
-}
-
-// Only count it as buffering if we had to sleep for 200ms or more before rendering the next frame.
-// Unfortunately, this has to be quite high because of b-frames.
-// TODO Maybe we need to detect b-frames and make this dynamic?
-const MIN_SYNC_WAIT_MS = 200 as Time.Milli;
-
// The maximum number of concurrent b-frames that we support.
const MAX_BFRAMES = 10;
// Responsible for switching between video tracks and buffering frames.
-export class Source {
- broadcast: Signal;
+export class Source implements Backend {
+ broadcast: Signal;
enabled: Signal; // Don't download any longer
- catalog = new Signal(undefined);
+ #catalog = new Signal(undefined);
+ readonly catalog: Getter = this.#catalog;
// The tracks supported by our video decoder.
#supported = new Signal>({});
@@ -65,67 +52,98 @@ export class Source {
#selectedConfig = new Signal(undefined);
// The name of the active rendition.
- active = new Signal(undefined);
+ #rendition = new Signal(undefined);
+ readonly rendition: Signal = this.#rendition;
+
+ // The config of the active rendition.
+ #config = new Signal(undefined);
+ config: Getter = this.#config;
// The current track running, held so we can cancel it when the new track is ready.
#pending?: Effect;
#active?: Effect;
// Used as a tiebreaker when there are multiple tracks (HD vs SD).
- target = new Signal(undefined);
+ target: Signal;
// Expose the current frame to render as a signal
- frame = new Signal(undefined);
+ #frame = new Signal(undefined);
+ readonly frame: Getter = this.#frame;
- // The target latency in milliseconds.
- latency: Signal;
+ // Additional buffer in milliseconds (on top of catalog's minBuffer).
+ buffer: Signal;
// The display size of the video in pixels, ideally sourced from the catalog.
- display = new Signal<{ width: number; height: number } | undefined>(undefined);
-
- // Whether to flip the video horizontally.
- flip = new Signal(undefined);
+ #display = new Signal<{ width: number; height: number } | undefined>(undefined);
+ readonly display: Getter<{ width: number; height: number } | undefined> = this.#display;
// Used to convert PTS to wall time.
#reference: DOMHighResTimeStamp | undefined;
- bufferStatus = new Signal({ state: "empty" });
- syncStatus = new Signal({ state: "ready" });
+ #buffering = new Signal(false);
+ readonly buffering: Getter = this.#buffering;
+
+ #stats = new Signal(undefined);
+ readonly stats: Getter = this.#stats;
+
+ // Empty stub for WebCodecs (no traditional buffering)
+ #buffered = new Signal([]);
+ readonly buffered: Getter = this.#buffered;
- #stats = new Signal(undefined);
- readonly stats: Getter = this.#stats;
+ #latency: Latency;
+ readonly latency: Getter;
#signals = new Effect();
- constructor(
- broadcast: Signal,
- catalog: Signal,
- props?: SourceProps,
- ) {
- this.broadcast = broadcast;
- this.latency = Signal.from(props?.latency ?? (100 as Time.Milli));
+ constructor(props?: SourceProps) {
+ this.broadcast = Signal.from(props?.broadcast);
+ this.buffer = Signal.from(props?.buffer ?? (100 as Time.Milli));
this.enabled = Signal.from(props?.enabled ?? false);
+ this.target = Signal.from(props?.target);
- this.#signals.effect((effect) => {
- const c = effect.get(catalog)?.video;
- effect.set(this.catalog, c);
- effect.set(this.flip, c?.flip);
+ this.#latency = new Latency({
+ buffer: this.buffer,
+ config: this.#config,
});
+ this.latency = this.#latency.combined;
+ this.#signals.effect(this.#runCatalog.bind(this));
this.#signals.effect(this.#runSupported.bind(this));
this.#signals.effect(this.#runSelected.bind(this));
this.#signals.effect(this.#runPending.bind(this));
this.#signals.effect(this.#runDisplay.bind(this));
- this.#signals.effect(this.#runBuffer.bind(this));
+ this.#signals.effect(this.#runBuffering.bind(this));
+ }
+
+ #runCatalog(effect: Effect): void {
+ const broadcast = effect.get(this.broadcast);
+ if (!broadcast) return;
+
+ const catalog = effect.get(broadcast.catalog);
+ if (!catalog) return;
+
+ effect.set(this.#catalog, catalog.video);
}
#runSupported(effect: Effect): void {
- const renditions = effect.get(this.catalog)?.renditions ?? {};
+ const renditions = effect.get(this.#catalog)?.renditions ?? {};
effect.spawn(async () => {
const supported: Record = {};
for (const [name, rendition] of Object.entries(renditions)) {
+ // For CMAF, we get description from init segment, so skip validation here
+ // and just check if the codec is supported
+ if (rendition.container.kind === "cmaf") {
+ const { supported: valid } = await VideoDecoder.isConfigSupported({
+ codec: rendition.codec,
+ optimizeForLatency: rendition.optimizeForLatency ?? true,
+ });
+ if (valid) supported[name] = rendition;
+ continue;
+ }
+
+ // Legacy container: validate with description from catalog
const description = rendition.description ? Hex.toBytes(rendition.description) : undefined;
const { supported: valid } = await VideoDecoder.isConfigSupported({
@@ -151,7 +169,7 @@ export class Source {
const supported = effect.get(this.#supported);
const target = effect.get(this.target);
- const manual = target?.rendition;
+ const manual = target?.name;
const selected = manual && manual in supported ? manual : this.#selectRendition(supported, target);
if (!selected) return;
@@ -160,6 +178,7 @@ export class Source {
// Remove the codedWidth/Height from the config to avoid a hard reload if nothing else has changed.
const config = { ...supported[selected], codedWidth: undefined, codedHeight: undefined };
effect.set(this.#selectedConfig, config);
+ effect.set(this.#config, config);
}
#runPending(effect: Effect): void {
@@ -167,13 +186,14 @@ export class Source {
const enabled = effect.get(this.enabled);
const selected = effect.get(this.#selected);
const config = effect.get(this.#selectedConfig);
+ const active = broadcast ? effect.get(broadcast.active) : undefined;
- if (!broadcast || !selected || !config || !enabled) {
+ if (!active || !selected || !config || !enabled) {
// Stop the active track.
this.#active?.close();
this.#active = undefined;
- this.frame.update((prev) => {
+ this.#frame.update((prev) => {
prev?.close();
return undefined;
});
@@ -188,22 +208,13 @@ export class Source {
// We use #pending here on purpose so we only close it when it hasn't caught up yet.
effect.cleanup(() => this.#pending?.close());
- this.#runTrack(this.#pending, broadcast, selected, config);
+ this.#runTrack(this.#pending, active, selected, config);
}
#runTrack(effect: Effect, broadcast: Moq.Broadcast, name: string, config: RequiredDecoderConfig): void {
- const sub = broadcast.subscribe(name, PRIORITY.video); // TODO use priority from catalog
+ const sub = broadcast.subscribe(name, Catalog.PRIORITY.video);
effect.cleanup(() => sub.close());
- // Create consumer that reorders groups/frames up to the provided latency.
- // Container defaults to "legacy" via Zod schema for backward compatibility
- console.log(`[Video Subscriber] Using container format: ${config.container}`);
- const consumer = new Frame.Consumer(sub, {
- latency: this.latency,
- container: config.container,
- });
- effect.cleanup(() => consumer.close());
-
// We need a queue because VideoDecoder doesn't block on a Promise returned by output.
// NOTE: We will drain this queue almost immediately, so the highWaterMark is just a safety net.
const queue = new TransformStream(
@@ -249,6 +260,7 @@ export class Source {
});
effect.cleanup(() => decoder.close());
+ // Output processing - same for both container types
effect.spawn(async () => {
for (;;) {
const { value: frame } = await reader.read();
@@ -262,11 +274,8 @@ export class Source {
this.#reference = ref;
// Don't sleep so we immediately render this frame.
} else {
- sleep = this.#reference - ref + this.latency.peek();
- }
-
- if (sleep > MIN_SYNC_WAIT_MS) {
- this.syncStatus.set({ state: "wait", bufferDuration: sleep });
+ const latency = this.latency.peek() ?? 0;
+ sleep = this.#reference - ref + latency;
}
if (sleep > 0) {
@@ -275,28 +284,38 @@ export class Source {
await new Promise((resolve) => setTimeout(resolve, sleep));
}
- if (sleep > MIN_SYNC_WAIT_MS) {
- // Include how long we slept if it was above the threshold.
- this.syncStatus.set({ state: "ready", bufferDuration: sleep });
- } else {
- this.syncStatus.set({ state: "ready" });
-
+ if (sleep <= BUFFERING_MS) {
// If the track switch was pending, complete it now.
if (this.#pending === effect) {
this.#active?.close();
this.#active = effect;
this.#pending = undefined;
- effect.set(this.active, name);
+ effect.set(this.rendition, name);
}
}
- this.frame.update((prev) => {
+ this.#frame.update((prev) => {
prev?.close();
return frame;
});
}
});
+ // Input processing - depends on container type
+ if (config.container.kind === "cmaf") {
+ this.#runCmafTrack(effect, broadcast, sub, config, decoder);
+ } else {
+ this.#runLegacyTrack(effect, sub, config, decoder);
+ }
+ }
+
+ #runLegacyTrack(effect: Effect, sub: Moq.Track, config: RequiredDecoderConfig, decoder: VideoDecoder): void {
+ // Create consumer that reorders groups/frames up to the provided latency.
+ const consumer = new Frame.Consumer(sub, {
+ latency: this.#latency.combined,
+ });
+ effect.cleanup(() => consumer.close());
+
decoder.configure({
...config,
description: config.description ? Hex.toBytes(config.description) : undefined,
@@ -328,6 +347,67 @@ export class Source {
});
}
+ #runCmafTrack(
+ effect: Effect,
+ _broadcast: Moq.Broadcast,
+ sub: Moq.Track,
+ config: RequiredDecoderConfig,
+ decoder: VideoDecoder,
+ ): void {
+ if (config.container.kind !== "cmaf") return;
+
+ const { timescale } = config.container;
+ const description = config.description ? Hex.toBytes(config.description) : undefined;
+
+ // Configure decoder with description from catalog
+ decoder.configure({
+ codec: config.codec,
+ description,
+ optimizeForLatency: config.optimizeForLatency ?? true,
+ // @ts-expect-error Only supported by Chrome, so the renderer has to flip manually.
+ flip: false,
+ });
+
+ effect.spawn(async () => {
+ // Process data segments
+ // TODO: Use a consumer wrapper for CMAF to support latency control
+ for (;;) {
+ const group = await sub.nextGroup();
+ if (!group) break;
+
+ effect.spawn(async () => {
+ try {
+ for (;;) {
+ const segment = await group.readFrame();
+ if (!segment) break;
+
+ const samples = Mp4.decodeDataSegment(segment, timescale);
+
+ for (const sample of samples) {
+ const chunk = new EncodedVideoChunk({
+ type: sample.keyframe ? "key" : "delta",
+ data: sample.data,
+ timestamp: sample.timestamp,
+ });
+
+ // Track stats
+ this.#stats.update((current) => ({
+ frameCount: (current?.frameCount ?? 0) + 1,
+ timestamp: sample.timestamp,
+ bytesReceived: (current?.bytesReceived ?? 0) + sample.data.byteLength,
+ }));
+
+ decoder.decode(chunk);
+ }
+ }
+ } finally {
+ group.close();
+ }
+ });
+ }
+ });
+ }
+
#selectRendition(renditions: Record, target?: Target): string | undefined {
const entries = Object.entries(renditions);
if (entries.length <= 1) return entries.at(0)?.[0];
@@ -365,12 +445,12 @@ export class Source {
}
#runDisplay(effect: Effect): void {
- const catalog = effect.get(this.catalog);
+ const catalog = effect.get(this.#catalog);
if (!catalog) return;
const display = catalog.display;
if (display) {
- effect.set(this.display, {
+ effect.set(this.#display, {
width: display.width,
height: display.height,
});
@@ -380,26 +460,31 @@ export class Source {
const frame = effect.get(this.frame);
if (!frame) return;
- effect.set(this.display, {
+ effect.set(this.#display, {
width: frame.displayWidth,
height: frame.displayHeight,
});
}
- #runBuffer(effect: Effect): void {
- const frame = effect.get(this.frame);
+ #runBuffering(effect: Effect): void {
const enabled = effect.get(this.enabled);
+ if (!enabled) return;
- const isBufferEmpty = enabled && !frame;
- if (isBufferEmpty) {
- this.bufferStatus.set({ state: "empty" });
- } else {
- this.bufferStatus.set({ state: "filled" });
+ const frame = effect.get(this.frame);
+ if (!frame) {
+ this.#buffering.set(true);
+ return;
}
+
+ this.#buffering.set(false);
+
+ effect.timer(() => {
+ this.#buffering.set(true);
+ }, BUFFERING_MS);
}
close() {
- this.frame.update((prev) => {
+ this.#frame.update((prev) => {
prev?.close();
return undefined;
});
diff --git a/js/lite/src/index.ts b/js/lite/src/index.ts
index cf64a4705..22d95662f 100644
--- a/js/lite/src/index.ts
+++ b/js/lite/src/index.ts
@@ -1,3 +1,4 @@
+export * as Signals from "@moq/signals";
export * from "./announced.ts";
export * from "./broadcast.ts";
export * as Connection from "./connection/index.ts";
diff --git a/js/lite/src/lite/connection.ts b/js/lite/src/lite/connection.ts
index a7ce8c922..c5bb3f07d 100644
--- a/js/lite/src/lite/connection.ts
+++ b/js/lite/src/lite/connection.ts
@@ -131,7 +131,7 @@ export class Connection implements Established {
// TODO use the session info
}
} finally {
- console.warn("session stream closed");
+ console.debug("session stream closed");
}
}
diff --git a/js/signals/src/index.ts b/js/signals/src/index.ts
index 467b884e6..bde85d302 100644
--- a/js/signals/src/index.ts
+++ b/js/signals/src/index.ts
@@ -554,4 +554,8 @@ export class Effect {
get cancel(): Promise {
return this.#stopped;
}
+
+ proxy(dst: Setter