diff --git a/AGENTS.md b/AGENTS.md index ad348883d11..c57b9021e18 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -112,6 +112,7 @@ The `quest` binary comes from the kixelated/quest flake input and serves the que | `rs/moq-relay` config/behavior | `doc/bin/relay/` | | `rs/moq-cli` | `doc/bin/cli.md` | | `rs/moq-gst` | `doc/bin/gstreamer.md` | +| `rs/moq-scene`, `rs/moq-studio` | `doc/bin/studio.md` | | `rs/moq-c` C ABI (`moq.h`) | `doc/lib/c` | | `js/{watch,publish}` UI/API | `demo/web` if it consumes the API | diff --git a/Cargo.lock b/Cargo.lock index 3f457cd227d..793737c7fff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,6 +18,15 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "366ffbaa4442f4684d91e2cd7c5ea7c4ed8add41959a31447066e279e432b618" +[[package]] +name = "accesskit" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3b7f7f85a7e5f68090000ed7622545829afd484d210358702ae4cb97dd0c320" +dependencies = [ + "uuid", +] + [[package]] name = "addr2line" version = "0.25.1" @@ -248,6 +257,26 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arboard" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeb9553e62312c8756a5979254fc6d7da8bbeb5986a1e17037fab932f7037917" +dependencies = [ + "clipboard-win", + "image", + "log", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-foundation 0.3.2", + "parking_lot", + "percent-encoding", + "windows-sys 0.61.2", + "x11rb 0.13.2", +] + [[package]] name = "arc-swap" version = "1.9.2" @@ -1058,6 +1087,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.12.1" @@ -1096,18 +1131,42 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "calloop" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b32313e937c8721c9d17ee9bf60699a6402b4b7c2cb6ac1bee0ffcaa4b4e5e" +dependencies = [ + "bitflags 2.13.2", + "polling", + "rustix 1.1.5", + "tracing", +] + [[package]] name = "calloop-wayland-source" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a66a987056935f7efce4ab5668920b5d0dac4a7c99991a67395f13702ddd20" dependencies = [ - "calloop", + "calloop 0.13.0", "rustix 0.38.44", "wayland-backend", "wayland-client", ] +[[package]] +name = "calloop-wayland-source" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138efcf0940a02ebf0cc8d1eff41a1682a46b431630f4c52450d6265876021fa" +dependencies = [ + "calloop 0.14.5", + "rustix 1.1.5", + "wayland-backend", + "wayland-client", +] + [[package]] name = "camino" version = "1.2.6" @@ -1239,6 +1298,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "cgl" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ced0551234e87afee12411d535648dd89d2e7f34c78b753395567aff3d447ff" +dependencies = [ + "libc", +] + [[package]] name = "chacha20" version = "0.10.2" @@ -1352,6 +1420,15 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" +[[package]] +name = "clipboard-win" +version = "5.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052af7878466feccda12ba7a6a67d019d712f296809559b47c12c8cc196d3a9a" +dependencies = [ + "error-code", +] + [[package]] name = "cmake" version = "0.1.58" @@ -1387,6 +1464,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "color" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ec7c5eb7a16992b1904d76c517d170ab353b0e0b3d5a0c81a8a0cd1037893cf" +dependencies = [ + "bytemuck", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -2257,6 +2343,17 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "duplicate" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e92f10a49176cbffacaedabfaa11d51db1ea0f80a83c26e1873b43cd1742c24" +dependencies = [ + "heck", + "proc-macro2", + "proc-macro2-diagnostics", +] + [[package]] name = "dyn-clone" version = "1.0.20" @@ -2308,6 +2405,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ecolor" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0656a7530b9e2841ee7a5f983965bdd9840c32f4012f759cfd44b912cced6b34" +dependencies = [ + "bytemuck", + "emath", +] + [[package]] name = "ed25519" version = "3.0.0" @@ -2335,6 +2442,127 @@ dependencies = [ "zeroize", ] +[[package]] +name = "eframe" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ad41fac6e3149abb4866e93df37dbd33baa95db525c3368c944584e95a5dc2" +dependencies = [ + "ahash", + "bytemuck", + "document-features", + "egui", + "egui-wgpu", + "egui-winit", + "egui_glow", + "glutin", + "glutin-winit", + "image", + "js-sys", + "log", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", + "objc2-foundation 0.3.2", + "parking_lot", + "percent-encoding", + "pollster", + "profiling", + "raw-window-handle", + "static_assertions", + "wasm-bindgen", + "web-sys", + "web-time", + "wgpu", + "windows-sys 0.61.2", + "winit", +] + +[[package]] +name = "egui" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc938cc27cd911415e1e4d151bc56ff9df063d8db081374e87aeb805ea74b2ba" +dependencies = [ + "accesskit", + "ahash", + "bitflags 2.13.2", + "emath", + "epaint", + "itertools 0.15.0", + "log", + "nohash-hasher", + "profiling", + "smallvec", + "unicode-segmentation", + "web-sys", +] + +[[package]] +name = "egui-wgpu" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cef1f6d1bba40b09a4d0d62cb822b81030922cfbf65c5373398c3a3caafc5bd" +dependencies = [ + "ahash", + "bytemuck", + "document-features", + "egui", + "epaint", + "log", + "profiling", + "thiserror 2.0.21", + "type-map", + "web-time", + "wgpu", + "winit", +] + +[[package]] +name = "egui-winit" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98466000559d66ba4db786a30448b01483946add4188e7f10f522a27b4fb89cc" +dependencies = [ + "arboard", + "bytemuck", + "egui", + "log", + "objc2 0.6.5", + "objc2-foundation 0.3.2", + "objc2-ui-kit 0.3.2", + "profiling", + "raw-window-handle", + "smithay-clipboard", + "web-time", + "winit", +] + +[[package]] +name = "egui_dock" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d9f668eb59ff0ab130c8bce71a513135e1d3017fb3103fb8758e9eea99d759f" +dependencies = [ + "duplicate", + "egui", + "paste", + "thiserror 2.0.21", +] + +[[package]] +name = "egui_glow" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "606ed4167daf341e5753d703285c4c1d562324b2c07aa8f405cd7d066fe58f3b" +dependencies = [ + "bytemuck", + "egui", + "glow", + "log", + "profiling", + "winit", +] + [[package]] name = "either" version = "1.18.0" @@ -2362,6 +2590,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "emath" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c8d4141cf0f60fb2f7aefaf352d3ffa5171ef0ab09731c64391801e95e48f1" +dependencies = [ + "bytemuck", +] + [[package]] name = "embedded-io" version = "0.4.0" @@ -2423,6 +2660,37 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "epaint" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0ec0308dc23130fc1a7d4623a8534c2ed069c6dc84e258ee4f3de99615aa1a3" +dependencies = [ + "ahash", + "bytemuck", + "ecolor", + "emath", + "epaint_default_fonts", + "font-types", + "harfrust", + "log", + "nohash-hasher", + "parking_lot", + "profiling", + "self_cell", + "skrifa", + "smallvec", + "unicode-general-category", + "unicode-segmentation", + "vello_cpu", +] + +[[package]] +name = "epaint_default_fonts" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773fa9c96dd0dbef887e39d0ed6177f141cce4d68e6041df77570aa3702dfa13" + [[package]] name = "equivalent" version = "1.0.2" @@ -2439,6 +2707,21 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "error-code" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5343afd4a8365a643ac588dab4cf234a190c7f6c88c9f6dd6ffe00837661b7" + +[[package]] +name = "euclid" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06" +dependencies = [ + "num-traits", +] + [[package]] name = "event-listener" version = "5.4.2" @@ -2498,6 +2781,27 @@ dependencies = [ "getrandom 0.4.3", ] +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "fearless_simd" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" + [[package]] name = "ff" version = "0.14.0" @@ -2573,6 +2877,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "font-types" +version = "0.12.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "835ae9d781464a79026cde387f9cbc650c9a4909d7cf7e658b127e3c35d2f35d" +dependencies = [ + "bytemuck", +] + [[package]] name = "foreign-types" version = "0.5.0" @@ -2922,6 +3235,22 @@ dependencies = [ "system-deps 9.0.0", ] +[[package]] +name = "glifo" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed4a1bb24121291d27230c1b1b44e07d6a9b28cefdb32fe1581dfb84e14f940a" +dependencies = [ + "bytemuck", + "foldhash", + "hashbrown 0.17.1", + "log", + "peniko", + "skrifa", + "smallvec", + "vello_common", +] + [[package]] name = "glob" version = "0.3.4" @@ -2952,6 +3281,63 @@ dependencies = [ "web-sys", ] +[[package]] +name = "glutin" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12124de845cacfebedff80e877bb37b5b75c34c5a4c89e47e1cdd67fb6041325" +dependencies = [ + "bitflags 2.13.2", + "cfg_aliases", + "cgl", + "dispatch2", + "glutin_egl_sys", + "glutin_glx_sys", + "glutin_wgl_sys", + "libloading 0.8.9", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", + "objc2-core-foundation", + "objc2-foundation 0.3.2", + "once_cell", + "raw-window-handle", + "wayland-sys", + "windows-sys 0.52.0", + "x11-dl", +] + +[[package]] +name = "glutin-winit" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85edca7075f8fc728f28cb8fbb111a96c3b89e930574369e3e9c27eb75d3788f" +dependencies = [ + "cfg_aliases", + "glutin", + "raw-window-handle", + "winit", +] + +[[package]] +name = "glutin_egl_sys" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c4680ba6195f424febdc3ba46e7a42a0e58743f2edb115297b86d7f8ecc02d2" +dependencies = [ + "gl_generator", + "windows-sys 0.52.0", +] + +[[package]] +name = "glutin_glx_sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7bb2938045a88b612499fbcba375a77198e01306f52272e692f8c1f3751185" +dependencies = [ + "gl_generator", + "x11-dl", +] + [[package]] name = "glutin_wgl_sys" version = "0.6.1" @@ -3112,6 +3498,15 @@ dependencies = [ "system-deps 9.0.0", ] +[[package]] +name = "guillotiere" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b17e70c989c36bad147b27a58d148c0741c51448aa5653436547323e524d0ab" +dependencies = [ + "euclid", +] + [[package]] name = "h2" version = "0.4.20" @@ -3199,6 +3594,18 @@ dependencies = [ "url", ] +[[package]] +name = "harfrust" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c03d949a14aa089bbb282f7dd76a498a7f684428e4257202efc119ec010376f9" +dependencies = [ + "bitflags 2.13.2", + "bytemuck", + "read-fonts", + "smallvec", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -3670,6 +4077,22 @@ dependencies = [ "xmltree", ] +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "moxcms", + "num-traits", + "png", + "tiff", + "zune-core", + "zune-jpeg", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -4224,6 +4647,18 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "kurbo" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b60dfc32f652b926df6192e55525b16d186c69d47876c3ead4da5cc9f8450e2" +dependencies = [ + "arrayvec", + "euclid", + "polycool", + "smallvec", +] + [[package]] name = "lazy_static" version = "1.5.1" @@ -4328,6 +4763,12 @@ dependencies = [ "semver-parser", ] +[[package]] +name = "linebender_resource_handle" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a5ff6bcca6c4867b1c4fd4ef63e4db7436ef363e0ad7531d1558856bae64f4" + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -4521,6 +4962,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -4597,6 +5039,7 @@ dependencies = [ "block2 0.6.2", "bytes", "cpal", + "criterion", "dispatch2", "fixed-resample", "hang", @@ -5300,6 +5743,29 @@ dependencies = [ "url", ] +[[package]] +name = "moq-scene" +version = "0.1.0" +dependencies = [ + "bytemuck", + "hang", + "humantime-serde", + "image", + "kio 0.6.2", + "moq-audio", + "moq-clock", + "moq-mux", + "moq-net", + "moq-video", + "serde", + "serde_with", + "tempfile", + "thiserror 2.0.21", + "tokio", + "toml 1.1.6+spec-1.1.0", + "tracing", +] + [[package]] name = "moq-shaper" version = "0.0.1" @@ -5366,6 +5832,31 @@ dependencies = [ "web-async", ] +[[package]] +name = "moq-studio" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "eframe", + "egui", + "egui_dock", + "moq-audio", + "moq-clock", + "moq-mux", + "moq-net", + "moq-relay", + "moq-scene", + "moq-tokio", + "moq-video", + "pollster", + "tempfile", + "tokio", + "tracing", + "url", + "usage-rs", +] + [[package]] name = "moq-token" version = "0.7.4" @@ -5604,6 +6095,16 @@ dependencies = [ "web-transport-wasm", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "mp4-atom" version = "0.16.2" @@ -6007,6 +6508,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + [[package]] name = "nom" version = "7.1.3" @@ -6336,6 +6843,19 @@ dependencies = [ "objc2-quartz-core 0.2.2", ] +[[package]] +name = "objc2-app-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-audio-toolbox" version = "0.3.2" @@ -6623,7 +7143,7 @@ checksum = "a1a1ae721c5e35be65f01a03b6d2ac13a54cb4fa70d8a5da293d7b0020261398" dependencies = [ "block2 0.5.1", "objc2 0.5.2", - "objc2-app-kit", + "objc2-app-kit 0.2.2", "objc2-foundation 0.2.2", ] @@ -6777,6 +7297,18 @@ dependencies = [ "objc2-user-notifications", ] +[[package]] +name = "objc2-ui-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87d638e33c06f577498cbcc50491496a3ed4246998a7fbba7ccb98b1e7eab22" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", + "objc2-core-foundation", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-uniform-type-identifiers" version = "0.2.2" @@ -7143,6 +7675,19 @@ dependencies = [ "base64ct", ] +[[package]] +name = "peniko" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "839c8299360d2e998bdb106dc0a6cd71dcc5f4df51df1b620361bf50e283cca6" +dependencies = [ + "bytemuck", + "color", + "kurbo", + "linebender_resource_handle", + "smallvec", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -7340,6 +7885,19 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags 2.13.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.9", +] + [[package]] name = "polling" version = "3.11.0" @@ -7360,6 +7918,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc6355899e1c9462875b6757c79f3caa011a1fdae12bbb1a2e72dd1f234f8336" +[[package]] +name = "polycool" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50596ddc09eb5ad5f75cacd40209568e66df71baf86e1499a0e99c4cff12a5a6" +dependencies = [ + "arrayvec", +] + [[package]] name = "polyval" version = "0.6.2" @@ -7534,6 +8101,18 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "version_check", +] + [[package]] name = "procfs" version = "0.18.0" @@ -7596,6 +8175,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + [[package]] name = "qlog" version = "0.18.1" @@ -7648,6 +8233,12 @@ dependencies = [ "web-transport-trait", ] +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quick-xml" version = "0.41.0" @@ -7930,6 +8521,17 @@ dependencies = [ "yasna 0.6.0", ] +[[package]] +name = "read-fonts" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046a7d674daf459825b32f5062056d6882db0d2f5a479fbd76ccfc870ac18709" +dependencies = [ + "bytemuck", + "font-types", + "once_cell", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -8635,7 +9237,7 @@ dependencies = [ "ab_glyph", "log", "memmap2", - "smithay-client-toolkit", + "smithay-client-toolkit 0.19.2", "tiny-skia", ] @@ -8783,6 +9385,12 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" +[[package]] +name = "self_cell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab42ca02749e120097e328d91d415325bdf43b1c72c4c8badf37375fe40a813" + [[package]] name = "semver" version = "1.0.28" @@ -9130,6 +9738,16 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" +[[package]] +name = "skrifa" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819ab7d62b1d3e72d9d9dea5650bac30424f9111364bb94928dbf5ecad1baa68" +dependencies = [ + "bytemuck", + "read-fonts", +] + [[package]] name = "slab" version = "0.4.12" @@ -9164,8 +9782,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3457dea1f0eb631b4034d61d4d8c32074caa6cd1ab2d59f2327bd8461e2c0016" dependencies = [ "bitflags 2.13.2", - "calloop", - "calloop-wayland-source", + "calloop 0.13.0", + "calloop-wayland-source 0.3.0", "cursor-icon", "libc", "log", @@ -9182,6 +9800,44 @@ dependencies = [ "xkeysym", ] +[[package]] +name = "smithay-client-toolkit" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0512da38f5e2b31201a93524adb8d3136276fa4fe4aafab4e1f727a82b534cc0" +dependencies = [ + "bitflags 2.13.2", + "calloop 0.14.5", + "calloop-wayland-source 0.4.1", + "cursor-icon", + "libc", + "log", + "memmap2", + "rustix 1.1.5", + "thiserror 2.0.21", + "wayland-backend", + "wayland-client", + "wayland-csd-frame", + "wayland-cursor", + "wayland-protocols", + "wayland-protocols-experimental", + "wayland-protocols-misc", + "wayland-protocols-wlr", + "wayland-scanner", + "xkeysym", +] + +[[package]] +name = "smithay-clipboard" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71704c03f739f7745053bde45fa203a46c58d25bc5c4efba1d9a60e9dba81226" +dependencies = [ + "libc", + "smithay-client-toolkit 0.20.0", + "wayland-backend", +] + [[package]] name = "smol_str" version = "0.2.2" @@ -9830,6 +10486,20 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + [[package]] name = "tikv-jemalloc-ctl" version = "0.7.0" @@ -10415,6 +11085,15 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "type-map" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb30dbbd9036155e74adad6812e9898d03ec374946234fbcebd5dfc7b9187b90" +dependencies = [ + "rustc-hash 2.1.3", +] + [[package]] name = "typenum" version = "1.20.1" @@ -10444,6 +11123,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + [[package]] name = "unicode-ident" version = "1.0.26" @@ -10833,6 +11518,33 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vello_common" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2e9aed918117e8152c9eddfd8362d73c465c23f26c44786aa331707b8a64fa2" +dependencies = [ + "bytemuck", + "fearless_simd", + "guillotiere", + "log", + "peniko", + "smallvec", + "thiserror 2.0.21", +] + +[[package]] +name = "vello_cpu" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac7349e1f55f6b801c7c277958df4ea53e7f20f21e8014910ad888b2ecda93ea" +dependencies = [ + "bytemuck", + "glifo", + "hashbrown 0.17.1", + "vello_common", +] + [[package]] name = "version-compare" version = "0.2.1" @@ -11098,6 +11810,32 @@ dependencies = [ "wayland-scanner", ] +[[package]] +name = "wayland-protocols-experimental" +version = "20250721.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40a1f863128dcaaec790d7b4b396cc9b9a7a079e878e18c47e6c2d2c5a8dcbb1" +dependencies = [ + "bitflags 2.13.2", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-misc" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e9567599ef23e09b8dad6e429e5738d4509dfc46b3b21f32841a304d16b29c8" +dependencies = [ + "bitflags 2.13.2", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + [[package]] name = "wayland-protocols-plasma" version = "0.3.12" @@ -11290,6 +12028,12 @@ dependencies = [ "nom 7.1.3", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "wgpu" version = "30.0.1" @@ -11348,6 +12092,7 @@ dependencies = [ "thiserror 2.0.21", "wgpu-core-deps-apple", "wgpu-core-deps-emscripten", + "wgpu-core-deps-wasm", "wgpu-core-deps-windows-linux-android", "wgpu-hal", "wgpu-naga-bridge", @@ -11372,6 +12117,15 @@ dependencies = [ "wgpu-hal", ] +[[package]] +name = "wgpu-core-deps-wasm" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d260ca2ca8adc654f513a869bf3cac0b47f5b8d3f87803d6942015f8de597aae" +dependencies = [ + "wgpu-hal", +] + [[package]] name = "wgpu-core-deps-windows-linux-android" version = "30.0.1" @@ -11874,7 +12628,7 @@ dependencies = [ "bitflags 2.13.2", "block2 0.5.1", "bytemuck", - "calloop", + "calloop 0.13.0", "cfg_aliases", "concurrent-queue", "core-foundation 0.9.4", @@ -11886,9 +12640,9 @@ dependencies = [ "memmap2", "ndk", "objc2 0.5.2", - "objc2-app-kit", + "objc2-app-kit 0.2.2", "objc2-foundation 0.2.2", - "objc2-ui-kit", + "objc2-ui-kit 0.2.2", "orbclient", "percent-encoding", "pin-project", @@ -11896,7 +12650,7 @@ dependencies = [ "redox_syscall 0.4.1", "rustix 0.38.44", "sctk-adwaita", - "smithay-client-toolkit", + "smithay-client-toolkit 0.19.2", "smol_str", "tracing", "unicode-segmentation", diff --git a/Cargo.toml b/Cargo.toml index 3cd19017fc8..2ac49ce1b72 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,10 +35,12 @@ default-members = [ "rs/moq-rtc", "rs/moq-rtmp", "rs/moq-rtsp", + "rs/moq-scene", "rs/moq-shaper", "rs/moq-sock", "rs/moq-srt", "rs/moq-stats", + "rs/moq-studio", "rs/moq-tokio", "rs/moq-transcode", "rs/moq-uring", @@ -146,6 +148,7 @@ moq-relay = { version = "0.17.2", path = "rs/moq-relay", default-features = fals moq-rtc = { version = "0.3.12", path = "rs/moq-rtc" } moq-rtmp = { version = "0.3.11", path = "rs/moq-rtmp" } moq-rtsp = { version = "0.1.0", path = "rs/moq-rtsp" } +moq-scene = { version = "0.1.0", path = "rs/moq-scene" } moq-sock = { version = "0.2.0", path = "rs/moq-sock" } moq-srt = { version = "0.3.11", path = "rs/moq-srt" } moq-stats = { version = "0.2.12", path = "rs/moq-stats" } diff --git a/doc/.vitepress/config.ts b/doc/.vitepress/config.ts index 4d321c6e98b..4492f72371c 100644 --- a/doc/.vitepress/config.ts +++ b/doc/.vitepress/config.ts @@ -145,6 +145,7 @@ export default defineConfig({ link: "/bin/cli", items: [{ text: "Inspect a relay", link: "/bin/inspect" }], }, + { text: "moq-studio", link: "/bin/studio" }, { text: "Gateways", items: [ diff --git a/doc/bin/index.md b/doc/bin/index.md index 6cfcfe7875a..487e71c93b5 100644 --- a/doc/bin/index.md +++ b/doc/bin/index.md @@ -16,6 +16,7 @@ Once connected, [Inspect a relay](/bin/inspect) shows what it is receiving. | --- | --- | | [moq-relay](/bin/relay/) | Route, cache, and fan out broadcasts. Authenticate with JWTs, cluster across regions, publish stats. | | [moq-cli](/bin/cli) | Move media in and out of MoQ: pipe FFmpeg in, play natively, capture a camera or screen, transcode, mint tokens. | +| [moq-studio](/bin/studio) | Compose cameras, screens, images, and other broadcasts into scenes on the GPU and go live. | ## Gateways diff --git a/doc/bin/studio.md b/doc/bin/studio.md new file mode 100644 index 00000000000..a64b9830f4c --- /dev/null +++ b/doc/bin/studio.md @@ -0,0 +1,105 @@ +--- +title: Studio +description: Compose scenes on the GPU and go live over MoQ from a native app +--- + +# Studio + +`moq-studio` is a native streaming studio for Linux, macOS, and Windows. It +composes cameras, screens, windows, images, colors, and broadcasts from a relay +into scenes on the GPU, mixes their audio, and publishes the result to a MoQ +relay. It is early and not packaged yet, so build it from source: + +```bash +cargo run --release -p moq-studio --features capture,openh264 -- \ + show.toml --connect https://cdn.moq.dev/anon --broadcast my-show +``` + +- `show.toml` is the scene collection. It is created on the first save. +- Without `--connect`, the studio composes and previews but publishes nothing, + and broadcast sources stay offline. +- `capture` adds camera, display, window, microphone, and system-audio sources. + On Linux it links ALSA; add `pipewire` for PipeWire cameras. `openh264` + is the software H.264 fallback for a machine without a hardware encoder. +- On Linux, display and window sources capture X11 and XWayland only; a + Wayland screen or window cannot be picked yet. +- The `--connect-*`, `--quic-*`, and `--log-level` flags are moq-cli's; see + `moq-studio --help`. + +## Panels + +Every panel is a tab you can drag, split, or stack. + +| Panel | Use it to | +| --- | --- | +| Preview, Program | See the scene you are editing and the one that is live. **Cut** puts the preview on program at once; **Fade** blends over the collection's fade time. | +| Scenes | Pick the scene in preview, add one, or remove one. | +| Items | Place, show, hide, and order the sources the preview scene draws, back to front. | +| Sources | Add a source by hand, see whether each is live, and remove one. | +| Devices | List this machine's cameras, displays, windows, and audio inputs, and add one as a source. | +| Audio | Set each audio source's gain and mute, and watch its meter and the program's. | +| Output | See the relay connection, whether anyone is watching, and the encoder's work. | + +A broadcast source subscribes through the same relay connection, plays the +best video and audio its catalog lists, switches when the catalog changes, and +comes back on its own when the publisher restarts. Audio from every source is mixed into the program whether +or not its scene is live; mute a source to leave it out. + +## Scene file + +The collection is TOML, so it diffs and reviews like code. It names devices, +files, and broadcast paths, never a relay URL or token. Check its image paths +and device IDs before sharing it, since they can name your home directory or +hardware. A file from another format version fails to load rather than being +guessed at. + +```toml +version = 1 + +[canvas] +width = 1920 +height = 1080 +framerate = 30 + +[transition] +fade = "300ms" + +[sources.camera] +kind = "camera" + +[sources.logo] +kind = "image" +path = "logo.png" # relative to the scene file + +[sources.guest] +kind = "broadcast" +path = "room/guest" + +[audio.guest] +gain = 0.5 + +[[scenes]] +name = "Main" +items = [ + { source = "camera", rect = { x = 0, y = 0, width = 1920, height = 1080 } }, + { source = "logo", rect = { x = 1600, y = 40, width = 280, height = 120 } }, +] +``` + +Source kinds are `camera`, `display`, and `microphone` (each with an optional +`id`, the default device without one), `window` (with an `id`), `system_audio` +(macOS), `image` (PNG or JPEG), `color` (`#rrggbb` or `#rrggbbaa`), and +`broadcast`. The Devices panel fills in ids. **File > Save** (Ctrl+S) writes +the file and **File > Reload** reads it back; a new canvas size or frame rate +takes a restart. + +## Output + +The program publishes as one H.264 rendition at the canvas size and frame rate, +with Opus stereo audio, under the `--broadcast` path. Timestamps come from the +studio's own clock, so a source that stalls repeats its last picture instead of +holding up the stream. + +Video is only encoded while someone watches it. Each frame is read back from +the GPU to the CPU for the encoder, which the Output panel counts; frames the +encoder cannot keep up with are dropped there rather than delaying the stream. diff --git a/doc/setup/repository.md b/doc/setup/repository.md index a7fbad11d29..fa64a993050 100644 --- a/doc/setup/repository.md +++ b/doc/setup/repository.md @@ -27,6 +27,7 @@ wrap the Rust core. [Layers](/lib/layers) maps the binding namespaces. | Browser capture, playback, and controls | [`js/publish`](https://github.com/moq-dev/moq/tree/main/js/publish), [`js/watch`](https://github.com/moq-dev/moq/tree/main/js/watch), [`js/video`](https://github.com/moq-dev/moq/tree/main/js/video), [`js/ui`](https://github.com/moq-dev/moq/tree/main/js/ui) | [TypeScript](/lib/js/) | | Browser reactive state and cleanup | [`js/signals`](https://github.com/moq-dev/moq/tree/main/js/signals) | [Signals](/lib/js/signals) | | Native capture, codecs, and rendering | [`rs/moq-audio`](https://github.com/moq-dev/moq/tree/main/rs/moq-audio), [`rs/moq-video`](https://github.com/moq-dev/moq/tree/main/rs/moq-video), [`rs/moq-transcode`](https://github.com/moq-dev/moq/tree/main/rs/moq-transcode) | [Audio](/lib/rs/moq-audio), [video](/lib/rs/moq-video) | +| The native studio: scenes, compositing, and its window | [`rs/moq-scene`](https://github.com/moq-dev/moq/tree/main/rs/moq-scene), [`rs/moq-studio`](https://github.com/moq-dev/moq/tree/main/rs/moq-studio) | [Studio](/bin/studio) | | CLI commands and protocol gateways | [`rs/moq-cli`](https://github.com/moq-dev/moq/tree/main/rs/moq-cli), then the corresponding `rs/moq-{rtmp,srt,rtc,rtsp,hls}` crate | [Applications](/bin/) | | Language bindings | [`rs/moq-ffi`](https://github.com/moq-dev/moq/tree/main/rs/moq-ffi), [`rs/moq-c`](https://github.com/moq-dev/moq/tree/main/rs/moq-c), then `py/`, `swift/`, `kt/`, `go/`, `dart/`, or `cpp/moq/` | [Binding layers](/lib/layers) | | Rooms, chat, and application data | [`rs/moq-room`](https://github.com/moq-dev/moq/tree/main/rs/moq-room), [`js/room`](https://github.com/moq-dev/moq/tree/main/js/room), `rs/moq-json`, `js/json` | [Rooms](/lib/js/room), [JSON tracks](/lib/js/json) | diff --git a/quest/m1/README.md b/quest/m1/README.md index 466f1cb331d..45cedadd5d7 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -104,6 +104,12 @@ Added 2026-10-25 (maintainer): pure-Rust crypto moved up from m2, so the cut ships a relay that compiles no C. It follows moq-quic's crypto reshape, whose surface its QUIC adapter plugs into. +Added 2026-10-25 (maintainer): the studio's +[GPU encode](/quest/m1/studio-gpu/encode.md) and +[source import](/quest/m1/studio-gpu/import.md) quests are in m1 so +moq-video's GPU encode and import API changes, mainly encoders taking a GPU +surface and zero-copy texture import, land before the cut. + ## Required - [moq-congestion](/quest/m1/congestion/README.md) - one QUIC congestion-control crate with a transport-independent trait, run by `moq-quic` and published for noq @@ -128,6 +134,7 @@ whose surface its QUIC adapter plugs into. - [More tests under load](/quest/m1/test-flakes-2/README.md) - the second round of load-only failures, one quest per flake, fixed at the cause - [Broadcast lifecycle](/quest/m1/broadcast-lifecycle/README.md) - broadcasts are online or offline with no finished state in moq-net, finished-broadcast retention is a fixed moq-mux policy, and each linger timer has one meaning - [A smaller, safer API than release](/quest/m1/api-audit/README.md) - every surface added since release has a consumer or is gone, overlapping operations are collapsed, names mirror across Rust, JS, and the bindings, and every break says how to migrate +- [Studio on the GPU](/quest/m1/studio-gpu/README.md) - moq-video encoders take a GPU surface and its renderer reports zero-copy import, proven through moq-studio on NVENC, VideoToolbox, Media Foundation, DMA-BUF, and Metal - [moq-transport 18 and 22 only](/quest/m1/ietf-drop/README.md) - drafts 14-17 and 19-21 are deleted from Rust, JS, tests, interop lanes, and docs, removing the legacy control-stream adapter and its request limit - [moq-quic's API improves on quinn's](/quest/m1/quic-api/README.md) - moq-quic and both QUIC drivers ship one mirrored API: `send::`/`recv::` namespaces, one error per direction, consuming terminal operations, plain configs, no quinn baggage - [Pure-Rust crypto for moq-relay](/quest/m1/pure-crypto/README.md) - moq-relay's release builds use graviola and RustCrypto and compile no C diff --git a/quest/m1/studio-gpu/README.md b/quest/m1/studio-gpu/README.md new file mode 100644 index 00000000000..951a5708e6d --- /dev/null +++ b/quest/m1/studio-gpu/README.md @@ -0,0 +1,31 @@ +# Studio on the GPU + +## Goal + +`moq-studio`'s pictures stay on the GPU from source to encoder on the +backends proven before the cut: sources reach the compositor as textures on +the shared device, and the program reaches the encoder as a GPU surface. A +CPU copy remains only as a counted fallback, visible in the studio's Output +panel. + +## Plan + +Decided 2026-10-25 (maintainer): in m1, so moq-video's GPU encode and import +API changes land before the cut. Each piece lands in moq-video or moq-scene, +never in the app, so other consumers inherit it. + +Scope is the API shape plus the backends provable now. AMD Vulkan Video +encode, Intel VA-API on an external Vulkan image, and Direct3D11 render +import stay in m2 as Related follow-ups; until they land, those backends take +the counted CPU path. Nothing here requires m2 or m3 work, including the +[macOS and Windows runs](/quest/m3/studio/app.md): moq-video and moq-scene +test each platform's path on that platform's CI. + +## Required + +- [Zero-copy program encode](/quest/m1/studio-gpu/encode.md) - the program reaches the encoder as a GPU surface instead of a CPU readback +- [Zero-copy source import](/quest/m1/studio-gpu/import.md) - captured and decoded frames reach the compositor as GPU textures, with uploads counted + +## Related + +- [Studio](/quest/m3/studio/README.md) - the app these quests serve diff --git a/quest/m1/studio-gpu/encode.md b/quest/m1/studio-gpu/encode.md new file mode 100644 index 00000000000..dfd18de62d0 --- /dev/null +++ b/quest/m1/studio-gpu/encode.md @@ -0,0 +1,36 @@ +# [L] Zero-copy program encode + +## Goal + +`moq-studio`'s program reaches moq-video's encoder as a GPU `Surface` on +NVENC, VideoToolbox, and Media Foundation, instead of the CPU readback every +frame takes today. Any other backend keeps the readback, which the Output +panel already counts. + +## Plan + +- The compositor's program texture is allocated as an exportable image on + the device the window and sources share, then handed to the encoder + without leaving the GPU. Each encoder below already accepts its surface; + the work is the export and the API that carries it: + - NVENC: a `moq_video::frame::vulkan` slot (opaque-fd memory and a timeline + semaphore), the path the NVDEC to NVENC transcode already uses. + - VideoToolbox: an IOSurface-backed `CVPixelBuffer` the Metal texture aliases. + - Media Foundation: a shared D3D11 texture. +- This reaches below `wgpu` (its `as_hal` escape hatch), so keep the export in + moq-video or moq-scene, never in the app. +- Shape the export so the Vulkan slot also serves VA-API and AMD Vulkan + Video later without an API change; wiring those backends is m2 work. +- **Tests:** extend `just rs render-sw`'s end-to-end test (a red canvas + published to a test relay and decoded) to assert the readback count, which + is nonzero there. VideoToolbox and Media Foundation assert no readback on + their platforms' CI, and NVENC in an ignored hardware test run by + `just rs gpu`. + +## Related + +- [Zero-copy source import](/quest/m1/studio-gpu/import.md) - the other half of keeping the studio's pictures on the GPU +- [VA-API encodes an external Vulkan image](/quest/m2/vaapi-vulkan-import.md) - the follow-up that gives Intel the same Vulkan slot +- [Vulkan Video encode on AMD](/quest/m2/vulkan-encode.md) - the follow-up that gives AMD the same Vulkan slot +- [NVENC reuses its input and output buffers](/quest/m2/nvenc-pool.md) - per-frame NVENC allocation the program encode also pays +- [Hardware validation](/quest/m3/video-hardware.md) - capture and zero-copy paths not yet run on real machines diff --git a/quest/m1/studio-gpu/import.md b/quest/m1/studio-gpu/import.md new file mode 100644 index 00000000000..aceb86e3ce7 --- /dev/null +++ b/quest/m1/studio-gpu/import.md @@ -0,0 +1,38 @@ +# [M] Zero-copy source import + +## Goal + +Captured and decoded frames reach `moq-studio`'s compositor as GPU textures +on the shared device, imported rather than uploaded, on Linux (PipeWire and +VA-API DMA-BUFs) and Apple (IOSurface-backed `CVPixelBuffer`s through Metal). +moq-video's renderer reports whether each frame was imported or uploaded, and +the studio counts uploads per source next to its readback stats. + +## Plan + +- moq-scene draws every frame through moq-video's `Renderer`, which already + imports DMA-BUF (`render/dmabuf.rs`) and Metal (`render/metal.rs`) surfaces + without a copy, and decoded broadcast sources already take the decoder's + native surface. What remains: + - The renderer API reports import or upload per frame, so a caller can see + its fallback. This is the moq-video API change the cut should carry. + - moq-scene's `capture` does not enable moq-video's `pipewire` feature + (off by default because it links libpipewire), so Linux display and + camera sources arrive as CPU frames. Decide its build default with + [packaging](/quest/m3/studio/packaging.md) in mind. +- Windows sources keep the counted upload until + [Direct3D11 render import](/quest/m2/render-d3d11.md) lands in m2. +- The still-image `queue.write_texture` in `rs/moq-scene/src/compositor.rs` + runs once per image and is not part of this. +- Keep the import in moq-video or moq-scene, never in the app. +- **Tests:** the compositor's lavapipe tests assert the upload count for a CPU + frame, macOS CI asserts a Metal import, and a PipeWire DMA-BUF import is an + ignored hardware test. + +## Related + +- [Zero-copy program encode](/quest/m1/studio-gpu/encode.md) - the other half of keeping the studio's pictures on the GPU +- [Validate PipeWire DMA-BUFs into the Vulkan renderer](/quest/m2/2819-moq-video-carry-pipewire-dma-bufs-safely-into-the-vulkan.md) - the hardware proof of the Linux import path +- [Direct3D11 render import](/quest/m2/render-d3d11.md) - the follow-up that brings Windows sources onto the GPU +- [VAAPI encode and decode](/quest/m2/video-vaapi.md) - decoded DMA-BUF surfaces the renderer imports +- [Hardware validation](/quest/m3/video-hardware.md) - capture and zero-copy paths not yet run on real machines diff --git a/quest/m3/studio/README.md b/quest/m3/studio/README.md index 42d2dfa9b66..4b7de4afbee 100644 --- a/quest/m3/studio/README.md +++ b/quest/m3/studio/README.md @@ -52,17 +52,24 @@ Decided 2026-10-22 with the maintainer (/quest-plan): owns, through a web join link or QR code. - **Naming:** quests name no streaming platforms; ingest requirements are written as generic constraints. -- **Docs:** inline. [The app](/quest/m3/studio/app.md) writes - `doc/bin/studio.md`, and each later quest extends it. +- **Docs:** inline. Each quest extends `doc/bin/studio.md`. Dropped from the first scoping round: relay fan-out ("encode once, the relay restreams") and its upload savings. Direct push with shared encoders matches OBS multistream, and the hosted version is product work outside this repository. +Added 2026-10-25 (maintainer): the studio's +[GPU encode](/quest/m1/studio-gpu/encode.md) and +[source import](/quest/m1/studio-gpu/import.md) quests are in m1 so +moq-video's GPU encode and import API changes, mainly encoders taking a GPU +surface and zero-copy texture import, land before the cut. + ## Required -- [The studio app](/quest/m3/studio/app.md) - egui studio with a minimal moq-scene and program bus: one canvas, scenes of capture, image, and MoQ sources, published to MoQ +- [Zero-copy program encode](/quest/m1/studio-gpu/encode.md) - the program reaches the encoder as a GPU surface instead of a CPU readback +- [Zero-copy source import](/quest/m1/studio-gpu/import.md) - captured and decoded frames reach the compositor as GPU textures instead of a CPU upload +- [Studio on macOS and Windows](/quest/m3/studio/app.md) - the studio runs on macOS and Windows, not only Linux - [Destinations](/quest/m3/studio/destinations.md) - RTMP, RTMPS, and WHIP push with per-destination codecs and shared encoders, on a UI-free egress library a hosted service can reuse - [Multiple canvases](/quest/m3/studio/multi-canvas.md) - landscape and portrait canvases composite at once from shared sources - [Guests](/quest/m3/studio/guests.md) - remote guests and phone cameras join through a moq-room link or QR code and appear as sources diff --git a/quest/m3/studio/app.md b/quest/m3/studio/app.md index af9f1a10aa0..a9f1da42464 100644 --- a/quest/m3/studio/app.md +++ b/quest/m3/studio/app.md @@ -1,46 +1,23 @@ -# [XL] The studio app +# [M] Studio on macOS and Windows ## Goal -`moq-studio` runs on Linux, macOS, and Windows with docked panels: -- scenes and sources -- a preview and program pair with cut and fade -- device pickers -- an audio bus panel with gain and mute - -Sources are a camera, display or window capture, a microphone, system audio -where moq-audio captures it (macOS today; the Linux and Windows capture quests -add the rest), an image, a solid color, and a subscribed MoQ broadcast. One canvas -publishes to a MoQ relay. A scene collection saves to a diffable file and -reloads. - -`rs/moq-scene` (scene graph, sources, compositing) and moq-audio's program -bus land here, with the app as their consumer. Their API surface is only what -the app uses. +`moq-studio` is run, not only compiled, on macOS and Windows, and whatever +only shows up there is fixed. Only Linux was built and run when the studio +landed. ## Plan -- **Rendering:** - - One wgpu device is shared by moq-video's render import paths (dmabuf, - Metal) and egui-wgpu. - - Preview and program are textures registered with egui, with no readback. - - Program frames reach moq-video's encoder as a GPU `Surface` wherever the - backend accepts one. Any CPU readback fallback shows in stats. -- **Timing:** output timestamps come from the program clock, not from source - timestamps. A stalled source repeats its last frame rather than stalling - the canvas. -- **Scene file:** choose the format in this quest. It must be diffable and - carry a schema version, and loading an unknown version fails loudly. It - never holds credentials. -- **Tests:** compositing tests render on a wgpu fallback adapter (llvmpipe or - WARP), so they run in CI without a GPU. -- **Same-PR updates:** - - `doc/bin/studio.md` - - an AGENTS.md cross-package row for `rs/moq-studio` and `rs/moq-scene` - - the repository map in `doc/setup/repository.md` -- **Splitting:** if this proves too large for one PR, split along moq-scene - versus app, as long as each piece still lands with a consumer. +- **macOS:** camera, display, microphone, and system audio sources, plus the + window, preview, program, and publish path. +- **Windows:** camera, display, window, and microphone sources, plus the + window, preview, program, and publish path. System audio is `Unsupported` + there until [Windows capture parity](/quest/m2/capture-windows.md) adds + WASAPI loopback, so that quest runs it. +- Build with `--features capture`, since a default build has image, color, + and broadcast sources only. ## Related -- [Hardware validation](/quest/m3/video-hardware.md) - capture and zero-copy paths not yet run on real machines +- [Zero-copy program encode](/quest/m1/studio-gpu/encode.md) - the GPU encode path, which lands without this run +- [Hardware validation](/quest/m3/video-hardware.md) - capture paths not yet run on real machines diff --git a/quest/m3/studio/audio-filters.md b/quest/m3/studio/audio-filters.md index a819d02ae60..99a5e7ed53c 100644 --- a/quest/m3/studio/audio-filters.md +++ b/quest/m3/studio/audio-filters.md @@ -11,7 +11,3 @@ from the studio and saved with the scene collection. Prefer maintained crates, such as `nnnoiseless` for RNNoise suppression, over hand-rolled DSP. Benchmark the chain per source so its cost per added source shows up as a slope. - -## Required - -- [The studio app](/quest/m3/studio/app.md) - the sources and bus the chain sits between diff --git a/quest/m3/studio/browser-source.md b/quest/m3/studio/browser-source.md index 42d10be5f2c..db1ae0da708 100644 --- a/quest/m3/studio/browser-source.md +++ b/quest/m3/studio/browser-source.md @@ -21,7 +21,3 @@ Decided 2026-10-22: use CEF through `cef-rs`, desktop only. - **Process:** CEF runs out of process. - **First check:** whether `cef-rs` exposes accelerated offscreen painting. If it does not, upstream that before building on it. - -## Required - -- [The studio app](/quest/m3/studio/app.md) - the compositor and sources this adds a source type to diff --git a/quest/m3/studio/destinations.md b/quest/m3/studio/destinations.md index 55ffcb55f4b..84892afdba5 100644 --- a/quest/m3/studio/destinations.md +++ b/quest/m3/studio/destinations.md @@ -64,6 +64,5 @@ Decided 2026-10-22 with the maintainer: ## Required -- [The studio app](/quest/m3/studio/app.md) - the canvas and MoQ output these destinations join - [AudioToolbox AAC encode](/quest/m2/audio-encode-audiotoolbox.md) - AAC on macOS - [Media Foundation AAC encode](/quest/m2/audio-encode-mediafoundation.md) - AAC on Windows diff --git a/quest/m3/studio/guests.md b/quest/m3/studio/guests.md index 1e84f3da4f4..32c91b8832e 100644 --- a/quest/m3/studio/guests.md +++ b/quest/m3/studio/guests.md @@ -26,10 +26,6 @@ remove a participant from the scene. inside the studio was considered and dropped, because it brings NAT traversal and TURN for no capability the relay lacks. -## Required - -- [The studio app](/quest/m3/studio/app.md) - the MoQ broadcast source guests reuse - ## Related - [Hang contribution references](/quest/m2/contribution-ref.md) - a composed broadcast naming each guest feed diff --git a/quest/m3/studio/mixes.md b/quest/m3/studio/mixes.md index 45956de0c9d..a9aef7338ba 100644 --- a/quest/m3/studio/mixes.md +++ b/quest/m3/studio/mixes.md @@ -15,7 +15,3 @@ output that falls behind drops its own audio, not the bus's. Benchmark source count against mix count, including sparse routing. A mix must cost only what is routed into it, never a scan of every source. - -## Required - -- [The studio app](/quest/m3/studio/app.md) - the single program bus this extends diff --git a/quest/m3/studio/multi-canvas.md b/quest/m3/studio/multi-canvas.md index e1d3004b5c0..296b7395c45 100644 --- a/quest/m3/studio/multi-canvas.md +++ b/quest/m3/studio/multi-canvas.md @@ -16,7 +16,3 @@ canvas together unless the creator unlinks one. Benchmark source count against canvas count, including sparse reuse. Per-canvas cost must grow only with the sources that canvas uses, never with the whole source table. - -## Required - -- [The studio app](/quest/m3/studio/app.md) - the single canvas this generalizes diff --git a/quest/m3/studio/packaging.md b/quest/m3/studio/packaging.md index a2ee938c65e..14278362e6a 100644 --- a/quest/m3/studio/packaging.md +++ b/quest/m3/studio/packaging.md @@ -9,7 +9,7 @@ is verified to launch and list devices on its platform. ## Required - [Signing identities are available](/quest/m3/studio/signing.md) - the credentials signing needs -- [The studio app](/quest/m3/studio/app.md) - the binary being packaged +- [Studio on macOS and Windows](/quest/m3/studio/app.md) - the studio running on every platform it packages ## Related diff --git a/quest/m3/studio/recording.md b/quest/m3/studio/recording.md index 707024ee2e2..5ccb1138252 100644 --- a/quest/m3/studio/recording.md +++ b/quest/m3/studio/recording.md @@ -12,7 +12,3 @@ Write fMP4 through moq-mux. A recording shares a live destination's encoder only when both have the same input (the same canvas or source, and the same audio mix) and identical encode settings. An isolated source track never shares the program's encoder. - -## Required - -- [The studio app](/quest/m3/studio/app.md) - the program and encoders a recording reuses diff --git a/quest/m3/studio/vcam/linux.md b/quest/m3/studio/vcam/linux.md index 5183dc39059..c5c22c6c6fd 100644 --- a/quest/m3/studio/vcam/linux.md +++ b/quest/m3/studio/vcam/linux.md @@ -5,7 +5,3 @@ The studio publishes a canvas's program as a PipeWire video node that PipeWire-aware apps (browsers and conferencing apps) list as a camera. It falls back to an existing v4l2loopback device when one is present. - -## Required - -- [The studio app](/quest/m3/studio/app.md) - the program this exposes diff --git a/quest/m3/studio/vcam/macos.md b/quest/m3/studio/vcam/macos.md index 314ae7c259f..9facb51e932 100644 --- a/quest/m3/studio/vcam/macos.md +++ b/quest/m3/studio/vcam/macos.md @@ -9,4 +9,4 @@ update installs the new extension without a reboot. ## Required - [Signing identities are available](/quest/m3/studio/signing.md) - system extensions only load when signed -- [The studio app](/quest/m3/studio/app.md) - the program this exposes +- [Studio on macOS and Windows](/quest/m3/studio/app.md) - the studio running on macOS diff --git a/quest/m3/studio/vcam/windows.md b/quest/m3/studio/vcam/windows.md index d57c2025dc4..18e335920cb 100644 --- a/quest/m3/studio/vcam/windows.md +++ b/quest/m3/studio/vcam/windows.md @@ -8,4 +8,4 @@ Apps list it as a camera while the studio runs. ## Required -- [The studio app](/quest/m3/studio/app.md) - the program this exposes +- [Studio on macOS and Windows](/quest/m3/studio/app.md) - the studio running on Windows diff --git a/quest/m3/studio/web/video-web.md b/quest/m3/studio/web/video-web.md index 5fe8b8e5490..154e61493e8 100644 --- a/quest/m3/studio/web/video-web.md +++ b/quest/m3/studio/web/video-web.md @@ -34,5 +34,4 @@ A CI `cargo check` for wasm32 keeps them building. ## Required -- [The studio app](/quest/m3/studio/app.md) - the app the page builds to wasm - [A runtime trait replaces web-async](/quest/m2/runtime-trait.md) - spawn and time on the browser instead of tokio's runtime diff --git a/rs/justfile b/rs/justfile index 78bb68fdfd9..3b891a4227b 100644 --- a/rs/justfile +++ b/rs/justfile @@ -145,14 +145,14 @@ macos-test: # Compile and lint the capture and playback features, which the default check skips. [positional-arguments] capture *args: - bash sh/local.sh cargo clippy --locked -p moq-video -p moq-audio --all-targets --features moq-video/capture,moq-audio/capture,moq-audio/playback "$@" -- -D warnings + bash sh/local.sh cargo clippy --locked -p moq-video -p moq-audio -p moq-scene -p moq-studio --all-targets --features moq-video/capture,moq-audio/capture,moq-audio/playback,moq-scene/capture,moq-studio/capture "$@" -- -D warnings # The capture and playback tests drive synthetic devices, so they need no hardware. # Test the capture and playback features, which the default test pass skips. [positional-arguments] capture-test *args: - just rs test -p moq-video -p moq-audio --features moq-video/capture,moq-audio/capture,moq-audio/playback "$@" + just rs test -p moq-video -p moq-audio -p moq-scene -p moq-studio --features moq-video/capture,moq-audio/capture,moq-audio/playback,moq-scene/capture,moq-studio/capture "$@" # Compare SHM and GetImage through production capture on the current X11 display. x11-bench: @@ -357,7 +357,8 @@ cli-export *args: quic-stress: just rs test -p moq-tokio --lib --run-ignored only -E 'test(/^quic::driver::tests::stress_/)' -# Render CPU frames through wgpu on Mesa's software Vulkan driver, so no GPU is needed. +# Render CPU frames, composite scenes, and publish a studio program through wgpu on +# Mesa's software Vulkan driver, so no GPU is needed. render-sw: #!/usr/bin/env bash set -euo pipefail @@ -367,6 +368,10 @@ render-sw: LD_LIBRARY_PATH="$loader/lib" VK_DRIVER_FILES="${icd[0]}" \ just rs test -p moq-video --features render --lib --run-ignored only \ -E 'test(/^render::renderer::tests::(the_declared_color_space_beats_the_size_guess|cpu_frames_survive_the_round_trip|config_size_overrides_the_frame_size)$/)' + LD_LIBRARY_PATH="$loader/lib" VK_DRIVER_FILES="${icd[0]}" \ + just rs test -p moq-scene --lib --run-ignored only -E 'test(/^compositor::tests::/)' + LD_LIBRARY_PATH="$loader/lib" VK_DRIVER_FILES="${icd[0]}" \ + just rs test -p moq-studio --features openh264 --run-ignored only -E 'test(=engine::tests::the_program_reaches_a_relay)' # Run the ignored mDNS tests, which multicast on the host's network interfaces. multicast: diff --git a/rs/moq-audio/Cargo.toml b/rs/moq-audio/Cargo.toml index 183de7b7d02..61dd5b1faa7 100644 --- a/rs/moq-audio/Cargo.toml +++ b/rs/moq-audio/Cargo.toml @@ -14,7 +14,7 @@ categories = ["multimedia", "multimedia::audio", "multimedia::encoding"] [features] # On by default: what costs the build nothing beyond cargo, which here is `aac` -# (pure Rust). Off by default: the device features, because on Linux cpal links +# and `bus` (pure Rust). Off by default: the device features, because on Linux cpal links # libasound at build time, the one system build dependency in this crate. macOS # and Windows would pay nothing, but a default is one setting for every host. # The features exist so a consumer can pick what it wants, not to hide code from @@ -23,13 +23,16 @@ categories = ["multimedia", "multimedia::audio", "multimedia::encoding"] # `just rs macos`. Workspace consumers opt in at the root manifest # (`default-features = false`) per crate, which keeps cpal out of the language # bindings. -default = ["aac"] +default = ["aac", "bus"] # AAC-LC decode via symphonia. Pure Rust, so it costs no toolchain: the trade # this crate already refused for Opus. Only a subscriber to an ingest-sourced # broadcast needs it (RTMP, SRT/TS, fmp4, and gstreamer all publish AAC, while # this crate encodes in software is Opus or PCM), so a publish-only build can # drop it and the two crates it pulls. aac = ["dep:symphonia-codec-aac", "dep:symphonia-core"] +# The program mixing bus. Pure Rust: it resamples and de-jitters each source +# through the same `fixed-resample` ring the playback mixer uses. +bus = ["dep:fixed-resample"] # Device capture. Microphones go through cpal (pure-Rust: CoreAudio / WASAPI / # ALSA); macOS system audio goes through ScreenCaptureKit, which is where Apple # puts it. On Linux cpal links libasound, the one system build dependency in @@ -137,10 +140,16 @@ objc2-foundation = { workspace = true, optional = true } objc2-screen-capture-kit = { workspace = true, features = ["SCStream", "SCShareableContent"], optional = true } [dev-dependencies] +criterion = { workspace = true } # Reads the jitter conformance corpus at doc/concept/audio-jitter/. serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true, features = ["macros", "rt", "test-util"] } +[[bench]] +name = "bus" +harness = false +required-features = ["bus"] + [lints] workspace = true diff --git a/rs/moq-audio/MIGRATE.md b/rs/moq-audio/MIGRATE.md index 250129f83dd..51df70a4bf0 100644 --- a/rs/moq-audio/MIGRATE.md +++ b/rs/moq-audio/MIGRATE.md @@ -4,6 +4,10 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `Level` is at the crate root + +`encode::Level` is `moq_audio::Level`, since the program bus's meters report it too. + ### Decoded reads carry gaps and encoder ends (#5687, #6475) `decode::Consumer::read()` returns `decode::Read` events instead of bare frames, the same shape as `moq_video::decode::Read`: `Frame(frame)`, `Gap { timestamp, duration }` for lost media, and `End { end }` where the publisher stopped its encoder; `None` still means the track is finished. diff --git a/rs/moq-audio/benches/bus.rs b/rs/moq-audio/benches/bus.rs new file mode 100644 index 00000000000..400f007c6ed --- /dev/null +++ b/rs/moq-audio/benches/bus.rs @@ -0,0 +1,52 @@ +//! Program bus cost per 10 ms block, swept over the number of channels and +//! whether they need resampling. +//! +//! Run with `cargo bench -p moq-audio --bench bus`. + +use std::hint::black_box; + +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use moq_audio::bus::{Bus, Config, Strip}; +use moq_audio::encode::Input; +use moq_audio::{Format, Layout}; + +/// The bus's 10 ms block at its default 48 kHz. +const BLOCK: usize = 480; + +fn bench_mix(c: &mut Criterion) { + let mut group = c.benchmark_group("bus_mix"); + group.throughput(Throughput::Elements(BLOCK as u64)); + + for rate in [48_000u32, 44_100] { + for channels in [1usize, 4, 16] { + let bus = Bus::new(Config::default()).unwrap(); + let frames = BLOCK * rate as usize / 48_000; + let block: Vec = std::iter::repeat_n(0.25f32, frames * 2) + .flat_map(f32::to_le_bytes) + .collect(); + let mut inputs: Vec<_> = (0..channels) + .map(|_| { + let mut input = Input::new(rate, Layout::Stereo); + input.format = Format::F32; + bus.channel(input, Strip::default()).unwrap() + }) + .collect(); + let mut out = vec![0.0f32; BLOCK * 2]; + // Writes before the first read are dropped, so open the channels first. + bus.read(&mut out); + + group.bench_function(BenchmarkId::new(format!("{rate}hz"), channels), |b| { + b.iter(|| { + for input in &mut inputs { + input.write(black_box(&block)).unwrap(); + } + bus.read(black_box(&mut out)); + }); + }); + } + } + group.finish(); +} + +criterion_group!(benches, bench_mix); +criterion_main!(benches); diff --git a/rs/moq-audio/src/bus.rs b/rs/moq-audio/src/bus.rs new file mode 100644 index 00000000000..6956770c7e7 --- /dev/null +++ b/rs/moq-audio/src/bus.rs @@ -0,0 +1,641 @@ +//! Mix PCM sources into one program feed, read on the caller's clock. +//! +//! A [`Bus`] sums its [`Channel`]s, each with its own [`Strip`] of gain, mute, +//! and level meter, into one interleaved `f32` stream at the bus's +//! [`Config`]. It is what a production switcher's program audio needs: +//! microphones, system audio, and decoded broadcasts mixed into the one feed +//! that gets encoded and published. +//! +//! Unlike `playback::Engine`, which a device callback drives, nothing here owns +//! a clock. Whoever owns the program clock calls [`Bus::read`] with as many +//! frames as that clock has advanced, so the mix stays in step with it exactly. +//! Each channel is resampled from its own rate to the bus's, and a source that +//! runs slightly fast or slow against that clock is corrected by the same +//! ring that absorbs its jitter, so it never drifts out of the mix. +//! +//! ``` +//! # fn example() -> Result<(), moq_audio::Error> { +//! use moq_audio::{bus, encode}; +//! +//! let bus = bus::Bus::new(bus::Config::default())?; +//! let strip = bus::Strip::default(); +//! let mut mic = bus.channel(encode::Input::new(44_100, moq_audio::Layout::Mono), strip.clone())?; +//! strip.set_gain(0.5); +//! +//! // On the program clock: 10 ms of stereo at 48 kHz. +//! let mut program = vec![0.0f32; 480 * 2]; +//! bus.read(&mut program); +//! # Ok(()) +//! # } +//! ``` +//! +//! Enabled by the default-on `bus` feature; `Bus::capture` also needs `capture`. + +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, Weak}; +use std::time::Duration; + +use fixed_resample::{ResamplerConfig, ResamplingChannelConfig, ResamplingCons, ResamplingProd, resampling_channel}; + +use crate::encode::Input; +use crate::resample::Remix; +use crate::{Error, Layout, Level}; + +/// Audio held between a [`Channel::write`] and the [`Bus::read`] that mixes it. +/// +/// The jitter a late write absorbs without a dropout, and the delay every +/// source carries into the mix. Matches the playback sink's default, which +/// rides out a stalled network read without being audible as delay. +const LATENCY: Duration = Duration::from_millis(50); + +/// Room above [`LATENCY`] before writes are dropped. A writer that runs ahead +/// (a decoder catching up after a stall) parks its samples here rather than +/// losing them, and the overflow correction then trims back to [`LATENCY`]. +const HEADROOM: Duration = Duration::from_millis(150); + +/// What the program feed is mixed to. +/// +/// `#[non_exhaustive]`: construct via [`Config::default`] and set fields, so new +/// options can be added without breaking callers. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct Config { + /// Samples per second per channel of [`Bus::read`]'s output. + pub sample_rate: u32, + /// Speaker layout of [`Bus::read`]'s output. Must name speaker positions, + /// since every channel is remixed to it. + pub layout: Layout, +} + +impl Default for Config { + fn default() -> Self { + Self { + sample_rate: 48_000, + layout: Layout::Stereo, + } + } +} + +/// One program mix that any number of [`Channel`]s feed. +/// +/// Cheap to clone; every clone reads and feeds the same mix. +#[derive(Clone)] +pub struct Bus { + inner: Arc, +} + +struct Inner { + config: Config, + mixer: Mutex, + next: AtomicU64, + level: Meter, +} + +struct Mixer { + entries: Vec, + /// One channel's share of the block being mixed, reused across reads. + scratch: Vec, +} + +/// A channel as the mixer sees it. +struct Entry { + id: u64, + cons: ResamplingCons, + strip: Strip, + /// Gain actually applied, ramping toward the strip's target across a block. + applied: f32, +} + +impl Bus { + /// An empty mix producing PCM in `config`'s format. + pub fn new(config: Config) -> Result { + if config.sample_rate == 0 { + return Err(Error::Unsupported("bus sample rate must be > 0".into())); + } + if config.layout.speakers().is_none() { + return Err(Error::Unsupported(format!( + "a bus needs speaker positions to remix into (got {:?})", + config.layout + ))); + } + + Ok(Self { + inner: Arc::new(Inner { + config, + mixer: Mutex::new(Mixer { + entries: Vec::new(), + scratch: Vec::new(), + }), + next: AtomicU64::new(0), + level: Meter::default(), + }), + }) + } + + /// The format [`read`](Self::read) produces. + pub fn config(&self) -> &Config { + &self.inner.config + } + + /// Add a source to the mix, taking PCM in the layout `input` describes and + /// controlled by `strip`. + /// + /// The strip is the source's identity to a UI: it outlives the channel, so a + /// source that reopens with another format (a new decoder, a replugged + /// device) keeps its gain, mute, and meter by passing the same strip again. + /// The channel leaves the mix when dropped. + pub fn channel(&self, input: Input, strip: Strip) -> Result { + if input.sample_rate == 0 { + return Err(Error::Unsupported("channel sample rate must be > 0".into())); + } + let output = self.inner.config.layout; + let remix = if input.layout == output { + None + } else { + Some(Remix::new(input.layout, output)?) + }; + + let (prod, cons) = resampling_channel::( + output.channels() as usize, + input.sample_rate, + self.inner.config.sample_rate, + // Only ever pushed interleaved, which skips the planar staging buffer. + true, + ResamplingChannelConfig { + latency_seconds: LATENCY.as_secs_f64(), + capacity_seconds: (LATENCY + HEADROOM).as_secs_f64(), + // Correct drift against the program clock by trimming or padding + // back to the target depth, not by letting the ring run dry or full. + underflow_autocorrect_percent_threshold: Some(25.0), + overflow_autocorrect_percent_threshold: Some(75.0), + resampler_config: ResamplerConfig::default(), + ..Default::default() + }, + ); + + let id = self.inner.next.fetch_add(1, Ordering::Relaxed); + self.inner.mixer.lock().unwrap().entries.push(Entry { + id, + cons, + strip: strip.clone(), + applied: strip.target(), + }); + + Ok(Channel { + id, + prod, + remix, + input, + strip, + bus: Arc::downgrade(&self.inner), + }) + } + + /// Mix the next `output.len() / channels` frames into `output`, interleaved + /// in the bus layout. + /// + /// Call it as the program clock advances, with exactly that much time's worth + /// of frames: the bus has no clock of its own, so this is what paces every + /// channel. A channel with nothing buffered contributes silence. Any samples + /// past the last whole frame are zeroed. + pub fn read(&self, output: &mut [f32]) { + output.fill(0.0); + let channels = self.inner.config.layout.channels() as usize; + let whole = output.len() - output.len() % channels; + let output = &mut output[..whole]; + if output.is_empty() { + return; + } + + let mut mixer = self.inner.mixer.lock().unwrap(); + let Mixer { entries, scratch } = &mut *mixer; + scratch.resize(output.len(), 0.0); + let frames = output.len() / channels; + + for entry in entries.iter_mut() { + entry.cons.read_interleaved(scratch, false); + + // Ramp across the block rather than stepping, so a fader move or a + // mute never clicks. + let target = entry.strip.target(); + let start = entry.applied; + let step = (target - start) / frames as f32; + for (index, frame) in scratch.chunks_exact_mut(channels).enumerate() { + let gain = start + step * (index + 1) as f32; + for sample in frame { + *sample *= gain; + } + } + entry.applied = target; + + entry.strip.shared.level.record(Level::measure(scratch)); + for (out, sample) in output.iter_mut().zip(scratch.iter()) { + *out += *sample; + } + } + drop(mixer); + + self.inner.level.record(Level::measure(output)); + } + + /// The loudest the mix has been since the previous call. + pub fn level(&self) -> Level { + self.inner.level.take() + } + + /// Capture an input device into the mix, described by `config` and + /// controlled by `strip`. + /// + /// Nothing opens until [`Capture::run`] is awaited, and the device's format + /// is whatever it offers: the channel adapts to it. + /// + /// Requires the `capture` feature. + #[cfg(feature = "capture")] + pub fn capture(&self, config: crate::capture::Config, strip: Strip) -> Capture { + Capture { + bus: self.clone(), + config, + strip, + } + } +} + +/// One source's PCM on its way into a [`Bus`]. +/// +/// Dropping it takes the source out of the mix. +pub struct Channel { + id: u64, + prod: ResamplingProd, + remix: Option, + input: Input, + strip: Strip, + bus: Weak, +} + +impl Channel { + /// Push PCM in the format, rate, and layout this channel was created with. + /// + /// Never blocks. Audio that arrives before the bus is first read, or that + /// overruns the channel's buffer, is dropped to keep the mix live. + /// + /// Refuses a NaN or infinite sample, which would reach the whole mix and + /// survive any gain, mute included. + pub fn write(&mut self, data: &[u8]) -> Result<(), Error> { + let pcm = self + .input + .format + .as_interleaved_f32(data, self.input.layout.channels())?; + if pcm.iter().any(|sample| !sample.is_finite()) { + return Err(Error::Unsupported("a non-finite PCM sample".into())); + } + self.push(&pcm); + Ok(()) + } + + fn push(&mut self, pcm: &[f32]) { + match &self.remix { + Some(remix) => { + let _ = self.prod.push_interleaved(&remix.process(pcm)); + } + None => { + let _ = self.prod.push_interleaved(pcm); + } + } + } + + /// This source's gain, mute, and meter. + pub fn strip(&self) -> &Strip { + &self.strip + } +} + +impl Drop for Channel { + fn drop(&mut self) { + if let Some(bus) = self.bus.upgrade() { + bus.mixer.lock().unwrap().entries.retain(|entry| entry.id != self.id); + } + } +} + +/// The gain, mute, and meter of one source in a [`Bus`]. +/// +/// Cheap to clone and `Send + Sync`: clones control the same source, so a UI +/// can hold one while a task owns the [`Channel`]. Changes apply from the next +/// [`Bus::read`], ramped across it. +#[derive(Clone, Debug, Default)] +pub struct Strip { + shared: Arc, +} + +#[derive(Debug)] +struct StripState { + /// Linear gain, as [`f32::to_bits`] since there is no atomic float. + gain: AtomicU32, + muted: AtomicBool, + level: Meter, +} + +impl Default for StripState { + fn default() -> Self { + Self { + gain: AtomicU32::new(1.0f32.to_bits()), + muted: AtomicBool::new(false), + level: Meter::default(), + } + } +} + +impl Strip { + /// Set the linear gain: `1.0` is unity, `0.0` silent, above `1.0` boosts. + /// + /// # Panics + /// + /// If `gain` is negative or not finite. + pub fn set_gain(&self, gain: f32) { + assert!(gain.is_finite() && gain >= 0.0, "invalid gain {gain}"); + self.shared.gain.store(gain.to_bits(), Ordering::Relaxed); + } + + /// The linear gain, independent of [`muted`](Self::muted). + pub fn gain(&self) -> f32 { + f32::from_bits(self.shared.gain.load(Ordering::Relaxed)) + } + + /// Silence this source without forgetting its gain. + pub fn set_muted(&self, muted: bool) { + self.shared.muted.store(muted, Ordering::Relaxed); + } + + /// Whether this source is muted. + pub fn muted(&self) -> bool { + self.shared.muted.load(Ordering::Relaxed) + } + + /// The loudest this source contributed to the mix since the previous call, + /// after its gain and mute. + pub fn level(&self) -> Level { + self.shared.level.take() + } + + fn target(&self) -> f32 { + if self.muted() { 0.0 } else { self.gain() } + } +} + +/// The loudest [`Level`] since it was last taken. +#[derive(Debug, Default)] +struct Meter(Mutex); + +impl Meter { + fn record(&self, level: Level) { + let mut held = self.0.lock().unwrap(); + *held = held.max(level); + } + + fn take(&self) -> Level { + std::mem::take(&mut *self.0.lock().unwrap()) + } +} + +/// An input device feeding a [`Bus`], built by [`Bus::capture`]. +/// +/// Requires the `capture` feature. +#[cfg(feature = "capture")] +pub struct Capture { + bus: Bus, + config: crate::capture::Config, + strip: Strip, +} + +#[cfg(feature = "capture")] +impl Capture { + /// Capture into the mix until the device stops or fails, or until the + /// future is dropped, which releases the device. + /// + /// A device that cannot open or stops delivering returns its error rather + /// than being reopened behind the caller's back, so a source that went + /// silent says why. On macOS the future is `!Send`, because the permission + /// prompt and ScreenCaptureKit hold ObjC handles across an await, so run it + /// on a local task there. + pub async fn run(self) -> Result<(), Error> { + let mut stream = crate::capture::open(&self.config) + .await + .map_err(|failure| failure.into_error())?; + let layout = stream.layout(); + let mut input = Input::new(layout.sample_rate, Layout::from_channels(layout.channels)?); + input.format = crate::Format::F32; + let mut channel = self.bus.channel(input, self.strip.clone())?; + + // A gap needs nothing here: the bus has no timeline of its own to + // re-anchor, so the channel just plays on. + while let Some(samples) = stream.read().await.map_err(|failure| failure.into_error())? { + channel.push(&samples.data); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Format; + + /// Interleaved bytes of `frames` frames of `channels` channels, every sample `value`. + fn constant(value: f32, frames: usize, channels: usize) -> Vec { + std::iter::repeat_n(value, frames * channels) + .flat_map(f32::to_le_bytes) + .collect() + } + + fn input(sample_rate: u32, layout: Layout) -> Input { + let mut input = Input::new(sample_rate, layout); + input.format = Format::F32; + input + } + + /// Read blocks of `frames` until `blocks` have passed, returning the last. + fn settle(bus: &Bus, frames: usize, blocks: usize) -> Vec { + let mut out = vec![0.0; frames * bus.config().layout.channels() as usize]; + for _ in 0..blocks { + bus.read(&mut out); + } + out + } + + /// Writes that land before the first read are dropped, so the first read + /// opens the channel and the test feeds it from there, one block per read the + /// way a live source keeps pace with the program clock. + fn feed(bus: &Bus, channel: &mut Channel, value: f32, blocks: usize) -> Vec { + let frames = 480; + let channels = channel.input.layout.channels() as usize; + let in_frames = frames * channel.input.sample_rate as usize / bus.config().sample_rate as usize; + let mut out = vec![0.0; frames * bus.config().layout.channels() as usize]; + bus.read(&mut out); + for _ in 0..blocks { + channel.write(&constant(value, in_frames, channels)).unwrap(); + bus.read(&mut out); + } + out + } + + #[test] + fn sums_channels_with_their_gain() { + let bus = Bus::new(Config::default()).unwrap(); + let mut a = bus.channel(input(48_000, Layout::Stereo), Strip::default()).unwrap(); + let mut b = bus.channel(input(48_000, Layout::Stereo), Strip::default()).unwrap(); + b.strip().set_gain(0.5); + + let mut out = vec![0.0; 480 * 2]; + bus.read(&mut out); + for _ in 0..20 { + a.write(&constant(0.25, 480, 2)).unwrap(); + b.write(&constant(0.25, 480, 2)).unwrap(); + bus.read(&mut out); + } + + // Past the latency fill and the gain ramp, every sample is 0.25 + 0.125. + for sample in &out { + assert!((sample - 0.375).abs() < 1e-4, "{sample}"); + } + } + + #[test] + fn mute_silences_and_keeps_the_gain() { + let bus = Bus::new(Config::default()).unwrap(); + let mut channel = bus.channel(input(48_000, Layout::Stereo), Strip::default()).unwrap(); + channel.strip().set_gain(0.8); + channel.strip().set_muted(true); + + let out = feed(&bus, &mut channel, 0.5, 20); + assert!(out.iter().all(|sample| *sample == 0.0)); + assert_eq!(channel.strip().gain(), 0.8); + assert_eq!(channel.strip().level(), Level::default()); + + channel.strip().set_muted(false); + let out = feed(&bus, &mut channel, 0.5, 20); + assert!((out.last().unwrap() - 0.4).abs() < 1e-4); + } + + /// A gain change ramps across one block instead of stepping, which would click. + #[test] + fn gain_changes_ramp() { + let bus = Bus::new(Config::default()).unwrap(); + let mut channel = bus.channel(input(48_000, Layout::Stereo), Strip::default()).unwrap(); + feed(&bus, &mut channel, 0.5, 20); + + channel.strip().set_gain(0.0); + channel.write(&constant(0.5, 480, 2)).unwrap(); + let mut out = vec![0.0; 480 * 2]; + bus.read(&mut out); + + let first = out[0]; + let last = out[out.len() - 1]; + assert!(first > 0.49, "the ramp starts at the old gain: {first}"); + assert!(last.abs() < 1e-6, "the ramp ends at the new gain: {last}"); + assert!(out.windows(2).all(|pair| pair[1] <= pair[0] + 1e-6), "monotonic"); + } + + /// A mono source plays from both speakers of a stereo bus, and a source at + /// another rate is resampled to the bus's rather than played fast or slow. + #[test] + fn converts_layout_and_rate() { + let bus = Bus::new(Config::default()).unwrap(); + let mut channel = bus.channel(input(16_000, Layout::Mono), Strip::default()).unwrap(); + let out = feed(&bus, &mut channel, 0.5, 40); + + for frame in out.as_chunks::<2>().0 { + assert!((frame[0] - 0.5).abs() < 1e-2, "{frame:?}"); + assert!((frame[1] - 0.5).abs() < 1e-2, "{frame:?}"); + } + } + + /// A source that stops writing leaves silence, not a repeat of its last block. + #[test] + fn a_stalled_source_is_silent() { + let bus = Bus::new(Config::default()).unwrap(); + let mut channel = bus.channel(input(48_000, Layout::Stereo), Strip::default()).unwrap(); + feed(&bus, &mut channel, 0.5, 20); + + let out = settle(&bus, 480, 20); + assert!(out.iter().all(|sample| *sample == 0.0)); + } + + #[test] + fn dropping_a_channel_leaves_the_mix() { + let bus = Bus::new(Config::default()).unwrap(); + let mut kept = bus.channel(input(48_000, Layout::Stereo), Strip::default()).unwrap(); + let mut dropped = bus.channel(input(48_000, Layout::Stereo), Strip::default()).unwrap(); + feed(&bus, &mut dropped, 0.5, 5); + drop(dropped); + + let out = feed(&bus, &mut kept, 0.25, 20); + assert!((out.last().unwrap() - 0.25).abs() < 1e-4); + } + + /// The meter holds the loudest reading until it is read, so a UI polling + /// slower than the program clock still sees a peak. + #[test] + fn levels_hold_until_read() { + let bus = Bus::new(Config::default()).unwrap(); + let mut channel = bus.channel(input(48_000, Layout::Stereo), Strip::default()).unwrap(); + feed(&bus, &mut channel, 0.5, 20); + settle(&bus, 480, 5); + + let level = channel.strip().level(); + assert!((level.peak() - 0.5).abs() < 1e-3, "{level:?}"); + assert_eq!(channel.strip().level(), Level::default(), "taken"); + assert!((bus.level().peak() - 0.5).abs() < 1e-3); + } + + /// A source that reopens in another format keeps its controls by passing the + /// same strip again, which is what a decoder restart or a replugged device does. + #[test] + fn a_strip_outlives_its_channel() { + let bus = Bus::new(Config::default()).unwrap(); + let strip = Strip::default(); + strip.set_gain(0.5); + let first = bus.channel(input(48_000, Layout::Stereo), strip.clone()).unwrap(); + drop(first); + + let mut second = bus.channel(input(16_000, Layout::Mono), strip.clone()).unwrap(); + let out = feed(&bus, &mut second, 0.5, 40); + assert!((out.last().unwrap() - 0.25).abs() < 1e-2, "{}", out.last().unwrap()); + assert!(strip.level().peak() > 0.2); + } + + #[test] + fn refuses_a_layout_without_speakers() { + let config = Config { + layout: Layout::Discrete(2), + ..Default::default() + }; + assert!(Bus::new(config).is_err()); + + let bus = Bus::new(Config::default()).unwrap(); + assert!( + bus.channel(input(48_000, Layout::Discrete(3)), Strip::default()) + .is_err() + ); + } + + /// A NaN survives every gain, mute included, so it is refused before it can + /// reach the mix the other channels share. + #[test] + fn refuses_non_finite_samples() { + let bus = Bus::new(Config::default()).unwrap(); + let mut channel = bus.channel(input(48_000, Layout::Stereo), Strip::default()).unwrap(); + for bad in [f32::NAN, f32::INFINITY] { + assert!(channel.write(&constant(bad, 480, 2)).is_err(), "{bad}"); + } + } + + /// A partial trailing frame in the output is zeroed rather than left with + /// whatever the caller had there. + #[test] + fn zeroes_a_partial_frame() { + let bus = Bus::new(Config::default()).unwrap(); + let mut out = vec![1.0; 5]; + bus.read(&mut out); + assert!(out.iter().all(|sample| *sample == 0.0)); + } +} diff --git a/rs/moq-audio/src/encode/capture.rs b/rs/moq-audio/src/encode/capture.rs index 8a7c29091bc..25420cb5ca4 100644 --- a/rs/moq-audio/src/encode/capture.rs +++ b/rs/moq-audio/src/encode/capture.rs @@ -14,7 +14,7 @@ use super::producer::Reserved; use super::{Input, Options, Producer}; use crate::capture; use crate::resample::{Remix, Resampler}; -use crate::{Error, Format, Frame, Layout as PcmLayout}; +use crate::{Error, Format, Frame, Layout as PcmLayout, Level}; /// Backoff bounds for reopening a capture source. The quick first retry covers /// a USB device re-enumerating; the ceiling keeps a missing device from spinning. @@ -39,47 +39,6 @@ pub enum Status { Ended, } -/// The post-processing level of the most recently captured buffer. -/// -/// Zero while the input is closed. Read it with [`Control::level`] at -/// whatever rate the meter draws at. -#[derive(Clone, Copy, Debug, Default, PartialEq)] -pub struct Level { - rms: f32, - peak: f32, -} - -impl Level { - /// Root mean square amplitude on a linear `0.0..=1.0` scale. - pub fn rms(&self) -> f32 { - self.rms - } - - /// Peak absolute amplitude on a linear `0.0..=1.0` scale. - pub fn peak(&self) -> f32 { - self.peak - } - - fn measure(samples: &[f32]) -> Self { - if samples.is_empty() { - return Self::default(); - } - - let mut squares = 0.0f64; - let mut peak = 0.0f32; - for &sample in samples { - let sample = sample.clamp(-1.0, 1.0); - squares += f64::from(sample) * f64::from(sample); - peak = peak.max(sample.abs()); - } - - Self { - rms: (squares / samples.len() as f64).sqrt() as f32, - peak, - } - } -} - /// A snapshot of a capture-backed publication's lifecycle. /// /// Levels are deliberately not here: they change every buffer, so they would diff --git a/rs/moq-audio/src/encode/mod.rs b/rs/moq-audio/src/encode/mod.rs index 9357a5ca4ca..173ba56822d 100644 --- a/rs/moq-audio/src/encode/mod.rs +++ b/rs/moq-audio/src/encode/mod.rs @@ -30,4 +30,4 @@ pub use encoder::{Codec, Config, Encoder, Finish, Input, Kind}; pub use producer::{Options, Producer}; #[cfg(feature = "capture")] -pub use capture::{Capture, Control, Driver, Level, State, Status, publish_capture}; +pub use capture::{Capture, Control, Driver, State, Status, publish_capture}; diff --git a/rs/moq-audio/src/level.rs b/rs/moq-audio/src/level.rs new file mode 100644 index 00000000000..f9797fc4afc --- /dev/null +++ b/rs/moq-audio/src/level.rs @@ -0,0 +1,54 @@ +/// How loud a stretch of audio was, for drawing a meter. +/// +/// Reported by a capture `encode::Control` and by a mixing +/// bus and its strips. Read it at whatever rate the meter draws at. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct Level { + rms: f32, + peak: f32, +} + +impl Level { + /// Root mean square amplitude on a linear `0.0..=1.0` scale. + pub fn rms(&self) -> f32 { + self.rms + } + + /// Peak absolute amplitude on a linear `0.0..=1.0` scale. + pub fn peak(&self) -> f32 { + self.peak + } + + #[cfg_attr( + not(any(feature = "capture", feature = "bus")), + allow(dead_code, reason = "only capture and the bus measure levels") + )] + pub(crate) fn measure(samples: &[f32]) -> Self { + if samples.is_empty() { + return Self::default(); + } + + let mut squares = 0.0f64; + let mut peak = 0.0f32; + for &sample in samples { + let sample = sample.clamp(-1.0, 1.0); + squares += f64::from(sample) * f64::from(sample); + peak = peak.max(sample.abs()); + } + + Self { + rms: (squares / samples.len() as f64).sqrt() as f32, + peak, + } + } + + /// The louder of two readings, channel by channel, so a meter read less often + /// than buffers arrive still shows the loudest of them. + #[cfg_attr(not(feature = "bus"), allow(dead_code, reason = "only the bus holds levels"))] + pub(crate) fn max(self, other: Self) -> Self { + Self { + rms: self.rms.max(other.rms), + peak: self.peak.max(other.peak), + } + } +} diff --git a/rs/moq-audio/src/lib.rs b/rs/moq-audio/src/lib.rs index 076e4391679..d7eab8773ea 100644 --- a/rs/moq-audio/src/lib.rs +++ b/rs/moq-audio/src/lib.rs @@ -32,6 +32,9 @@ //! output device and mixes the `playback::Sink`s registered with it, so one //! device serves every track in a call. Requires the `playback` feature, so //! these names are unlinked here too. +//! - `bus` mixes any number of PCM sources, each with its own gain, mute, and +//! meter, into one program feed that the caller reads on its own clock: a +//! studio's program audio. Behind the default-on `bus` feature. //! - `aec` keeps the speaker out of the microphone, which is what a conference //! on a laptop needs to not send itself back. `playback::Engine::canceller` //! builds cloneable `aec::Control` handles from the mix it is playing and @@ -54,12 +57,15 @@ mod format; mod frame; mod jitter; mod layout; +mod level; mod opus; mod pcm; mod resample; #[cfg(feature = "aec")] pub mod aec; +#[cfg(feature = "bus")] +pub mod bus; #[cfg(feature = "capture")] pub mod capture; pub mod decode; @@ -73,3 +79,4 @@ pub use error::Error; pub use format::Format; pub use frame::Frame; pub use layout::Layout; +pub use level::Level; diff --git a/rs/moq-scene/Cargo.toml b/rs/moq-scene/Cargo.toml new file mode 100644 index 00000000000..2f5d764476b --- /dev/null +++ b/rs/moq-scene/Cargo.toml @@ -0,0 +1,51 @@ +[package] +# Unpublished until the studio settles its API; moq-studio is its only consumer. +publish = false +name = "moq-scene" +description = "Scene graph, live sources, and a wgpu compositor for a MoQ streaming studio" +authors = ["Luke Curley "] +repository = "https://github.com/moq-dev/moq" +license = "MIT OR Apache-2.0" + +version = "0.1.0" +edition = "2024" +rust-version.workspace = true + +keywords = ["quic", "media", "video", "compositing", "live"] +categories = ["multimedia", "multimedia::video", "rendering"] + +[features] +# Camera, display, window, and microphone sources. Off by default for the same +# reason as moq-video's and moq-audio's `capture`: on Linux the microphone links +# libasound at build time, and a workspace build would otherwise unify it on for +# every crate. Without it, those sources report that the build cannot capture. +capture = ["moq-video/capture", "moq-audio/capture"] + +[dependencies] +bytemuck = { version = "1", features = ["derive"] } +# The catalog's rendition configs, which a broadcast source follows. +hang = { workspace = true } +humantime-serde = { workspace = true } +# PNG and JPEG still images. Already in a studio's graph through eframe. +image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } +kio = { workspace = true } +moq-audio = { workspace = true, features = ["bus"] } +moq-clock = { workspace = true } +moq-mux = { workspace = true } +moq-net = { workspace = true } +# `render` is the compositor's whole input path: it turns every decoded or +# captured frame into a texture on the studio's device. +moq-video = { workspace = true, features = ["render"] } +serde = { workspace = true } +serde_with = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["macros", "time"] } +toml = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt", "test-util"] } + +[lints] +workspace = true diff --git a/rs/moq-scene/src/collection.rs b/rs/moq-scene/src/collection.rs new file mode 100644 index 00000000000..77a91829e83 --- /dev/null +++ b/rs/moq-scene/src/collection.rs @@ -0,0 +1,600 @@ +//! The scene graph, and the TOML file a [`Collection`] saves to. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; +use std::path::PathBuf; +use std::str::FromStr; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use serde_with::{DeserializeFromStr, SerializeDisplay}; + +use crate::Error; + +/// Everything a studio composes: the canvas, the sources, and the scenes that +/// place them. +/// +/// Saved with [`to_toml`](Self::to_toml) and loaded with +/// [`from_toml`](Self::from_toml). The file names devices, files, and broadcast +/// paths, never a relay URL or a token, so it is safe to commit and share. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Collection { + /// The output every scene is drawn onto. + pub canvas: Canvas, + /// How a fade from preview to program behaves. + #[serde(default)] + pub transition: Transition, + /// Every source, by its unique name. Scenes refer to sources by that name, + /// so one camera can appear in many scenes while being opened once. + #[serde(default)] + pub sources: BTreeMap, + /// Gain and mute of each audio-carrying source, by source name. A source + /// without an entry plays at unity. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub audio: BTreeMap, + /// The scenes, in the order the scene list shows them. + #[serde(default)] + pub scenes: Vec, +} + +impl Collection { + /// The scene file format version this build reads and writes. + /// + /// A file names its version, and loading any other one fails rather than + /// guessing at fields whose meaning changed. + pub const VERSION: u32 = 1; + + /// An empty collection with one blank scene on a 1080p, 30 fps canvas. + pub fn new() -> Self { + Self { + canvas: Canvas::default(), + transition: Transition::default(), + sources: BTreeMap::new(), + audio: BTreeMap::new(), + scenes: vec![Scene::new("Scene")], + } + } + + /// Parse and validate a scene file. + pub fn from_toml(text: &str) -> Result { + let mut table: toml::Table = toml::from_str(text).map_err(|err| Error::Format(err.to_string()))?; + let version = match table.remove("version") { + Some(toml::Value::Integer(version)) => version, + Some(other) => return Err(Error::Format(format!("`version` must be an integer, got {other}"))), + None => return Err(Error::Format("missing `version`".into())), + }; + if version != i64::from(Self::VERSION) { + return Err(Error::Version { + found: version, + supported: Self::VERSION, + }); + } + + let collection = Self::deserialize(table).map_err(|err| Error::Format(err.to_string()))?; + collection.validate()?; + Ok(collection) + } + + /// Serialize to the scene file format, `version` first. + pub fn to_toml(&self) -> Result { + self.validate()?; + let body = toml::to_string(self).map_err(|err| Error::Format(err.to_string()))?; + Ok(format!("version = {}\n\n{body}", Self::VERSION)) + } + + /// The scene called `name`. + pub fn scene(&self, name: &str) -> Option<&Scene> { + self.scenes.iter().find(|scene| scene.name == name) + } + + /// The scene called `name`, to edit. + pub fn scene_mut(&mut self, name: &str) -> Option<&mut Scene> { + self.scenes.iter_mut().find(|scene| scene.name == name) + } + + /// Check what the types cannot: the canvas can be encoded, names are unique, + /// and every item and audio entry names a source that exists. + pub fn validate(&self) -> Result<(), Error> { + self.canvas.validate()?; + + let mut names = BTreeSet::new(); + for scene in &self.scenes { + if scene.name.is_empty() { + return Err(Error::Invalid("a scene has an empty name".into())); + } + if !names.insert(scene.name.as_str()) { + return Err(Error::Invalid(format!("two scenes are named `{}`", scene.name))); + } + for item in &scene.items { + if !self.sources.contains_key(&item.source) { + return Err(Error::Invalid(format!( + "scene `{}` places `{}`, which is not a source", + scene.name, item.source + ))); + } + item.rect.validate(&scene.name, &item.source)?; + } + } + + if let Some(name) = self.sources.keys().find(|name| name.is_empty()) { + return Err(Error::Invalid(format!("a source has an empty name: `{name}`"))); + } + + for (name, audio) in &self.audio { + if !self.sources.contains_key(name) { + return Err(Error::Invalid(format!( + "audio settings for `{name}`, which is not a source" + ))); + } + if !audio.gain.is_finite() || audio.gain < 0.0 { + return Err(Error::Invalid(format!("`{name}` has gain {}", audio.gain))); + } + } + + Ok(()) + } +} + +impl Default for Collection { + fn default() -> Self { + Self::new() + } +} + +/// The output picture: its size and how often it is drawn. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Canvas { + /// Width in pixels. Even, since the encoder subsamples chroma. + pub width: u32, + /// Height in pixels. Even, since the encoder subsamples chroma. + pub height: u32, + /// Pictures drawn and encoded per second. + pub framerate: u32, +} + +impl Canvas { + /// The canvas size. + pub fn size(&self) -> moq_video::Size { + moq_video::Size::new(self.width, self.height) + } + + /// How often a picture is drawn. + pub fn framerate(&self) -> moq_video::FrameRate { + moq_video::FrameRate::new(self.framerate, 1).expect("validated canvas framerate") + } + + fn validate(&self) -> Result<(), Error> { + if self.width == 0 || self.height == 0 || !self.width.is_multiple_of(2) || !self.height.is_multiple_of(2) { + return Err(Error::Invalid(format!( + "canvas {}x{} must be non-zero and even", + self.width, self.height + ))); + } + moq_video::FrameRate::new(self.framerate, 1) + .map_err(|err| Error::Invalid(format!("canvas framerate {}: {err}", self.framerate)))?; + Ok(()) + } +} + +impl Default for Canvas { + fn default() -> Self { + Self { + width: 1920, + height: 1080, + framerate: 30, + } + } +} + +/// How the program changes when the preview is faded in. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Transition { + /// How long a fade takes. + #[serde(with = "humantime_serde")] + pub fade: Duration, +} + +impl Default for Transition { + fn default() -> Self { + Self { + fade: Duration::from_millis(300), + } + } +} + +/// One arrangement of sources on the canvas. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Scene { + /// Unique among the collection's scenes. + pub name: String, + /// What the scene draws, back to front. + #[serde(default)] + pub items: Vec, +} + +impl Scene { + /// An empty scene. + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + items: Vec::new(), + } + } +} + +/// A source placed in a scene. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Item { + /// The [`Collection::sources`] entry to draw. + pub source: String, + /// Where on the canvas, stretched to fill it. + pub rect: Rect, + /// Whether the item is drawn. A hidden item keeps its place. + #[serde(default = "visible")] + pub visible: bool, +} + +fn visible() -> bool { + true +} + +/// A rectangle in canvas pixels, from the top-left corner. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Rect { + /// Left edge. + pub x: f32, + /// Top edge. + pub y: f32, + /// Width. + pub width: f32, + /// Height. + pub height: f32, +} + +impl Rect { + /// The whole of `canvas`. + pub fn fill(canvas: &Canvas) -> Self { + Self { + x: 0.0, + y: 0.0, + width: canvas.width as f32, + height: canvas.height as f32, + } + } + + fn validate(&self, scene: &str, source: &str) -> Result<(), Error> { + let finite = [self.x, self.y, self.width, self.height].iter().all(|v| v.is_finite()); + if !finite || self.width < 0.0 || self.height < 0.0 { + return Err(Error::Invalid(format!( + "scene `{scene}` places `{source}` at {self:?}, which is not a rectangle" + ))); + } + Ok(()) + } +} + +/// Where a source's picture or sound comes from. +/// +/// Device ids are the ones the matching `moq_video::capture` and +/// `moq_audio::capture` listings report; `None` means the system default. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum Source { + /// A camera. + Camera { + /// The camera's id, or the default camera. + #[serde(default, skip_serializing_if = "Option::is_none")] + id: Option, + }, + /// A whole display. + Display { + /// The display's id, or the main display. + #[serde(default, skip_serializing_if = "Option::is_none")] + id: Option, + }, + /// One window. + Window { + /// The window's id. + id: String, + }, + /// A microphone or other audio input. + Microphone { + /// The input's id, or the default input. + #[serde(default, skip_serializing_if = "Option::is_none")] + id: Option, + }, + /// Everything the machine plays, where the platform can capture it. + SystemAudio, + /// A PNG or JPEG still. + Image { + /// The file, relative to the scene file's directory unless absolute. + path: PathBuf, + }, + /// A solid color. + Color { + /// The color, as `#rrggbb` or `#rrggbbaa`. + color: Rgba, + }, + /// A broadcast subscribed from the relay the studio is connected to. + Broadcast { + /// The broadcast path on that relay. + path: String, + }, +} + +impl Source { + /// Whether the source has a picture to place in a scene. + pub fn has_video(&self) -> bool { + !matches!(self, Self::Microphone { .. } | Self::SystemAudio) + } + + /// Whether the source feeds the program audio. + pub fn has_audio(&self) -> bool { + matches!( + self, + Self::Microphone { .. } | Self::SystemAudio | Self::Broadcast { .. } + ) + } +} + +/// The gain and mute of one source in the program audio. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Audio { + /// Linear gain: 1.0 is unity. + pub gain: f32, + /// Whether the source is silenced, keeping its gain. + #[serde(default)] + pub muted: bool, +} + +impl Default for Audio { + fn default() -> Self { + Self { + gain: 1.0, + muted: false, + } + } +} + +/// An sRGB color with straight alpha, written `#rrggbb` or `#rrggbbaa`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, SerializeDisplay, DeserializeFromStr)] +pub struct Rgba(pub [u8; 4]); + +impl fmt::Display for Rgba { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let [r, g, b, a] = self.0; + if a == u8::MAX { + write!(f, "#{r:02x}{g:02x}{b:02x}") + } else { + write!(f, "#{r:02x}{g:02x}{b:02x}{a:02x}") + } + } +} + +impl FromStr for Rgba { + type Err = Error; + + fn from_str(text: &str) -> Result { + let invalid = || Error::Invalid(format!("`{text}` is not a #rrggbb or #rrggbbaa color")); + let hex = text.strip_prefix('#').ok_or_else(invalid)?; + if !matches!(hex.len(), 6 | 8) || !hex.is_ascii() { + return Err(invalid()); + } + let mut rgba = [u8::MAX; 4]; + for (channel, pair) in rgba.iter_mut().zip(hex.as_bytes().as_chunks::<2>().0) { + let pair = std::str::from_utf8(pair).map_err(|_| invalid())?; + *channel = u8::from_str_radix(pair, 16).map_err(|_| invalid())?; + } + Ok(Self(rgba)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn example() -> Collection { + let mut collection = Collection::new(); + collection.sources.insert( + "camera".into(), + Source::Camera { + id: Some("/dev/video0".into()), + }, + ); + collection.sources.insert( + "logo".into(), + Source::Image { + path: "logo.png".into(), + }, + ); + collection.sources.insert( + "background".into(), + Source::Color { + color: Rgba([0x10, 0x20, 0x30, 0xff]), + }, + ); + collection.sources.insert("mic".into(), Source::Microphone { id: None }); + collection.sources.insert( + "guest".into(), + Source::Broadcast { + path: "studio/guest".into(), + }, + ); + collection.audio.insert("mic".into(), Audio { gain: 0.5, muted: true }); + + let canvas = collection.canvas; + let scene = &mut collection.scenes[0]; + scene.items.push(Item { + source: "background".into(), + rect: Rect::fill(&canvas), + visible: true, + }); + scene.items.push(Item { + source: "camera".into(), + rect: Rect { + x: 100.0, + y: 50.0, + width: 640.0, + height: 360.0, + }, + visible: false, + }); + collection + } + + #[test] + fn round_trips() { + let collection = example(); + let text = collection.to_toml().unwrap(); + assert!(text.starts_with("version = 1\n"), "{text}"); + assert_eq!(Collection::from_toml(&text).unwrap(), collection); + } + + /// The file is meant to be read and diffed by people, so pin what it looks like. + #[test] + fn reads_a_handwritten_file() { + let text = r##" +version = 1 + +[canvas] +width = 1280 +height = 720 +framerate = 60 + +[transition] +fade = "500ms" + +[sources.cam] +kind = "camera" + +[sources.bg] +kind = "color" +color = "#ff000080" + +[audio.cam2] +gain = 2.0 + +[sources.cam2] +kind = "broadcast" +path = "room/alice" + +[[scenes]] +name = "Main" +items = [ + { source = "bg", rect = { x = 0, y = 0, width = 1280, height = 720 } }, + { source = "cam", rect = { x = 10, y = 10, width = 320, height = 180 }, visible = false }, +] +"##; + let collection = Collection::from_toml(text).unwrap(); + assert_eq!(collection.canvas.framerate, 60); + assert_eq!(collection.transition.fade, Duration::from_millis(500)); + assert_eq!(collection.sources["cam"], Source::Camera { id: None }); + assert_eq!( + collection.sources["bg"], + Source::Color { + color: Rgba([255, 0, 0, 128]) + } + ); + assert_eq!( + collection.audio["cam2"], + Audio { + gain: 2.0, + muted: false + } + ); + let main = collection.scene("Main").unwrap(); + assert!(main.items[0].visible); + assert!(!main.items[1].visible); + } + + #[test] + fn refuses_another_version() { + let text = example().to_toml().unwrap().replacen("version = 1", "version = 2", 1); + let err = Collection::from_toml(&text).unwrap_err(); + assert!(matches!(err, Error::Version { found: 2, supported: 1 }), "{err}"); + + let text = example().to_toml().unwrap().replacen("version = 1", "", 1); + assert!(matches!(Collection::from_toml(&text), Err(Error::Format(_)))); + } + + /// A typo or a field from a newer build fails instead of being dropped on the + /// next save. + #[test] + fn refuses_unknown_fields() { + let text = example() + .to_toml() + .unwrap() + .replacen("[canvas]", "[canvas]\ndepth = 3", 1); + assert!(matches!(Collection::from_toml(&text), Err(Error::Format(_)))); + + let text = example() + .to_toml() + .unwrap() + .replacen("kind = \"camera\"", "kind = \"camera\"\nzoom = 2", 1); + assert!(matches!(Collection::from_toml(&text), Err(Error::Format(_)))); + + let text = example() + .to_toml() + .unwrap() + .replacen("kind = \"camera\"", "kind = \"hologram\"", 1); + assert!(matches!(Collection::from_toml(&text), Err(Error::Format(_)))); + } + + #[test] + fn refuses_dangling_references() { + let mut collection = example(); + collection.sources.remove("camera"); + assert!(matches!(collection.to_toml(), Err(Error::Invalid(_)))); + + let mut collection = example(); + collection.audio.insert("nobody".into(), Audio::default()); + assert!(matches!(collection.validate(), Err(Error::Invalid(_)))); + } + + #[test] + fn refuses_duplicate_scenes() { + let mut collection = example(); + collection.scenes.push(Scene::new("Scene")); + assert!(matches!(collection.validate(), Err(Error::Invalid(_)))); + } + + #[test] + fn refuses_a_canvas_the_encoder_cannot_take() { + for (width, height, framerate) in [(1921, 1080, 30), (0, 1080, 30), (1920, 1080, 0)] { + let mut collection = example(); + collection.canvas = Canvas { + width, + height, + framerate, + }; + assert!( + matches!(collection.validate(), Err(Error::Invalid(_))), + "{width}x{height}@{framerate}" + ); + } + } + + #[test] + fn refuses_a_degenerate_rect() { + let mut collection = example(); + collection.scenes[0].items[0].rect.width = f32::NAN; + assert!(matches!(collection.validate(), Err(Error::Invalid(_)))); + } + + #[test] + fn colors_parse_and_print() { + assert_eq!("#102030".parse::().unwrap(), Rgba([0x10, 0x20, 0x30, 0xff])); + assert_eq!("#10203040".parse::().unwrap().to_string(), "#10203040"); + assert_eq!(Rgba([1, 2, 3, 255]).to_string(), "#010203"); + for bad in ["102030", "#1020", "#gg0000", "#ééé"] { + assert!(bad.parse::().is_err(), "{bad}"); + } + } +} diff --git a/rs/moq-scene/src/compositor.rs b/rs/moq-scene/src/compositor.rs new file mode 100644 index 00000000000..e509e8522db --- /dev/null +++ b/rs/moq-scene/src/compositor.rs @@ -0,0 +1,936 @@ +//! Draw scenes onto the canvas on the GPU: a preview, a program, and the fade +//! between them. + +use std::collections::HashMap; +use std::num::NonZeroU64; +use std::time::Duration; + +use moq_clock::Instant; +use moq_video::render::{Renderer, wgpu}; + +use crate::Error; +use crate::collection::{Canvas, Collection, Rect, Source}; +use crate::source::{Picture, Sources}; + +/// The format of every canvas texture: gamma-encoded RGBA, what +/// `moq_video::render` produces and what an encoder reads back. +pub const FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Rgba8Unorm; + +/// One draw's uniform: where on the canvas, in clip space, and how opaque. +#[repr(C)] +#[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)] +struct Uniform { + /// Top-left x, top-left y, bottom-right x, bottom-right y, in clip space. + rect: [f32; 4], + /// Opacity in `x`; the rest pads the struct to WGSL's 16-byte vectors. + opacity: [f32; 4], +} + +const SHADER: &str = r#" +struct Item { + rect: vec4, + opacity: vec4, +} + +@group(0) @binding(0) var item: Item; +@group(1) @binding(0) var image: texture_2d; +@group(1) @binding(1) var image_sampler: sampler; + +struct Varyings { + @builtin(position) position: vec4, + @location(0) uv: vec2, +} + +// A four-vertex triangle strip covering the item's rectangle. +@vertex fn vs_main(@builtin(vertex_index) index: u32) -> Varyings { + let corner = vec2(f32(index & 1u), f32((index >> 1u) & 1u)); + var out: Varyings; + out.position = vec4(mix(item.rect.xy, item.rect.zw, corner), 0.0, 1.0); + out.uv = corner; + return out; +} + +@fragment fn fs_main(in: Varyings) -> @location(0) vec4 { + let color = textureSample(image, image_sampler, in.uv); + return vec4(color.rgb, color.a * item.opacity.x); +} +"#; + +/// Draws a [`Collection`]'s scenes onto its canvas: one texture for the scene +/// in preview, one for the scene on program, and the fade from one to the other. +/// +/// Both textures are plain [`wgpu::Texture`]s on the caller's device, so a UI +/// shows them without a copy and an encoder reads the program from the same +/// memory. [`render`](Self::render) redraws both from the newest picture each +/// source has, so a source that stalls repeats its last picture rather than +/// holding up the canvas. +/// +/// Pixels are blended in the gamma-encoded space sources arrive in, as a +/// broadcast switcher does, so a 50% fade is a 50% mix of the stored values. +pub struct Compositor { + device: wgpu::Device, + queue: wgpu::Queue, + canvas: Canvas, + + pipeline: wgpu::RenderPipeline, + image_layout: wgpu::BindGroupLayout, + sampler: wgpu::Sampler, + + uniform_layout: wgpu::BindGroupLayout, + uniforms: Uniforms, + + preview: Target, + program: Target, + /// The scene fading in, drawn whole so it blends over the program as one + /// picture rather than layer by layer. + incoming: Target, + + layers: HashMap, + preview_scene: Option, + program_scene: Option, + fade: Option, +} + +/// A fade from `from` to the program scene. +struct Fade { + from: Option, + start: Instant, + duration: Duration, +} + +/// A canvas-sized texture drawn into, and the bind group that samples it. +struct Target { + texture: wgpu::Texture, + view: wgpu::TextureView, + bind: wgpu::BindGroup, +} + +/// One source's newest picture on the GPU. +struct Layer { + /// What the picture came from. An edited source starts a new layer, so it + /// never shows the old source's last picture. + source: Source, + /// Converts video frames, created on the first one. + renderer: Option, + /// Samples the newest picture; `None` until the first arrives. + bind: Option, + /// Why the newest picture failed to draw, logged when it first does. + error: Option, +} + +/// A growable uniform buffer holding one [`Uniform`] per draw of a render. +struct Uniforms { + buffer: wgpu::Buffer, + bind: wgpu::BindGroup, + /// Bytes between draws, rounded up to the device's dynamic-offset alignment. + stride: u64, + capacity: u64, +} + +/// One textured quad: what to sample, and the uniform that places it. +struct Draw { + bind: wgpu::BindGroup, + uniform: Uniform, +} + +/// One render pass: a target cleared to black and the quads drawn into it. +struct Pass<'a> { + view: &'a wgpu::TextureView, + draws: Vec, +} + +impl Compositor { + /// A compositor drawing `canvas`-sized pictures on `device`. + /// + /// Fails if the device cannot hold a texture that large. + pub fn new(device: &wgpu::Device, queue: &wgpu::Queue, canvas: Canvas) -> Result { + let limit = device.limits().max_texture_dimension_2d; + if canvas.width > limit || canvas.height > limit { + return Err(Error::Invalid(format!( + "canvas {}x{} exceeds this GPU's {limit}px texture limit", + canvas.width, canvas.height + ))); + } + + let image_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("moq-scene image"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }); + let uniform_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("moq-scene item"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX_FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: true, + min_binding_size: NonZeroU64::new(size_of::() as u64), + }, + count: None, + }], + }); + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("moq-scene"), + bind_group_layouts: &[Some(&uniform_layout), Some(&image_layout)], + immediate_size: 0, + }); + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("moq-scene"), + source: wgpu::ShaderSource::Wgsl(SHADER.into()), + }); + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("moq-scene"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + compilation_options: Default::default(), + buffers: &[], + }, + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleStrip, + ..Default::default() + }, + depth_stencil: None, + multisample: Default::default(), + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format: FORMAT, + // Straight alpha over an opaque canvas, which stays opaque. + blend: Some(wgpu::BlendState { + color: wgpu::BlendComponent { + src_factor: wgpu::BlendFactor::SrcAlpha, + dst_factor: wgpu::BlendFactor::OneMinusSrcAlpha, + operation: wgpu::BlendOperation::Add, + }, + alpha: wgpu::BlendComponent { + src_factor: wgpu::BlendFactor::One, + dst_factor: wgpu::BlendFactor::OneMinusSrcAlpha, + operation: wgpu::BlendOperation::Add, + }, + }), + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }); + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("moq-scene"), + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + ..Default::default() + }); + + let alignment = u64::from(device.limits().min_uniform_buffer_offset_alignment); + let stride = (size_of::() as u64).next_multiple_of(alignment); + let uniforms = Uniforms::new(device, &uniform_layout, stride, 16); + + let target = |label| Target::new(device, &image_layout, &sampler, canvas, label); + Ok(Self { + preview: target("moq-scene preview"), + program: target("moq-scene program"), + incoming: target("moq-scene incoming"), + device: device.clone(), + queue: queue.clone(), + canvas, + pipeline, + image_layout, + sampler, + uniform_layout, + uniforms, + layers: HashMap::new(), + preview_scene: None, + program_scene: None, + fade: None, + }) + } + + /// The canvas this compositor draws. + pub fn canvas(&self) -> Canvas { + self.canvas + } + + /// The scene in preview, as of the last [`render`](Self::render). + pub fn preview(&self) -> &wgpu::Texture { + &self.preview.texture + } + + /// The program output, as of the last [`render`](Self::render). Created + /// with `COPY_SRC` so an encoder can read it back. + pub fn program(&self) -> &wgpu::Texture { + &self.program.texture + } + + /// The scene in preview. + pub fn preview_scene(&self) -> Option<&str> { + self.preview_scene.as_deref() + } + + /// The scene on program, or the one fading in. + pub fn program_scene(&self) -> Option<&str> { + self.program_scene.as_deref() + } + + /// Why `source`'s newest picture could not be drawn, until one is. + pub fn error(&self, source: &str) -> Option<&str> { + self.layers.get(source)?.error.as_deref() + } + + /// Whether a fade is under way. + pub fn fading(&self) -> bool { + self.fade.is_some() + } + + /// Put the scene called `scene` in preview, or nothing. + pub fn set_preview(&mut self, scene: Option) { + self.preview_scene = scene; + } + + /// Put the preview on program at once, ending any fade. + pub fn cut(&mut self) { + self.fade = None; + self.program_scene = self.preview_scene.clone(); + } + + /// Fade the preview onto program over `duration`, starting at `now`. + /// + /// A fade already under way lands on its scene first, so this one starts + /// from what that fade was heading to. + pub fn fade(&mut self, duration: Duration, now: Instant) { + let from = std::mem::replace(&mut self.program_scene, self.preview_scene.clone()); + self.fade = (!duration.is_zero()).then_some(Fade { + from, + start: now, + duration, + }); + } + + /// Draw preview and program as of `now`, from the newest picture each of + /// `sources` has. + /// + /// A source whose picture fails to draw keeps its last one and reports why + /// through [`error`](Self::error); this fails only when the canvas itself + /// cannot be drawn. + pub fn render(&mut self, collection: &Collection, sources: &Sources, now: Instant) -> Result<(), Error> { + self.update(collection, sources); + + let progress = self.fade.as_ref().map(|fade| { + let elapsed = now.saturating_duration_since(fade.start); + (elapsed.as_secs_f32() / fade.duration.as_secs_f32()).min(1.0) + }); + if progress.is_some_and(|progress| progress >= 1.0) { + self.fade = None; + } + + let mut passes = vec![Pass { + view: &self.preview.view, + draws: self.scene(collection, self.preview_scene.as_deref()), + }]; + match (&self.fade, progress) { + (Some(fade), Some(progress)) => { + passes.push(Pass { + view: &self.incoming.view, + draws: self.scene(collection, self.program_scene.as_deref()), + }); + let mut draws = self.scene(collection, fade.from.as_deref()); + draws.push(Draw { + bind: self.incoming.bind.clone(), + uniform: uniform(Rect::fill(&self.canvas), self.canvas, progress), + }); + passes.push(Pass { + view: &self.program.view, + draws, + }); + } + _ => passes.push(Pass { + view: &self.program.view, + draws: self.scene(collection, self.program_scene.as_deref()), + }), + } + + let count: usize = passes.iter().map(|pass| pass.draws.len()).sum(); + self.uniforms.reserve(&self.device, &self.uniform_layout, count as u64); + let stride = self.uniforms.stride as usize; + let mut bytes = vec![0u8; count * stride]; + for (index, draw) in passes.iter().flat_map(|pass| &pass.draws).enumerate() { + let offset = index * stride; + bytes[offset..offset + size_of::()].copy_from_slice(bytemuck::bytes_of(&draw.uniform)); + } + if !bytes.is_empty() { + self.queue.write_buffer(&self.uniforms.buffer, 0, &bytes); + } + + let mut encoder = self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("moq-scene"), + }); + let mut index = 0u32; + for pass in &passes { + let mut render = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("moq-scene"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: pass.view, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + render.set_pipeline(&self.pipeline); + for draw in &pass.draws { + render.set_bind_group(0, &self.uniforms.bind, &[index * stride as u32]); + render.set_bind_group(1, &draw.bind, &[]); + render.draw(0..4, 0..1); + index += 1; + } + } + self.queue.submit([encoder.finish()]); + Ok(()) + } + + /// The quads drawing `scene`, back to front. A missing scene, a hidden item, + /// or a source with no picture yet draws nothing. + fn scene(&self, collection: &Collection, scene: Option<&str>) -> Vec { + let Some(scene) = scene.and_then(|name| collection.scene(name)) else { + return Vec::new(); + }; + scene + .items + .iter() + .filter(|item| item.visible) + .filter_map(|item| { + let bind = self.layers.get(&item.source)?.bind.clone()?; + Some(Draw { + bind, + uniform: uniform(item.rect, self.canvas, 1.0), + }) + }) + .collect() + } + + /// Pull the newest picture from every source onto the GPU, and forget the + /// sources the collection no longer has or has since edited. + fn update(&mut self, collection: &Collection, sources: &Sources) { + self.layers + .retain(|name, layer| collection.sources.get(name) == Some(&layer.source)); + + for (name, source) in &collection.sources { + let Some(feed) = sources.get(name).filter(|feed| feed.source() == source) else { + continue; + }; + let Some(picture) = feed.take() else { + continue; + }; + let layer = self.layers.entry(name.clone()).or_insert_with(|| Layer { + source: source.clone(), + renderer: None, + bind: None, + error: None, + }); + let texture = match picture { + Picture::Frame(frame) => { + let renderer = match layer.renderer.as_mut() { + Some(renderer) => Ok(renderer), + None => Renderer::new(&self.device, &self.queue, Default::default()) + .map(|renderer| layer.renderer.insert(renderer)), + }; + renderer + .and_then(|renderer| renderer.render(&frame)) + .map_err(Error::from) + } + Picture::Rgba { size, pixels } => upload(&self.device, &self.queue, size, &pixels), + }; + + match texture { + Ok(texture) => { + layer.error = None; + let view = texture.create_view(&Default::default()); + layer.bind = Some(image_bind(&self.device, &self.image_layout, &self.sampler, &view)); + } + Err(err) => { + if layer.error.is_none() { + tracing::warn!(source = %name, %err, "cannot draw source"); + } + layer.error = Some(err.to_string()); + } + } + } + } +} + +/// Place `rect` (canvas pixels) in clip space. +fn uniform(rect: Rect, canvas: Canvas, opacity: f32) -> Uniform { + let (width, height) = (canvas.width as f32, canvas.height as f32); + let x = |px: f32| px / width * 2.0 - 1.0; + let y = |px: f32| 1.0 - px / height * 2.0; + Uniform { + rect: [x(rect.x), y(rect.y), x(rect.x + rect.width), y(rect.y + rect.height)], + opacity: [opacity, 0.0, 0.0, 0.0], + } +} + +/// A texture holding tightly packed RGBA pixels. +fn upload( + device: &wgpu::Device, + queue: &wgpu::Queue, + size: moq_video::Size, + pixels: &[u8], +) -> Result { + let limit = device.limits().max_texture_dimension_2d; + if size.width == 0 || size.height == 0 || size.width > limit || size.height > limit { + return Err(Error::Invalid(format!( + "a {size} picture does not fit this GPU's {limit}px texture limit" + ))); + } + + let extent = wgpu::Extent3d { + width: size.width, + height: size.height, + depth_or_array_layers: 1, + }; + let texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("moq-scene picture"), + size: extent, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: FORMAT, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + pixels, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(size.width * 4), + rows_per_image: Some(size.height), + }, + extent, + ); + Ok(texture) +} + +fn image_bind( + device: &wgpu::Device, + layout: &wgpu::BindGroupLayout, + sampler: &wgpu::Sampler, + view: &wgpu::TextureView, +) -> wgpu::BindGroup { + device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("moq-scene image"), + layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(sampler), + }, + ], + }) +} + +impl Target { + fn new( + device: &wgpu::Device, + layout: &wgpu::BindGroupLayout, + sampler: &wgpu::Sampler, + canvas: Canvas, + label: &str, + ) -> Self { + let texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size: wgpu::Extent3d { + width: canvas.width, + height: canvas.height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: FORMAT, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let view = texture.create_view(&Default::default()); + let bind = image_bind(device, layout, sampler, &view); + Self { texture, view, bind } + } +} + +impl Uniforms { + fn new(device: &wgpu::Device, layout: &wgpu::BindGroupLayout, stride: u64, capacity: u64) -> Self { + let buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("moq-scene items"), + size: stride * capacity, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + let bind = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("moq-scene items"), + layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::Buffer(wgpu::BufferBinding { + buffer: &buffer, + offset: 0, + size: NonZeroU64::new(size_of::() as u64), + }), + }], + }); + Self { + buffer, + bind, + stride, + capacity, + } + } + + /// Grow to hold `count` draws, doubling so a growing scene reallocates rarely. + fn reserve(&mut self, device: &wgpu::Device, layout: &wgpu::BindGroupLayout, count: u64) { + if count > self.capacity { + *self = Self::new(device, layout, self.stride, count.next_power_of_two()); + } + } +} + +#[cfg(test)] +mod tests { + use moq_net::Timestamp; + + use super::*; + use crate::collection::{Item, Rgba, Scene, Source}; + use crate::readback::Readback; + use crate::source::test::{context, local, start}; + + const RED: [u8; 4] = [255, 0, 0, 255]; + const BLUE: [u8; 4] = [0, 0, 255, 255]; + const BLACK: [u8; 4] = [0, 0, 0, 255]; + + async fn gpu() -> (wgpu::Device, wgpu::Queue) { + let instance = wgpu::Instance::default(); + let adapter = instance + .request_adapter(&wgpu::RequestAdapterOptions::default()) + .await + .expect("a GPU adapter"); + adapter + .request_device(&wgpu::DeviceDescriptor::default()) + .await + .expect("a GPU device") + } + + /// An 8x8 canvas with a red and a blue source, and scenes "red" (red filling + /// the canvas, blue over its top-right quarter) and "blue" (blue filling it). + fn collection() -> Collection { + let mut collection = Collection::new(); + collection.canvas = Canvas { + width: 8, + height: 8, + framerate: 30, + }; + collection + .sources + .insert("red".into(), Source::Color { color: Rgba(RED) }); + collection + .sources + .insert("blue".into(), Source::Color { color: Rgba(BLUE) }); + let item = |source: &str, rect| Item { + source: source.into(), + rect, + visible: true, + }; + let fill = Rect::fill(&collection.canvas); + let corner = Rect { + x: 4.0, + y: 0.0, + width: 4.0, + height: 4.0, + }; + collection.scenes = vec![ + Scene { + name: "red".into(), + items: vec![item("red", fill), item("blue", corner)], + }, + Scene { + name: "blue".into(), + items: vec![item("blue", fill)], + }, + ]; + collection.validate().unwrap(); + collection + } + + /// A compositor over running sources, read back to check pixels. + struct Studio { + collection: Collection, + sources: Sources, + compositor: Compositor, + readback: Readback, + _origin: moq_net::origin::Producer, + _dir: tempfile::TempDir, + } + + impl Studio { + async fn new(collection: Collection) -> Self { + let (device, queue) = gpu().await; + let dir = tempfile::tempdir().unwrap(); + let (context, origin) = context(dir.path()); + let mut sources = Sources::new(context); + let drivers = sources.sync(&collection); + start(&sources, drivers).await; + Self { + compositor: Compositor::new(&device, &queue, collection.canvas).unwrap(), + readback: Readback::new(&device, &queue, collection.canvas), + collection, + sources, + _origin: origin, + _dir: dir, + } + } + + fn render(&mut self, now: Instant) { + self.compositor.render(&self.collection, &self.sources, now).unwrap(); + } + + fn read(&mut self, texture: &wgpu::Texture) -> Vec { + self.readback.copy(texture, Timestamp::ZERO).unwrap().rgba().unwrap() + } + + fn program(&mut self) -> Vec { + let texture = self.compositor.program().clone(); + self.read(&texture) + } + + fn preview(&mut self) -> Vec { + let texture = self.compositor.preview().clone(); + self.read(&texture) + } + } + + fn pixel(rgba: &[u8], x: usize, y: usize) -> [u8; 4] { + let at = (y * 8 + x) * 4; + rgba[at..at + 4].try_into().unwrap() + } + + fn near(actual: [u8; 4], expected: [u8; 4]) -> bool { + actual.iter().zip(expected).all(|(a, e)| a.abs_diff(e) <= 1) + } + + /// Items draw where their rect says, later items over earlier ones, and + /// a hidden item not at all. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn items_draw_in_place_and_in_order() { + local(async { + let mut studio = Studio::new(collection()).await; + studio.compositor.set_preview(Some("red".into())); + studio.compositor.cut(); + studio.render(Instant::now()); + let program = studio.program(); + assert_eq!(pixel(&program, 1, 1), RED); + assert_eq!(pixel(&program, 6, 1), BLUE); + assert_eq!(pixel(&program, 6, 6), RED); + assert_eq!(studio.preview(), program, "preview shows the same scene"); + + studio.collection.scenes[0].items[1].visible = false; + studio.render(Instant::now()); + assert_eq!(pixel(&studio.program(), 6, 1), RED); + }) + .await; + } + + /// Straight alpha blends over what is below, and the canvas stays opaque. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn alpha_blends_over_black() { + local(async { + let mut collection = collection(); + collection.sources.insert( + "red".into(), + Source::Color { + color: Rgba([255, 0, 0, 128]), + }, + ); + collection.scenes[0].items.truncate(1); + let mut studio = Studio::new(collection).await; + studio.compositor.set_preview(Some("red".into())); + studio.compositor.cut(); + studio.render(Instant::now()); + let actual = pixel(&studio.program(), 1, 1); + assert!(near(actual, [128, 0, 0, 255]), "{actual:?}"); + }) + .await; + } + + /// A fade mixes the outgoing and incoming scenes by its progress on the + /// program clock, and lands on the incoming scene; preview never fades. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn a_fade_mixes_by_progress() { + local(async { + let mut studio = Studio::new(collection()).await; + let start = Instant::now(); + studio.compositor.set_preview(Some("red".into())); + studio.compositor.cut(); + studio.compositor.set_preview(Some("blue".into())); + studio.compositor.fade(Duration::from_secs(1), start); + assert_eq!(studio.compositor.program_scene(), Some("blue")); + + studio.render(start + Duration::from_millis(500)); + assert!(studio.compositor.fading()); + let program = studio.program(); + let actual = pixel(&program, 1, 1); + assert!(near(actual, [128, 0, 127, 255]), "{actual:?}"); + assert_eq!(pixel(&program, 6, 1), BLUE, "blue under blue stays blue"); + assert_eq!(pixel(&studio.preview(), 1, 1), BLUE); + + studio.render(start + Duration::from_secs(1)); + assert!(!studio.compositor.fading()); + assert_eq!(pixel(&studio.program(), 1, 1), BLUE); + }) + .await; + } + + /// A cut lands at once, even mid-fade, and an empty program is black. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn a_cut_ends_a_fade() { + local(async { + let mut studio = Studio::new(collection()).await; + let start = Instant::now(); + studio.render(start); + assert_eq!(pixel(&studio.program(), 1, 1), BLACK); + + studio.compositor.set_preview(Some("blue".into())); + studio.compositor.fade(Duration::from_secs(1), start); + studio.render(start + Duration::from_millis(500)); + studio.compositor.set_preview(Some("red".into())); + studio.compositor.cut(); + assert!(!studio.compositor.fading()); + studio.render(start + Duration::from_millis(600)); + assert_eq!(pixel(&studio.program(), 1, 1), RED); + }) + .await; + } + + /// A source with no new picture repeats its last one; an edited source never + /// shows the old one's picture, and a removed source disappears. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn a_stalled_source_repeats_its_last_picture() { + local(async { + let mut studio = Studio::new(collection()).await; + studio.compositor.set_preview(Some("blue".into())); + studio.compositor.cut(); + studio.render(Instant::now()); + assert!(studio.sources.get("blue").unwrap().take().is_none(), "drawn once"); + studio.render(Instant::now()); + assert_eq!(pixel(&studio.program(), 1, 1), BLUE); + + // An offline broadcast has no picture yet, so it draws nothing. + studio.collection.sources.insert( + "blue".into(), + Source::Broadcast { + path: "room/offline".into(), + }, + ); + let drivers = studio.sources.sync(&studio.collection); + start(&studio.sources, drivers).await; + studio.render(Instant::now()); + assert_eq!(pixel(&studio.program(), 1, 1), BLACK); + + studio.collection.sources.remove("blue"); + studio.collection.scenes.clear(); + studio.render(Instant::now()); + assert_eq!(pixel(&studio.program(), 1, 1), BLACK); + }) + .await; + } + + /// A picture the GPU cannot hold says why, rather than leaving its source + /// live and missing from the canvas. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn a_picture_that_cannot_draw_reports_why() { + local(async { + let mut collection = collection(); + let mut studio = Studio::new(collection.clone()).await; + let limit = studio.compositor.device.limits().max_texture_dimension_2d; + image::RgbaImage::new(limit + 1, 1) + .save(studio._dir.path().join("wide.png")) + .unwrap(); + collection.sources.insert( + "red".into(), + Source::Image { + path: studio._dir.path().join("wide.png"), + }, + ); + let drivers = studio.sources.sync(&collection); + start(&studio.sources, drivers).await; + studio.collection = collection; + studio.render(Instant::now()); + let err = studio.compositor.error("red").expect("the picture failed"); + assert!(err.contains("texture limit"), "{err}"); + assert!(studio.compositor.error("blue").is_none()); + }) + .await; + } + + /// A slow encoder runs out of buffers and drops pictures rather than + /// queuing them, and a finished copy frees its buffer. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn readback_drops_past_its_depth() { + local(async { + let mut studio = Studio::new(collection()).await; + studio.render(Instant::now()); + let program = studio.compositor.program().clone(); + let mut held: Vec<_> = (0..Readback::DEPTH) + .map(|_| studio.readback.copy(&program, Timestamp::ZERO).unwrap()) + .collect(); + assert!(studio.readback.copy(&program, Timestamp::ZERO).is_none()); + + let frame = held.pop().unwrap().wait().unwrap(); + assert_eq!(frame.size(), moq_video::Size::new(8, 8)); + held.push(studio.readback.copy(&program, Timestamp::ZERO).unwrap()); + assert!(studio.readback.copy(&program, Timestamp::ZERO).is_none()); + }) + .await; + } +} diff --git a/rs/moq-scene/src/lib.rs b/rs/moq-scene/src/lib.rs new file mode 100644 index 00000000000..50e1a5338fc --- /dev/null +++ b/rs/moq-scene/src/lib.rs @@ -0,0 +1,85 @@ +//! Scenes, live sources, and a GPU compositor for a MoQ streaming studio. +//! +//! The UI-free half of `moq-studio`, so the same scenes can be composed by a +//! windowed app, a headless renderer, or a test: +//! +//! - [`Collection`] is the scene graph: a [`Canvas`], named [`Source`]s, and +//! [`Scene`]s placing them. It saves to a versioned TOML file that holds no +//! credentials. +//! - [`source::Sources`] runs each source: a camera, display, window, +//! microphone, or system-audio capture (with the `capture` feature), a still +//! image, a solid color, or a broadcast subscribed over MoQ. Pictures go to the +//! compositor and sound goes to a `moq_audio::bus::Bus`, the program audio. +//! - [`Compositor`] draws the scene in preview and the scene on program into +//! textures on the caller's `wgpu` device, cutting or fading between them. +//! - [`Readback`] copies the program back to the CPU for an encoder that cannot +//! take a GPU surface. +//! +//! Every picture is drawn on the program clock the caller ticks +//! [`Compositor::render`] with, from whatever each source delivered last: a +//! source that stalls repeats its last picture rather than holding up the +//! canvas. + +mod collection; +mod compositor; +mod readback; +pub mod source; + +pub use collection::{Audio, Canvas, Collection, Item, Rect, Rgba, Scene, Source, Transition}; +pub use compositor::{Compositor, FORMAT}; +pub use readback::{Pending, Readback}; + +/// The `wgpu` the compositor draws with: `moq_video`'s, so its textures and +/// the decoder's are the same type. +pub use moq_video::render::wgpu; + +/// Errors from loading scenes, opening sources, and drawing them. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// The scene file is not valid TOML of the expected shape. + #[error("scene file: {0}")] + Format(String), + + /// The scene file is from another version of the format. + #[error("scene file version {found} is not supported; this build reads version {supported}")] + Version { + /// The version the file names. + found: i64, + /// The version this build reads. + supported: u32, + }, + + /// The collection breaks a rule the types cannot express, such as an item + /// naming a source that does not exist. + #[error("invalid scene collection: {0}")] + Invalid(String), + + /// An image source could not be read or decoded. + #[error("image: {0}")] + Image(String), + + /// This build or platform cannot open the source. + #[error("unsupported source: {0}")] + Unsupported(String), + + /// The GPU failed while reading a picture back. + #[error("gpu: {0}")] + Gpu(String), + + /// Capturing, decoding, or drawing video failed. + #[error(transparent)] + Video(#[from] moq_video::Error), + + /// Capturing, decoding, or mixing audio failed. + #[error(transparent)] + Audio(#[from] moq_audio::Error), + + /// Reading a broadcast's catalog or media failed. + #[error(transparent)] + Mux(#[from] moq_mux::Error), + + /// The broadcast subscription failed. + #[error(transparent)] + Net(#[from] moq_net::Error), +} diff --git a/rs/moq-scene/src/readback.rs b/rs/moq-scene/src/readback.rs new file mode 100644 index 00000000000..0ba7c9a7202 --- /dev/null +++ b/rs/moq-scene/src/readback.rs @@ -0,0 +1,190 @@ +//! Copy canvas pictures back to the CPU, for an encoder that cannot take a GPU +//! surface. + +use std::sync::{Arc, Mutex}; + +use moq_net::Timestamp; +use moq_video::render::wgpu; + +use crate::Error; +use crate::collection::Canvas; + +/// Copies of a canvas texture on their way back to the CPU, through a small +/// pool of staging buffers. +/// +/// [`copy`](Self::copy) records the copy and returns at once; the [`Pending`] +/// it hands back waits for the GPU and converts the pixels, on whatever thread +/// the encoder runs on, so the canvas keeps drawing meanwhile. A copy that +/// finds every buffer still in flight is refused rather than queued, so an +/// encoder that falls behind drops pictures instead of adding latency. +pub struct Readback { + device: wgpu::Device, + queue: wgpu::Queue, + canvas: Canvas, + /// Bytes per row in a staging buffer, padded to the copy alignment. + row: u32, + pool: Arc>, +} + +/// Staging buffers not held by a [`Pending`]. +#[derive(Default)] +struct Pool { + free: Vec, + /// Buffers in existence, up to [`Readback::DEPTH`]. + allocated: usize, +} + +impl Readback { + /// Copies in flight at once: one being converted, one being copied, and one + /// spare so a slow conversion does not drop the very next picture. + pub const DEPTH: usize = 3; + + /// Read back `canvas`-sized textures from `device`. + pub fn new(device: &wgpu::Device, queue: &wgpu::Queue, canvas: Canvas) -> Self { + Self { + device: device.clone(), + queue: queue.clone(), + canvas, + row: (canvas.width * 4).next_multiple_of(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT), + pool: Arc::default(), + } + } + + /// Start copying `texture`, a canvas-sized [`FORMAT`](crate::compositor::FORMAT) + /// texture created with `COPY_SRC`, to be encoded at `timestamp`. + /// + /// `None` when every buffer is still held by an earlier [`Pending`]. + pub fn copy(&mut self, texture: &wgpu::Texture, timestamp: Timestamp) -> Option { + let mut pool = self.pool.lock().unwrap(); + let buffer = match pool.free.pop() { + Some(buffer) => buffer, + None if pool.allocated < Self::DEPTH => { + pool.allocated += 1; + self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("moq-scene readback"), + size: u64::from(self.row) * u64::from(self.canvas.height), + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }) + } + None => return None, + }; + drop(pool); + + let mut encoder = self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("moq-scene readback"), + }); + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &buffer, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(self.row), + rows_per_image: Some(self.canvas.height), + }, + }, + wgpu::Extent3d { + width: self.canvas.width, + height: self.canvas.height, + depth_or_array_layers: 1, + }, + ); + let submission = self.queue.submit([encoder.finish()]); + + Some(Pending { + device: self.device.clone(), + buffer: Some(buffer), + submission, + canvas: self.canvas, + row: self.row, + timestamp, + pool: self.pool.clone(), + mapping: false, + }) + } +} + +/// One copy in flight. Dropping it returns its buffer to the pool. +pub struct Pending { + device: wgpu::Device, + buffer: Option, + submission: wgpu::SubmissionIndex, + canvas: Canvas, + row: u32, + timestamp: Timestamp, + pool: Arc>, + /// Set while a mapping was requested and not yet undone, so a wait that + /// failed partway does not hand a mapped buffer to the next copy. + mapping: bool, +} + +impl Pending { + /// Block until the copy lands, then hand back the picture as a frame an + /// encoder takes. + pub fn wait(mut self) -> Result { + let rgba = self.rgba()?; + let surface = moq_video::Surface::rgba(&rgba, self.canvas.size())?; + Ok(moq_video::Frame::new(surface, self.timestamp)) + } + + /// Block until the copy lands, then hand back tightly packed RGBA rows. + pub fn rgba(&mut self) -> Result, Error> { + let buffer = self.buffer.as_ref().expect("held until drop"); + let (send, recv) = std::sync::mpsc::channel(); + self.mapping = true; + buffer.map_async(wgpu::MapMode::Read, .., move |result| { + let _ = send.send(result); + }); + self.device + .poll(wgpu::PollType::Wait { + submission_index: Some(self.submission.clone()), + timeout: None, + }) + .map_err(|err| Error::Gpu(format!("waiting for the readback: {err}")))?; + recv.recv() + .map_err(|_| Error::Gpu("the readback was dropped".into()))? + .map_err(|err| Error::Gpu(format!("mapping the readback: {err}")))?; + + let width = self.canvas.width as usize * 4; + let mut rgba = Vec::with_capacity(width * self.canvas.height as usize); + { + let mapped = buffer + .slice(..) + .get_mapped_range() + .map_err(|err| Error::Gpu(format!("reading the readback: {err}")))?; + for row in mapped.chunks_exact(self.row as usize) { + rgba.extend_from_slice(&row[..width]); + } + } + buffer.unmap(); + self.mapping = false; + Ok(rgba) + } + + /// When the picture is shown. + pub fn timestamp(&self) -> Timestamp { + self.timestamp + } +} + +impl Drop for Pending { + fn drop(&mut self) { + let Some(buffer) = self.buffer.take() else { + return; + }; + let mut pool = self.pool.lock().unwrap(); + // The pool only holds buffers the next copy can write. One a failed wait + // left mapped, or waiting on a mapping, is let go and replaced. + if self.mapping { + pool.allocated -= 1; + } else { + pool.free.push(buffer); + } + } +} diff --git a/rs/moq-scene/src/source.rs b/rs/moq-scene/src/source.rs new file mode 100644 index 00000000000..a37ad2dc038 --- /dev/null +++ b/rs/moq-scene/src/source.rs @@ -0,0 +1,801 @@ +//! Live sources: what each [`Source`] of a collection shows and plays right now. +//! +//! [`Sources::sync`] keeps one [`Feed`] per source and hands back a [`Driver`] +//! for each one it opens. The caller runs the drivers wherever suits it (a +//! capture driver is `!Send` on macOS); a driver stops once its feed is +//! dropped, which happens when the source is removed or edited. + +use std::collections::BTreeMap; +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; + +use hang::catalog::Reference; +use moq_audio::bus::{Bus, Strip}; +use moq_mux::catalog::Stream; + +use crate::Error; +use crate::collection::{Collection, Source}; + +/// A source's newest picture, waiting for the compositor to draw it. +pub(crate) enum Picture { + /// A captured or decoded video frame. + Frame(moq_video::Frame), + /// Tightly packed RGBA with straight alpha, such as a still image. + Rgba { size: moq_video::Size, pixels: Vec }, +} + +/// The one picture a source holds for the compositor. A newer picture replaces +/// an undrawn one, so a slow canvas skips pictures instead of queuing them. +type Mailbox = Arc>>; + +/// One instance of a broadcast being decoded. +type Playing<'a> = Pin> + 'a>>; + +/// What a running source is doing. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum Status { + /// Opening its device, file, or subscription. + Starting, + /// Producing pictures or sound. + Live, + /// Waiting for its broadcast to be announced. + Offline, + /// Stopped, with the reason. A broadcast source keeps waiting for the + /// broadcast to come back; every other source stays stopped until edited. + Failed(String), +} + +/// What every source shares. +#[derive(Clone)] +#[non_exhaustive] +pub struct Context { + /// The program audio that microphones, system audio, and broadcasts feed. + pub bus: Bus, + /// Where broadcast sources are announced: the studio's relay connection. + pub origin: moq_net::origin::Consumer, + /// What a relative image path is relative to: the scene file's directory. + pub dir: PathBuf, +} + +impl Context { + /// Sources feeding `bus`, subscribing through `origin`, with images + /// relative to `dir`. + pub fn new(bus: Bus, origin: moq_net::origin::Consumer, dir: PathBuf) -> Self { + Self { bus, origin, dir } + } +} + +/// The running sources of a collection. +pub struct Sources { + context: Context, + feeds: BTreeMap, +} + +impl Sources { + /// No sources running yet; [`sync`](Self::sync) starts them. + pub fn new(context: Context) -> Self { + Self { + context, + feeds: BTreeMap::new(), + } + } + + /// What every source shares. + pub fn context(&self) -> &Context { + &self.context + } + + /// Run exactly the sources `collection` has: start new and edited ones and + /// stop removed and edited ones. Returns a driver for each source started, + /// which does nothing until run. + /// + /// Also applies the collection's gain and mute to every audio strip, which + /// never restarts a source. Panics on a gain [`Collection::validate`] refuses. + pub fn sync(&mut self, collection: &Collection) -> Vec { + self.feeds + .retain(|name, feed| collection.sources.get(name) == Some(&feed.source)); + + let mut drivers = Vec::new(); + for (name, source) in &collection.sources { + if !self.feeds.contains_key(name) { + let (feed, driver) = open(name, source, &self.context); + self.feeds.insert(name.clone(), feed); + drivers.push(driver); + } + + if let Some(strip) = self.feeds[name].strip() { + let audio = collection.audio.get(name).copied().unwrap_or_default(); + strip.set_gain(audio.gain); + strip.set_muted(audio.muted); + } + } + drivers + } + + /// The running source called `name`. + pub fn get(&self, name: &str) -> Option<&Feed> { + self.feeds.get(name) + } + + /// Every running source, by name. + pub fn iter(&self) -> impl Iterator { + self.feeds.iter().map(|(name, feed)| (name.as_str(), feed)) + } +} + +/// One running source. Dropping it stops its [`Driver`]. +pub struct Feed { + source: Source, + picture: Mailbox, + strip: Option, + status: kio::Consumer, +} + +impl Feed { + /// What this feed was opened for. + pub fn source(&self) -> &Source { + &self.source + } + + /// What the source is doing. + pub fn status(&self) -> Status { + self.status.read().clone() + } + + /// The source's gain, mute, and meter in the program audio, if it has sound. + pub fn strip(&self) -> Option<&Strip> { + self.strip.as_ref() + } + + /// The newest picture not yet drawn. + pub(crate) fn take(&self) -> Option { + self.picture.lock().unwrap().take() + } +} + +/// The work behind one [`Feed`]: opening the source and delivering what it +/// produces. Built by [`Sources::sync`]. +pub struct Driver { + name: String, + source: Source, + context: Context, + picture: Mailbox, + strip: Option, + status: kio::Producer, +} + +impl Driver { + /// The source's name in the collection. + pub fn name(&self) -> &str { + &self.name + } + + /// Run the source until its [`Feed`] is dropped. + /// + /// A failure is reported through [`Feed::status`] rather than returned. On + /// macOS a capture source's future is `!Send`, so run it on a local task. + pub async fn run(self) { + let status = self.status.clone(); + let result = tokio::select! { + _ = status.unused() => return, + result = self.produce() => result, + }; + let status = match result { + Ok(()) => Status::Failed("the source ended".into()), + Err(err) => Status::Failed(err.to_string()), + }; + tracing::warn!(source = %self.name, ?status, "source stopped"); + set(&self.status, status); + } + + async fn produce(&self) -> Result<(), Error> { + match &self.source { + Source::Color { color } => { + self.show(Picture::Rgba { + size: moq_video::Size::new(1, 1), + pixels: color.0.to_vec(), + }); + std::future::pending().await + } + Source::Image { path } => { + let picture = image(&self.context.dir.join(path))?; + self.show(picture); + std::future::pending().await + } + Source::Broadcast { path } => self.broadcast(path).await, + device => self.device(device).await, + } + } + + fn show(&self, picture: Picture) { + *self.picture.lock().unwrap() = Some(picture); + set(&self.status, Status::Live); + } + + /// Capture a camera, display, window, or audio input. + #[cfg(feature = "capture")] + async fn device(&self, source: &Source) -> Result<(), Error> { + use moq_audio::capture::Source as Audio; + use moq_video::capture::Source as Video; + + let video = match source { + Source::Camera { id } => Video::Camera(id.clone()), + Source::Display { id } => Video::Display(id.clone()), + Source::Window { id } => Video::Window(id.clone()), + Source::Microphone { id } => return self.listen(Audio::Microphone(id.clone())).await, + Source::SystemAudio => return self.listen(Audio::System).await, + Source::Color { .. } | Source::Image { .. } | Source::Broadcast { .. } => { + unreachable!("not a device source") + } + }; + + let mut config = moq_video::capture::Config::default(); + config.source = video; + let mut stream = moq_video::capture::open(&config).await?; + while let Some(frame) = stream.read().await? { + self.show(Picture::Frame(frame)); + } + Ok(()) + } + + #[cfg(feature = "capture")] + async fn listen(&self, source: moq_audio::capture::Source) -> Result<(), Error> { + let strip = self.strip.clone().expect("audio sources have a strip"); + let mut config = moq_audio::capture::Config::default(); + config.source = source; + let capture = self.context.bus.capture(config, strip); + set(&self.status, Status::Live); + Ok(capture.run().await?) + } + + #[cfg(not(feature = "capture"))] + async fn device(&self, _: &Source) -> Result<(), Error> { + Err(Error::Unsupported( + "this build cannot capture devices (enable the `capture` feature)".into(), + )) + } + + /// Follow the broadcast at `path` for as long as the feed lives, playing + /// each instance of it that comes online. + async fn broadcast(&self, path: &str) -> Result<(), Error> { + let mut instances = moq_mux::follow::Instances::new(&self.context.origin, path, None)?; + set(&self.status, Status::Offline); + + let mut playing: Option> = None; + loop { + tokio::select! { + event = instances.next() => match event { + Ok(moq_mux::follow::Event::Start(broadcast)) => { + set(&self.status, Status::Live); + playing = Some(Box::pin(self.play(broadcast, path))); + } + Ok(moq_mux::follow::Event::End) => { + playing = None; + set(&self.status, Status::Offline); + } + Err(err) => return Err(err.into()), + }, + result = async { playing.as_mut().expect("guarded").await }, if playing.is_some() => { + playing = None; + set(&self.status, match result { + Ok(()) => Status::Offline, + Err(err) => Status::Failed(err.to_string()), + }); + } + } + } + } + + /// Decode one instance of a broadcast: the first video and audio renditions + /// each catalog offers, switching when a catalog update changes them, until + /// the catalog ends. + async fn play(&self, broadcast: moq_net::broadcast::Consumer, path: &str) -> Result<(), Error> { + let format = moq_mux::catalog::CatalogFormat::detect(path).unwrap_or_default(); + let mut catalogs = moq_mux::catalog::Consumer::<()>::new(&broadcast, format).await?; + + // A track name always means the same content, so only a new track + // restarts a decoder. That is the track an entry resolves to, not its + // catalog ID: a publisher may keep the ID and point it at a replacement + // track, as the JS publisher does on a codec change. A metadata-only + // update, such as a jitter estimate, leaves playback alone. + let mut video: Option = None; + let mut audio: Option = None; + let mut video_playing: Playing<'_> = Box::pin(std::future::pending()); + let mut audio_playing: Playing<'_> = Box::pin(std::future::pending()); + + loop { + tokio::select! { + catalog = catalogs.next() => { + let Some(catalog) = catalog? else { + return Ok(()); + }; + let next = pick(catalog.video.enabled())?; + let track = next.as_ref().map(|(id, config)| Reference::track(config, id).to_string()); + if track != video { + video = track; + video_playing = match next { + Some((id, config)) => Box::pin(self.play_video(&broadcast, id, config)), + None => Box::pin(std::future::pending()), + }; + } + let next = pick(catalog.audio.enabled())?; + let track = next.as_ref().map(|(id, config)| Reference::track(config, id).to_string()); + if track != audio { + audio = track; + audio_playing = match next { + Some((id, config)) => Box::pin(self.play_audio(&broadcast, id, config)), + None => Box::pin(std::future::pending()), + }; + } + } + // A rendition that ends waits for a catalog update to replace it. + result = &mut video_playing => { + result?; + video_playing = Box::pin(std::future::pending()); + } + result = &mut audio_playing => { + result?; + audio_playing = Box::pin(std::future::pending()); + } + } + } + } + + async fn play_video( + &self, + broadcast: &moq_net::broadcast::Consumer, + name: String, + config: hang::catalog::VideoConfig, + ) -> Result<(), Error> { + let mut options = moq_video::decode::Options::new(); + options.start = moq_video::decode::Start::Latest; + let mut consumer = moq_video::decode::Consumer::new(broadcast, &config, name, options).await?; + while let Some(read) = consumer.read().await? { + if let moq_video::decode::Read::Frame(frame) = read { + self.show(Picture::Frame(frame)); + } + } + Ok(()) + } + + async fn play_audio( + &self, + broadcast: &moq_net::broadcast::Consumer, + name: String, + config: hang::catalog::AudioConfig, + ) -> Result<(), Error> { + let Some(strip) = self.strip.clone() else { + return Ok(()); + }; + let bus = self.context.bus.config().clone(); + let mut options = moq_audio::decode::Options::new(); + options.start = moq_audio::decode::Start::Latest; + options.output.format = moq_audio::Format::F32; + options.output.sample_rate = Some(bus.sample_rate); + options.output.layout = Some(bus.layout); + let mut consumer = moq_audio::decode::Consumer::new(broadcast, &config, name, options).await?; + + let mut input = moq_audio::encode::Input::new(bus.sample_rate, bus.layout); + input.format = moq_audio::Format::F32; + let mut channel = self.context.bus.channel(input, strip)?; + // Gaps and encoder boundaries need nothing. Like a picture, which shows + // on arrival, the bus plays a source at its live edge on the program + // clock: a stall plays as silence while it lasts, and a burst after it + // is bounded by the bus's buffer. Replaying the publisher's + // timeline, holes included, would put the audio behind the live edge + // and its own video. + while let Some(read) = consumer.read().await? { + if let moq_audio::decode::Read::Frame(frame) = read { + channel.write(&frame.data)?; + } + } + Ok(()) + } +} + +/// The best enabled rendition the broadcast serves itself, of `renditions` +/// ranked best first. +/// +/// One served from another broadcast would need that broadcast's own +/// subscription, which a source does not open, so a catalog offering only +/// those is refused rather than shown as black or played as silence. +fn pick<'a, C: Reference + Clone + 'a>( + renditions: impl Iterator, +) -> Result, Error> { + let mut foreign = None; + for (id, config) in renditions { + if config.broadcast().is_none() { + return Ok(Some((id.clone(), config.clone()))); + } + foreign.get_or_insert(id); + } + match foreign { + Some(id) => Err(Error::Unsupported(format!( + "rendition {id} is served from another broadcast" + ))), + None => Ok(None), + } +} + +/// Start the feed for `source`, and the driver that fills it. +fn open(name: &str, source: &Source, context: &Context) -> (Feed, Driver) { + let picture = Mailbox::default(); + let strip = source.has_audio().then(Strip::default); + let status = kio::Producer::new(Status::Starting); + + let feed = Feed { + source: source.clone(), + picture: picture.clone(), + strip: strip.clone(), + status: status.consume(), + }; + let driver = Driver { + name: name.to_string(), + source: source.clone(), + context: context.clone(), + picture, + strip, + status, + }; + (feed, driver) +} + +fn set(status: &kio::Producer, value: Status) { + if let Ok(mut status) = status.write() { + *status = value; + } +} + +/// Decode a still image to straight-alpha RGBA. +fn image(path: &Path) -> Result { + let bytes = std::fs::read(path).map_err(|err| Error::Image(format!("{}: {err}", path.display())))?; + let image = image::load_from_memory(&bytes) + .map_err(|err| Error::Image(format!("{}: {err}", path.display())))? + .into_rgba8(); + Ok(Picture::Rgba { + size: moq_video::Size::new(image.width(), image.height()), + pixels: image.into_raw(), + }) +} + +/// Helpers for tests across the crate. +#[cfg(test)] +pub(crate) mod test { + use super::*; + + /// A context with no relay behind it, so broadcasts stay offline while the + /// returned origin lives. Call from a [`tokio::task::LocalSet`]. + pub(crate) fn context(dir: &Path) -> (Context, moq_net::origin::Producer) { + let (origin, driver) = moq_net::origin::Producer::new(Default::default()); + tokio::task::spawn_local(moq_net::time::run(driver)); + let bus = Bus::new(Default::default()).unwrap(); + (Context::new(bus, origin.consume(), dir.to_path_buf()), origin) + } + + /// Run `drivers` on the local task set, and wait until every source in + /// `sources` has left [`Status::Starting`]. + pub(crate) async fn start(sources: &Sources, drivers: Vec) { + for driver in drivers { + tokio::task::spawn_local(driver.run()); + } + for _ in 0..1000 { + if sources.iter().all(|(_, feed)| feed.status() != Status::Starting) { + return; + } + tokio::task::yield_now().await; + } + panic!("sources never started"); + } + + /// Run `f` on a local task set, as drivers need. + pub(crate) async fn local(f: F) -> F::Output { + tokio::task::LocalSet::new().run_until(f).await + } +} + +#[cfg(test)] +mod tests { + use super::test::{context, local, start}; + use super::*; + use crate::collection::{Audio, Rgba}; + + fn collection() -> Collection { + let mut collection = Collection::new(); + collection.sources.insert( + "red".into(), + Source::Color { + color: Rgba([255, 0, 0, 255]), + }, + ); + collection.sources.insert( + "guest".into(), + Source::Broadcast { + path: "room/guest".into(), + }, + ); + collection + } + + #[tokio::test] + async fn a_color_shows_one_pixel() { + local(async { + let dir = tempfile::tempdir().unwrap(); + let (context, _origin) = context(dir.path()); + let mut sources = Sources::new(context); + let drivers = sources.sync(&collection()); + assert_eq!(drivers.len(), 2); + start(&sources, drivers).await; + + let red = sources.get("red").unwrap(); + assert_eq!(red.status(), Status::Live); + assert!(red.strip().is_none(), "a color is silent"); + let Some(Picture::Rgba { size, pixels }) = red.take() else { + panic!("a color shows a picture"); + }; + assert_eq!(size, moq_video::Size::new(1, 1)); + assert_eq!(pixels, [255, 0, 0, 255]); + assert!(red.take().is_none(), "taken once"); + + let guest = sources.get("guest").unwrap(); + assert_eq!(guest.status(), Status::Offline, "nothing is announced"); + assert!(guest.strip().is_some(), "a broadcast feeds the bus"); + }) + .await; + } + + /// Editing a source restarts it, removing one stops it, and an untouched one + /// keeps running. Gain changes reach the strip without restarting anything. + #[tokio::test] + async fn sync_restarts_only_what_changed() { + local(async { + let dir = tempfile::tempdir().unwrap(); + let (context, _origin) = context(dir.path()); + let mut sources = Sources::new(context); + let mut collection = collection(); + let drivers = sources.sync(&collection); + start(&sources, drivers).await; + let guest = sources.get("guest").unwrap().strip().unwrap().clone(); + + collection.audio.insert( + "guest".into(), + Audio { + gain: 0.25, + muted: true, + }, + ); + assert!(sources.sync(&collection).is_empty()); + assert_eq!(guest.gain(), 0.25); + assert!(guest.muted()); + + collection.sources.insert( + "red".into(), + Source::Color { + color: Rgba([0, 0, 255, 255]), + }, + ); + let drivers = sources.sync(&collection); + assert_eq!(drivers.iter().map(Driver::name).collect::>(), ["red"]); + start(&sources, drivers).await; + let Some(Picture::Rgba { pixels, .. }) = sources.get("red").unwrap().take() else { + panic!("the edited color shows"); + }; + assert_eq!(pixels, [0, 0, 255, 255]); + + collection.sources.remove("red"); + assert!(sources.sync(&collection).is_empty()); + assert!(sources.get("red").is_none()); + }) + .await; + } + + /// A broadcast source follows catalog updates: when a better rendition + /// appears, it switches to it without the broadcast restarting. + #[tokio::test] + async fn a_broadcast_follows_its_catalog() { + local(async { + let dir = tempfile::tempdir().unwrap(); + let (context, origin) = context(dir.path()); + let publication = origin.create_broadcast("room/guest").unwrap(); + let mut broadcast = publication.broadcast().clone(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + // A known bitrate ranks a rendition above one without. + let audio = |name: &str, bitrate: Option| { + let track = broadcast + .create_track(name, catalog.track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut config = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); + config.bitrate = bitrate; + catalog + .audio( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), + config, + ) + .unwrap() + }; + let first = audio("first", None); + publication + .announce(moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint())) + .unwrap(); + + let mut sources = Sources::new(context); + let drivers = sources.sync(&collection()); + start(&sources, drivers).await; + + let mut first = first.demand(); + while first.wanted_changed().await.unwrap().is_none() {} + + let second = audio("second", Some(128_000)); + let mut second = second.demand(); + while second.wanted_changed().await.unwrap().is_none() {} + while first.wanted_changed().await.unwrap().is_some() {} + assert_eq!(sources.get("guest").unwrap().status(), Status::Live); + }) + .await; + } + + /// A rendition that keeps its catalog ID but points at a replacement track, + /// as the JS publisher does on a codec change, moves playback to that track. + #[tokio::test] + async fn a_broadcast_follows_a_renditions_track() { + local(async { + let dir = tempfile::tempdir().unwrap(); + let (context, origin) = context(dir.path()); + let publication = origin.create_broadcast("room/guest").unwrap(); + let mut broadcast = publication.broadcast().clone(); + let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let info = catalog.track_info(hang::catalog::PRIORITY.audio); + let first = broadcast.create_track("first", info.clone()).unwrap(); + let second = broadcast.create_track("second", info).unwrap(); + let point = |catalog: &mut moq_mux::catalog::Producer, track: &str| { + let mut config = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); + config.track = Some(track.into()); + catalog + .mutate(|catalog| { + catalog.audio.renditions.insert("audio".into(), config); + }) + .unwrap(); + }; + point(&mut catalog, "first"); + publication + .announce(moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint())) + .unwrap(); + + let mut sources = Sources::new(context); + let drivers = sources.sync(&collection()); + start(&sources, drivers).await; + + let mut first = first.demand(); + while first.wanted_changed().await.unwrap().is_none() {} + + point(&mut catalog, "second"); + let mut second = second.demand(); + while second.wanted_changed().await.unwrap().is_none() {} + while first.wanted_changed().await.unwrap().is_some() {} + assert_eq!(sources.get("guest").unwrap().status(), Status::Live); + }) + .await; + } + + /// A catalog whose only rendition lives in another broadcast fails the + /// source rather than leaving it live and silent. + #[tokio::test] + async fn a_foreign_rendition_is_refused() { + local(async { + let dir = tempfile::tempdir().unwrap(); + let (context, origin) = context(dir.path()); + let publication = origin.create_broadcast("room/guest").unwrap(); + let mut broadcast = publication.broadcast().clone(); + let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let mut config = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); + config.broadcast = Some("../other".into()); + catalog + .mutate(|catalog| { + catalog.audio.renditions.insert("audio".into(), config); + }) + .unwrap(); + publication + .announce(moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint())) + .unwrap(); + + let mut sources = Sources::new(context); + let drivers = sources.sync(&collection()); + start(&sources, drivers).await; + + let guest = sources.get("guest").unwrap(); + for _ in 0..1000 { + if let Status::Failed(reason) = guest.status() { + assert!(reason.contains("another broadcast"), "{reason}"); + return; + } + tokio::task::yield_now().await; + } + panic!("the source never failed: {:?}", guest.status()); + }) + .await; + } + + /// Dropping a feed stops its driver, which is what releases a device. + #[tokio::test] + async fn a_dropped_feed_stops_its_driver() { + local(async { + let dir = tempfile::tempdir().unwrap(); + let (context, _origin) = context(dir.path()); + let mut sources = Sources::new(context); + let mut drivers = sources.sync(&collection()); + let driver = drivers.pop().unwrap(); + let running = tokio::task::spawn_local(driver.run()); + tokio::task::yield_now().await; + assert!(!running.is_finished()); + + sources.sync(&Collection::new()); + running.await.unwrap(); + }) + .await; + } + + #[tokio::test] + async fn an_image_loads_relative_to_the_scene_file() { + local(async { + let dir = tempfile::tempdir().unwrap(); + let mut png = image::RgbaImage::new(3, 2); + png.put_pixel(2, 1, image::Rgba([1, 2, 3, 4])); + png.save(dir.path().join("logo.png")).unwrap(); + + let mut collection = Collection::new(); + collection.sources.insert( + "logo".into(), + Source::Image { + path: "logo.png".into(), + }, + ); + collection.sources.insert( + "missing".into(), + Source::Image { + path: "missing.png".into(), + }, + ); + let (context, _origin) = context(dir.path()); + let mut sources = Sources::new(context); + let drivers = sources.sync(&collection); + start(&sources, drivers).await; + + let Some(Picture::Rgba { size, pixels }) = sources.get("logo").unwrap().take() else { + panic!("an image shows a picture"); + }; + assert_eq!(size, moq_video::Size::new(3, 2)); + assert_eq!(pixels[pixels.len() - 4..], [1, 2, 3, 4]); + + let Status::Failed(reason) = sources.get("missing").unwrap().status() else { + panic!("a missing image fails"); + }; + assert!(reason.contains("missing.png"), "{reason}"); + }) + .await; + } + + #[cfg(not(feature = "capture"))] + #[tokio::test] + async fn a_build_without_capture_says_so() { + local(async { + let dir = tempfile::tempdir().unwrap(); + let mut collection = Collection::new(); + collection.sources.insert("cam".into(), Source::Camera { id: None }); + collection.sources.insert("mic".into(), Source::Microphone { id: None }); + let (context, _origin) = context(dir.path()); + let mut sources = Sources::new(context); + let drivers = sources.sync(&collection); + start(&sources, drivers).await; + for name in ["cam", "mic"] { + let Status::Failed(reason) = sources.get(name).unwrap().status() else { + panic!("{name} cannot capture"); + }; + assert!(reason.contains("capture"), "{reason}"); + } + }) + .await; + } +} diff --git a/rs/moq-studio/Cargo.toml b/rs/moq-studio/Cargo.toml new file mode 100644 index 00000000000..824ebc7d333 --- /dev/null +++ b/rs/moq-studio/Cargo.toml @@ -0,0 +1,64 @@ +[package] +# Unpublished until the packaging quest decides how the studio ships. +publish = false +name = "moq-studio" +description = "A native streaming studio that composites scenes on the GPU and goes live over MoQ" +authors = ["Luke Curley "] +repository = "https://github.com/moq-dev/moq" +license = "MIT OR Apache-2.0" + +version = "0.1.0" +edition = "2024" +# egui 0.36 and egui_dock 0.21 need 1.92 and 1.95. This is an application, so +# the bump stays here rather than raising the libraries' floor. +rust-version = "1.95" + +keywords = ["quic", "streaming", "studio", "live", "media"] +categories = ["multimedia", "multimedia::video", "gui"] + +[features] +default = ["quic", "websocket", "nvidia"] +quic = ["moq-tokio/quic"] +websocket = ["moq-tokio/websocket"] +# Camera, display, window, microphone, and system-audio sources. Off by default +# like moq-cli's `capture`: on Linux the microphone links libasound at build +# time. Enable with `cargo run -p moq-studio --features capture,openh264`. +capture = ["moq-scene/capture"] +# NVIDIA hardware codecs (NVENC + NVDEC) on Linux, as in moq-cli. +nvidia = ["moq-video/nvidia"] +# The vendored OpenH264 software H.264 fallback, opt-in like moq-video's. +openh264 = ["moq-video/openh264"] +# Intel/AMD VAAPI hardware codecs (Linux); its bindgen needs libclang. +vaapi = ["moq-video/vaapi"] +# PipeWire cameras and audio. Wayland screens and windows are not reachable yet: +# display and window sources capture X11 only. +pipewire = ["capture", "moq-video/pipewire", "moq-audio/pipewire"] + +[dependencies] +anyhow = { workspace = true, features = ["backtrace"] } +bytes = { workspace = true } +# The window, its event loop, and the egui integration. `wgpu` is the renderer +# the compositor shares a device with; no glow, no system font discovery. +eframe = { version = "0.36", default-features = false, features = ["default_fonts", "wayland", "wgpu", "x11"] } +egui = "0.36" +# Docked, rearrangeable panels. 0.21 is the newest release and pins egui 0.36. +egui_dock = "0.21" +moq-audio = { workspace = true, features = ["bus"] } +moq-clock = { workspace = true } +moq-mux = { workspace = true } +moq-net = { workspace = true } +moq-scene = { workspace = true } +moq-tokio = { workspace = true, default-features = false, features = ["aws-lc-rs", "net"] } +moq-video = { workspace = true } +pollster = { workspace = true } +tokio = { workspace = true, features = ["full"] } +tracing = { workspace = true } +url = { workspace = true } +usage-rs = { workspace = true, features = ["completions", "config"] } + +[dev-dependencies] +moq-relay = { path = "../moq-relay", default-features = false, features = ["test-support"] } +tempfile = { workspace = true } + +[lints] +workspace = true diff --git a/rs/moq-studio/src/engine.rs b/rs/moq-studio/src/engine.rs new file mode 100644 index 00000000000..15a20e819cd --- /dev/null +++ b/rs/moq-studio/src/engine.rs @@ -0,0 +1,659 @@ +//! Everything behind the window: the GPU, the sources, the program clock that +//! draws the canvas and mixes the audio, and the broadcast it publishes. +//! +//! Threads: +//! - **program** draws preview and program once per canvas frame and hands the +//! program to the encoder, stamped with the program clock. +//! - **audio** reads the program bus on the same clock and publishes it. +//! - **encode** reads the program back from the GPU and encodes it. +//! - **source-***: one per running source, since a capture stream may not +//! leave the thread that opened it. +//! - The UI runs on the main thread and edits [`State`] under its lock. + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, mpsc}; +use std::time::Duration; + +use anyhow::Context; +use bytes::Bytes; +use moq_clock::Instant; +use moq_scene::source::{self, Driver, Sources}; +use moq_scene::{Collection, Compositor, Pending, Readback, wgpu}; + +/// The GPU everything shares: sources upload to it, the compositor draws on +/// it, the window presents from it, and the encoder reads back from it. +#[derive(Clone)] +pub struct Gpu { + pub instance: wgpu::Instance, + pub adapter: wgpu::Adapter, + pub device: wgpu::Device, + pub queue: wgpu::Queue, +} + +impl Gpu { + pub async fn new() -> anyhow::Result { + let instance = wgpu::Instance::default(); + let adapter = instance + .request_adapter(&wgpu::RequestAdapterOptions { + power_preference: wgpu::PowerPreference::HighPerformance, + ..Default::default() + }) + .await + .context("no GPU adapter")?; + // Lets a hardware-decoded or captured DMA-BUF reach the compositor + // without a copy, where the driver can import one. + let features = adapter.features() & wgpu::Features::VULKAN_EXTERNAL_MEMORY_DMA_BUF; + let (device, queue) = adapter + .request_device(&wgpu::DeviceDescriptor { + label: Some("moq-studio"), + required_features: features, + ..Default::default() + }) + .await + .context("cannot open the GPU")?; + Ok(Self { + instance, + adapter, + device, + queue, + }) + } +} + +/// What the UI edits and the program clock draws. +pub struct State { + /// The scene file, saved to and reloaded from. + pub path: PathBuf, + pub collection: Collection, + pub sources: Sources, + pub compositor: Compositor, + /// Edited since the sources were last synced. + stale: bool, + /// Edited since the last save or load. + unsaved: bool, +} + +impl State { + /// The collection, for an edit: the sources follow it on the next frame. + pub fn edit(&mut self) -> &mut Collection { + self.stale = true; + self.unsaved = true; + &mut self.collection + } + + pub fn unsaved(&self) -> bool { + self.unsaved + } + + pub fn save(&mut self) -> anyhow::Result<()> { + let text = self.collection.to_toml()?; + std::fs::write(&self.path, text).with_context(|| format!("writing {}", self.path.display()))?; + self.unsaved = false; + Ok(()) + } + + /// Replace the collection with what the scene file holds now. + pub fn reload(&mut self) -> anyhow::Result<()> { + let collection = load(&self.path)?; + anyhow::ensure!( + collection.canvas == self.collection.canvas, + "the canvas changed; restart the studio to use {}x{} at {} fps", + collection.canvas.width, + collection.canvas.height, + collection.canvas.framerate + ); + // A scene in preview or on program that the file no longer has draws + // black until another is picked. + self.collection = collection; + self.stale = true; + self.unsaved = false; + Ok(()) + } +} + +/// Read a scene file, or start a new collection if there is none yet. +pub fn load(path: &Path) -> anyhow::Result { + match std::fs::read_to_string(path) { + Ok(text) => Collection::from_toml(&text).with_context(|| format!("loading {}", path.display())), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(Collection::new()), + Err(err) => Err(err).with_context(|| format!("reading {}", path.display())), + } +} + +/// How the output is doing, for the stream panel. +#[derive(Default)] +pub struct Stats { + /// Canvas frames drawn. + pub rendered: AtomicU64, + /// Program frames copied back to the CPU for the encoder. + pub read_back: AtomicU64, + /// Microseconds the last readback waited on the GPU and converted pixels. + pub readback_us: AtomicU64, + /// Program frames not encoded because the encoder still held every buffer. + pub dropped: AtomicU64, + /// Program frames encoded. + pub encoded: AtomicU64, + /// Microseconds the last encode took. + pub encode_us: AtomicU64, + /// Whether any viewer is subscribed to the video. + pub watched: AtomicBool, + /// The encoder backend, once one opened. + pub encoder: Mutex>, + /// Why the output stopped, if it did. + pub error: Mutex>, +} + +impl Stats { + fn fail(&self, err: impl std::fmt::Display) { + tracing::error!(%err, "output stopped"); + *self.error.lock().unwrap() = Some(err.to_string()); + } +} + +/// Where the program goes. +pub struct Output { + /// The broadcast path on the relay. + pub path: String, + /// The relay connection, which reconnects on its own. + pub connection: moq_tokio::Connection, + publication: moq_net::origin::Publication, +} + +impl Drop for Output { + fn drop(&mut self) { + self.publication.close(); + } +} + +/// The running studio. +pub struct Engine { + pub gpu: Gpu, + pub state: Mutex, + pub stats: Stats, + pub bus: moq_audio::bus::Bus, + /// `None` when running without a relay. + pub output: Option, + /// Keeps broadcast sources' origin open when no connection holds it. + _origin: Option, +} + +/// The audio the program publishes. +const AUDIO_BITRATE: moq_net::bandwidth::Rate = moq_net::bandwidth::Rate::from_kbps(128); + +/// How often the audio thread reads the bus. Short enough that a read is never +/// close to the bus's buffering, long enough to cost nothing. +const AUDIO_PERIOD: Duration = Duration::from_millis(10); + +impl Engine { + /// Open the collection at `path` on `gpu` and start the program clock. + /// With `connect`, publish the program at `broadcast` on that relay. + /// + /// Call inside the tokio runtime that serves the relay connection. + pub async fn start( + gpu: Gpu, + path: PathBuf, + collection: Collection, + connect: Option<(moq_tokio::Client, url::Url)>, + broadcast: &str, + ) -> anyhow::Result> { + let bus = moq_audio::bus::Bus::new(Default::default())?; + let subscribe = moq_tokio::origin::spawn(); + let dir = path.parent().map(Path::to_path_buf).unwrap_or_default(); + let sources = Sources::new(source::Context::new(bus.clone(), subscribe.consume(), dir)); + let mut compositor = Compositor::new(&gpu.device, &gpu.queue, collection.canvas)?; + compositor.set_preview(collection.scenes.first().map(|scene| scene.name.clone())); + compositor.cut(); + let canvas = collection.canvas; + + let (output, origin, publish) = match connect { + Some((client, url)) => { + let (output, publish) = publish(client, url, subscribe, broadcast, canvas).await?; + (Some(output), None, Some(publish)) + } + None => (None, Some(subscribe), None), + }; + + let engine = Arc::new(Self { + state: Mutex::new(State { + path, + collection, + sources, + compositor, + stale: true, + unsaved: false, + }), + gpu, + stats: Stats::default(), + bus, + output, + _origin: origin, + }); + + let (video, audio, clock) = match publish { + Some(Publish { + video, + audio, + config, + clock, + }) => { + // One more than the readback pool, for a pause behind a full pool. + let (frames, pending) = mpsc::sync_channel(Readback::DEPTH + 1); + tokio::spawn(watch(video.demand(), engine.clone())); + let encoder = engine.clone(); + std::thread::Builder::new() + .name("encode".into()) + .spawn(move || encode(&encoder, pending, video, config))?; + (Some(frames), Some(audio), Some(clock)) + } + None => (None, None, None), + }; + + let start = Instant::now(); + // Where the program clock starts on the catalog's, so the wall time the + // catalog advertises for a frame is when the program drew it. + let epoch = match clock { + Some(clock) => Duration::from(clock.capture(start)?), + None => Duration::ZERO, + }; + let program = engine.clone(); + std::thread::Builder::new() + .name("program".into()) + .spawn(move || clocked(program.run_program(start, epoch, video)))?; + let mix = engine.clone(); + std::thread::Builder::new() + .name("audio".into()) + .spawn(move || clocked(mix.run_audio(start, epoch, audio)))?; + + Ok(engine) + } + + /// Draw a canvas frame per tick of the program clock, and hand the program + /// to the encoder while anyone watches, stamped `epoch` past `start`. + /// + /// Drawing outlives the encoder: a failed output stops reading the program + /// back, and the preview and program keep moving. + async fn run_program(&self, start: Instant, epoch: Duration, mut frames: Option>) { + let canvas = self.state.lock().unwrap().collection.canvas; + let mut readback = Readback::new(&self.gpu.device, &self.gpu.queue, canvas); + let period = Duration::from_secs(1) / canvas.framerate; + let mut clock = moq_clock::interval_at(start, period); + clock.set_missed_tick_behavior(moq_clock::MissedTickBehavior::Skip); + let mut watched = false; + + loop { + let tick = clock.tick().await; + let program = { + let mut guard = self.state.lock().unwrap(); + let state = &mut *guard; + if std::mem::take(&mut state.stale) { + for driver in state.sources.sync(&state.collection) { + spawn(driver); + } + } + if let Err(err) = state.compositor.render(&state.collection, &state.sources, tick) { + tracing::error!(%err, "cannot draw the canvas"); + continue; + } + state.compositor.program().clone() + }; + self.stats.rendered.fetch_add(1, Ordering::Relaxed); + + let Some(sender) = &frames else { continue }; + let was = std::mem::replace(&mut watched, self.stats.watched.load(Ordering::Relaxed)); + if !watched { + // Behind every frame already sent, so the break lands after them. + // A send fails only once the encoder stopped, having reported why. + if was && sender.send(Video::Pause).is_err() { + frames = None; + } + continue; + } + let Ok(timestamp) = moq_net::Timestamp::try_from(epoch + tick.saturating_duration_since(start)) else { + continue; + }; + match readback.copy(&program, timestamp) { + // The pool bounds the frames in flight, so this waits only behind + // a pause the encoder has not reached yet. + Some(pending) => { + if sender.send(Video::Frame(pending)).is_err() { + frames = None; + } + } + None => { + self.stats.dropped.fetch_add(1, Ordering::Relaxed); + } + } + } + } + + /// Read the program bus as the program clock advances, and publish it + /// stamped `epoch` past `start`. + /// + /// Runs without a relay too, so the meters move and sources never back up. + async fn run_audio(&self, start: Instant, epoch: Duration, mut producer: Option) { + let config = self.bus.config().clone(); + let rate = u128::from(config.sample_rate); + let channels = config.layout.channels() as usize; + let mut clock = moq_clock::interval_at(start, AUDIO_PERIOD); + clock.set_missed_tick_behavior(moq_clock::MissedTickBehavior::Skip); + let mut samples = Vec::new(); + let mut written: u128 = 0; + + loop { + let tick = clock.tick().await; + // Sample-exact against the program clock, so audio and video stay + // aligned however the ticks jitter. + let due = tick.saturating_duration_since(start).as_nanos() * rate / 1_000_000_000; + let frames = (due - written) as usize; + if frames == 0 { + continue; + } + samples.resize(frames * channels, 0.0); + self.bus.read(&mut samples); + let timestamp = moq_net::Timestamp::from_micros((epoch.as_micros() + written * 1_000_000 / rate) as u64); + written = due; + + let Some(publish) = producer.as_mut() else { continue }; + let Ok(timestamp) = timestamp else { continue }; + let data: Vec = samples.iter().flat_map(|sample| sample.to_le_bytes()).collect(); + if let Err(err) = publish.write(&moq_audio::Frame::new(Bytes::from(data), timestamp)) { + self.stats.fail(format!("audio: {err}")); + // Viewers see the audio fail rather than go away unfinished. + if let Some(producer) = producer.take() { + producer.abort(moq_net::StreamError::Internal.into()); + } + } + } + } +} + +/// The encoders behind an [`Output`], handed to their threads. +struct Publish { + video: moq_video::encode::Producer, + config: moq_video::encode::Config, + audio: moq_audio::encode::Producer, + /// The catalog's clock, which the codec producers write timestamps + /// against verbatim. + clock: moq_mux::Clock, +} + +/// Announce the program broadcast and connect to the relay, which also +/// carries `subscribe`'s broadcast sources. +async fn publish( + client: moq_tokio::Client, + url: url::Url, + subscribe: moq_net::origin::Producer, + path: &str, + canvas: moq_scene::Canvas, +) -> anyhow::Result<(Output, Publish)> { + let origin = moq_tokio::origin::spawn(); + let publication = origin.create_broadcast(path).context("cannot create the broadcast")?; + let mut broadcast = publication.broadcast().clone(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default())?; + let clock = catalog.clock(); + + let config = moq_video::encode::Config::new(canvas.width, canvas.height, canvas.framerate()); + // Opens a throwaway encoder now, so a machine that cannot encode the + // canvas fails at startup rather than when the first viewer arrives. + let rendition = config.probe().await.context("cannot encode the canvas")?; + let reserved = catalog + .reserve() + .with_timeline_hint(config.gop.duration(config.framerate)); + let video = moq_video::encode::Producer::new(broadcast.clone(), reserved, rendition)?; + + let bus = moq_audio::bus::Config::default(); + let mut input = moq_audio::encode::Input::new(bus.sample_rate, bus.layout); + input.format = moq_audio::Format::F32; + let mut options = moq_audio::encode::Options::default(); + options.track = Some("audio".into()); + options.config = moq_audio::encode::Config::from_input(moq_audio::encode::Codec::Opus, &input); + options.config.bitrate = Some(AUDIO_BITRATE); + let audio = moq_audio::encode::Producer::new(&mut broadcast, catalog, input, &options)?; + + publication + .announce(moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint())) + .context("cannot announce the broadcast")?; + tracing::info!(url = %moq_tokio::RedactedUrl::new(&url), broadcast = %path, "connecting"); + let connection = client.with_publisher(&origin).with_subscriber(subscribe).connect(url); + + let output = Output { + path: path.to_string(), + connection, + publication, + }; + Ok(( + output, + Publish { + video, + config, + audio, + clock, + }, + )) +} + +/// Track whether anyone watches the video, so the program is only read back +/// and encoded while someone does. +async fn watch(mut demand: moq_net::track::Demand, engine: Arc) { + loop { + match demand.wanted_changed().await { + Ok(wanted) => engine.stats.watched.store(wanted.is_some(), Ordering::Relaxed), + Err(err) => { + engine.stats.watched.store(false, Ordering::Relaxed); + engine.stats.fail(format!("video: {err}")); + return; + } + } + } +} + +/// What the program clock hands the encoder. +enum Video { + /// A program frame on its way back from the GPU. + Frame(Pending), + /// The last viewer left, so frames stop until one returns. + Pause, +} + +/// Read back and encode program frames until the program clock stops. +/// +/// A failure aborts the video track, so viewers see that it failed rather +/// than a track that went away unfinished. +fn encode( + engine: &Engine, + video: mpsc::Receiver