From be990350e32b9685f9f02c654f19e9fa0fad8a92 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 09:21:54 -0700 Subject: [PATCH 01/15] chore(quest): claim quest/m3/studio/app.md Quest-Claim: 9bb42c9a3f0daa9f From 9cbe6f0b26b24e901ad33139a4472ad4d576dadd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 09:52:06 -0700 Subject: [PATCH 02/15] feat(moq-scene): scene collection, sources, and a wgpu compositor Adds the UI-free half of moq-studio: a versioned TOML scene collection, live sources (capture, image, color, subscribed broadcast), a compositor with preview, program, cut, and fade, and a pooled program readback. Adds moq-audio's program bus with per-source strips, and moves Level to the crate root so the bus and capture share it. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 771 ++++++++++++++++++++++++- Cargo.toml | 3 + rs/justfile | 4 +- rs/moq-audio/Cargo.toml | 7 +- rs/moq-audio/src/bus.rs | 658 ++++++++++++++++++++++ rs/moq-audio/src/encode/capture.rs | 43 +- rs/moq-audio/src/encode/mod.rs | 2 +- rs/moq-audio/src/level.rs | 54 ++ rs/moq-audio/src/lib.rs | 7 + rs/moq-scene/Cargo.toml | 51 ++ rs/moq-scene/src/collection.rs | 577 +++++++++++++++++++ rs/moq-scene/src/compositor.rs | 866 +++++++++++++++++++++++++++++ rs/moq-scene/src/lib.rs | 85 +++ rs/moq-scene/src/readback.rs | 190 +++++++ rs/moq-scene/src/source.rs | 605 ++++++++++++++++++++ rs/moq-studio/Cargo.toml | 62 +++ rs/moq-studio/src/main.rs | 1 + 17 files changed, 3931 insertions(+), 55 deletions(-) create mode 100644 rs/moq-audio/src/bus.rs create mode 100644 rs/moq-audio/src/level.rs create mode 100644 rs/moq-scene/Cargo.toml create mode 100644 rs/moq-scene/src/collection.rs create mode 100644 rs/moq-scene/src/compositor.rs create mode 100644 rs/moq-scene/src/lib.rs create mode 100644 rs/moq-scene/src/readback.rs create mode 100644 rs/moq-scene/src/source.rs create mode 100644 rs/moq-studio/Cargo.toml create mode 100644 rs/moq-studio/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 3bb883965a7..6180dbf54a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,6 +18,15 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "366ffbaa4442f4684d91e2cd7c5ea7c4ed8add41959a31447066e279e432b618" +[[package]] +name = "accesskit" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3b7f7f85a7e5f68090000ed7622545829afd484d210358702ae4cb97dd0c320" +dependencies = [ + "uuid", +] + [[package]] name = "addr2line" version = "0.25.1" @@ -248,6 +257,26 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arboard" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeb9553e62312c8756a5979254fc6d7da8bbeb5986a1e17037fab932f7037917" +dependencies = [ + "clipboard-win", + "image", + "log", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-foundation 0.3.2", + "parking_lot", + "percent-encoding", + "windows-sys 0.61.2", + "x11rb 0.13.2", +] + [[package]] name = "arc-swap" version = "1.9.2" @@ -1058,6 +1087,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.12.1" @@ -1096,18 +1131,42 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "calloop" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b32313e937c8721c9d17ee9bf60699a6402b4b7c2cb6ac1bee0ffcaa4b4e5e" +dependencies = [ + "bitflags 2.13.2", + "polling", + "rustix 1.1.5", + "tracing", +] + [[package]] name = "calloop-wayland-source" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a66a987056935f7efce4ab5668920b5d0dac4a7c99991a67395f13702ddd20" dependencies = [ - "calloop", + "calloop 0.13.0", "rustix 0.38.44", "wayland-backend", "wayland-client", ] +[[package]] +name = "calloop-wayland-source" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138efcf0940a02ebf0cc8d1eff41a1682a46b431630f4c52450d6265876021fa" +dependencies = [ + "calloop 0.14.5", + "rustix 1.1.5", + "wayland-backend", + "wayland-client", +] + [[package]] name = "camino" version = "1.2.6" @@ -1239,6 +1298,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "cgl" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ced0551234e87afee12411d535648dd89d2e7f34c78b753395567aff3d447ff" +dependencies = [ + "libc", +] + [[package]] name = "chacha20" version = "0.10.2" @@ -1352,6 +1420,15 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" +[[package]] +name = "clipboard-win" +version = "5.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052af7878466feccda12ba7a6a67d019d712f296809559b47c12c8cc196d3a9a" +dependencies = [ + "error-code", +] + [[package]] name = "cmake" version = "0.1.58" @@ -1387,6 +1464,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "color" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ec7c5eb7a16992b1904d76c517d170ab353b0e0b3d5a0c81a8a0cd1037893cf" +dependencies = [ + "bytemuck", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -2253,6 +2339,17 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "duplicate" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e92f10a49176cbffacaedabfaa11d51db1ea0f80a83c26e1873b43cd1742c24" +dependencies = [ + "heck", + "proc-macro2", + "proc-macro2-diagnostics", +] + [[package]] name = "dyn-clone" version = "1.0.20" @@ -2304,6 +2401,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ecolor" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0656a7530b9e2841ee7a5f983965bdd9840c32f4012f759cfd44b912cced6b34" +dependencies = [ + "bytemuck", + "emath", +] + [[package]] name = "ed25519" version = "3.0.0" @@ -2331,6 +2438,127 @@ dependencies = [ "zeroize", ] +[[package]] +name = "eframe" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ad41fac6e3149abb4866e93df37dbd33baa95db525c3368c944584e95a5dc2" +dependencies = [ + "ahash", + "bytemuck", + "document-features", + "egui", + "egui-wgpu", + "egui-winit", + "egui_glow", + "glutin", + "glutin-winit", + "image", + "js-sys", + "log", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", + "objc2-foundation 0.3.2", + "parking_lot", + "percent-encoding", + "pollster", + "profiling", + "raw-window-handle", + "static_assertions", + "wasm-bindgen", + "web-sys", + "web-time", + "wgpu", + "windows-sys 0.61.2", + "winit", +] + +[[package]] +name = "egui" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc938cc27cd911415e1e4d151bc56ff9df063d8db081374e87aeb805ea74b2ba" +dependencies = [ + "accesskit", + "ahash", + "bitflags 2.13.2", + "emath", + "epaint", + "itertools 0.15.0", + "log", + "nohash-hasher", + "profiling", + "smallvec", + "unicode-segmentation", + "web-sys", +] + +[[package]] +name = "egui-wgpu" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cef1f6d1bba40b09a4d0d62cb822b81030922cfbf65c5373398c3a3caafc5bd" +dependencies = [ + "ahash", + "bytemuck", + "document-features", + "egui", + "epaint", + "log", + "profiling", + "thiserror 2.0.21", + "type-map", + "web-time", + "wgpu", + "winit", +] + +[[package]] +name = "egui-winit" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98466000559d66ba4db786a30448b01483946add4188e7f10f522a27b4fb89cc" +dependencies = [ + "arboard", + "bytemuck", + "egui", + "log", + "objc2 0.6.5", + "objc2-foundation 0.3.2", + "objc2-ui-kit 0.3.2", + "profiling", + "raw-window-handle", + "smithay-clipboard", + "web-time", + "winit", +] + +[[package]] +name = "egui_dock" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d9f668eb59ff0ab130c8bce71a513135e1d3017fb3103fb8758e9eea99d759f" +dependencies = [ + "duplicate", + "egui", + "paste", + "thiserror 2.0.21", +] + +[[package]] +name = "egui_glow" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "606ed4167daf341e5753d703285c4c1d562324b2c07aa8f405cd7d066fe58f3b" +dependencies = [ + "bytemuck", + "egui", + "glow", + "log", + "profiling", + "winit", +] + [[package]] name = "either" version = "1.18.0" @@ -2358,6 +2586,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "emath" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c8d4141cf0f60fb2f7aefaf352d3ffa5171ef0ab09731c64391801e95e48f1" +dependencies = [ + "bytemuck", +] + [[package]] name = "embedded-io" version = "0.4.0" @@ -2419,6 +2656,37 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "epaint" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0ec0308dc23130fc1a7d4623a8534c2ed069c6dc84e258ee4f3de99615aa1a3" +dependencies = [ + "ahash", + "bytemuck", + "ecolor", + "emath", + "epaint_default_fonts", + "font-types", + "harfrust", + "log", + "nohash-hasher", + "parking_lot", + "profiling", + "self_cell", + "skrifa", + "smallvec", + "unicode-general-category", + "unicode-segmentation", + "vello_cpu", +] + +[[package]] +name = "epaint_default_fonts" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773fa9c96dd0dbef887e39d0ed6177f141cce4d68e6041df77570aa3702dfa13" + [[package]] name = "equivalent" version = "1.0.2" @@ -2435,6 +2703,21 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "error-code" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5343afd4a8365a643ac588dab4cf234a190c7f6c88c9f6dd6ffe00837661b7" + +[[package]] +name = "euclid" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06" +dependencies = [ + "num-traits", +] + [[package]] name = "event-listener" version = "5.4.2" @@ -2494,6 +2777,27 @@ dependencies = [ "getrandom 0.4.3", ] +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "fearless_simd" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" + [[package]] name = "ff" version = "0.14.0" @@ -2569,6 +2873,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "font-types" +version = "0.12.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "835ae9d781464a79026cde387f9cbc650c9a4909d7cf7e658b127e3c35d2f35d" +dependencies = [ + "bytemuck", +] + [[package]] name = "foreign-types" version = "0.5.0" @@ -2918,6 +3231,22 @@ dependencies = [ "system-deps 9.0.0", ] +[[package]] +name = "glifo" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed4a1bb24121291d27230c1b1b44e07d6a9b28cefdb32fe1581dfb84e14f940a" +dependencies = [ + "bytemuck", + "foldhash", + "hashbrown 0.17.1", + "log", + "peniko", + "skrifa", + "smallvec", + "vello_common", +] + [[package]] name = "glob" version = "0.3.4" @@ -2948,6 +3277,63 @@ dependencies = [ "web-sys", ] +[[package]] +name = "glutin" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12124de845cacfebedff80e877bb37b5b75c34c5a4c89e47e1cdd67fb6041325" +dependencies = [ + "bitflags 2.13.2", + "cfg_aliases", + "cgl", + "dispatch2", + "glutin_egl_sys", + "glutin_glx_sys", + "glutin_wgl_sys", + "libloading 0.8.9", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", + "objc2-core-foundation", + "objc2-foundation 0.3.2", + "once_cell", + "raw-window-handle", + "wayland-sys", + "windows-sys 0.52.0", + "x11-dl", +] + +[[package]] +name = "glutin-winit" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85edca7075f8fc728f28cb8fbb111a96c3b89e930574369e3e9c27eb75d3788f" +dependencies = [ + "cfg_aliases", + "glutin", + "raw-window-handle", + "winit", +] + +[[package]] +name = "glutin_egl_sys" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c4680ba6195f424febdc3ba46e7a42a0e58743f2edb115297b86d7f8ecc02d2" +dependencies = [ + "gl_generator", + "windows-sys 0.52.0", +] + +[[package]] +name = "glutin_glx_sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7bb2938045a88b612499fbcba375a77198e01306f52272e692f8c1f3751185" +dependencies = [ + "gl_generator", + "x11-dl", +] + [[package]] name = "glutin_wgl_sys" version = "0.6.1" @@ -3108,6 +3494,15 @@ dependencies = [ "system-deps 9.0.0", ] +[[package]] +name = "guillotiere" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b17e70c989c36bad147b27a58d148c0741c51448aa5653436547323e524d0ab" +dependencies = [ + "euclid", +] + [[package]] name = "h2" version = "0.4.20" @@ -3195,6 +3590,18 @@ dependencies = [ "url", ] +[[package]] +name = "harfrust" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c03d949a14aa089bbb282f7dd76a498a7f684428e4257202efc119ec010376f9" +dependencies = [ + "bitflags 2.13.2", + "bytemuck", + "read-fonts", + "smallvec", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -3666,6 +4073,22 @@ dependencies = [ "xmltree", ] +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "moxcms", + "num-traits", + "png", + "tiff", + "zune-core", + "zune-jpeg", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -4219,6 +4642,18 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "kurbo" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b60dfc32f652b926df6192e55525b16d186c69d47876c3ead4da5cc9f8450e2" +dependencies = [ + "arrayvec", + "euclid", + "polycool", + "smallvec", +] + [[package]] name = "lazy_static" version = "1.5.1" @@ -4323,6 +4758,12 @@ dependencies = [ "semver-parser", ] +[[package]] +name = "linebender_resource_handle" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a5ff6bcca6c4867b1c4fd4ef63e4db7436ef363e0ad7531d1558856bae64f4" + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -4516,6 +4957,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -5292,6 +5734,30 @@ dependencies = [ "url", ] +[[package]] +name = "moq-scene" +version = "0.1.0" +dependencies = [ + "bytemuck", + "hang", + "humantime-serde", + "image", + "kio 0.6.2", + "moq-audio", + "moq-clock", + "moq-mux", + "moq-net", + "moq-video", + "pollster", + "serde", + "serde_with", + "tempfile", + "thiserror 2.0.21", + "tokio", + "toml 1.1.6+spec-1.1.0", + "tracing", +] + [[package]] name = "moq-shaper" version = "0.0.1" @@ -5358,6 +5824,30 @@ dependencies = [ "web-async", ] +[[package]] +name = "moq-studio" +version = "0.1.0" +dependencies = [ + "anyhow", + "eframe", + "egui", + "egui_dock", + "hang", + "moq-audio", + "moq-clock", + "moq-mux", + "moq-net", + "moq-relay", + "moq-scene", + "moq-tokio", + "moq-video", + "pollster", + "tempfile", + "tokio", + "tracing", + "usage-rs", +] + [[package]] name = "moq-token" version = "0.7.4" @@ -5596,6 +6086,16 @@ dependencies = [ "web-transport-wasm", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "mp4-atom" version = "0.16.2" @@ -5999,6 +6499,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + [[package]] name = "nom" version = "7.1.3" @@ -6328,6 +6834,19 @@ dependencies = [ "objc2-quartz-core 0.2.2", ] +[[package]] +name = "objc2-app-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-audio-toolbox" version = "0.3.2" @@ -6615,7 +7134,7 @@ checksum = "a1a1ae721c5e35be65f01a03b6d2ac13a54cb4fa70d8a5da293d7b0020261398" dependencies = [ "block2 0.5.1", "objc2 0.5.2", - "objc2-app-kit", + "objc2-app-kit 0.2.2", "objc2-foundation 0.2.2", ] @@ -6769,6 +7288,18 @@ dependencies = [ "objc2-user-notifications", ] +[[package]] +name = "objc2-ui-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87d638e33c06f577498cbcc50491496a3ed4246998a7fbba7ccb98b1e7eab22" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", + "objc2-core-foundation", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-uniform-type-identifiers" version = "0.2.2" @@ -7135,6 +7666,19 @@ dependencies = [ "base64ct", ] +[[package]] +name = "peniko" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "839c8299360d2e998bdb106dc0a6cd71dcc5f4df51df1b620361bf50e283cca6" +dependencies = [ + "bytemuck", + "color", + "kurbo", + "linebender_resource_handle", + "smallvec", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -7332,6 +7876,19 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags 2.13.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.9", +] + [[package]] name = "polling" version = "3.11.0" @@ -7352,6 +7909,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc6355899e1c9462875b6757c79f3caa011a1fdae12bbb1a2e72dd1f234f8336" +[[package]] +name = "polycool" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50596ddc09eb5ad5f75cacd40209568e66df71baf86e1499a0e99c4cff12a5a6" +dependencies = [ + "arrayvec", +] + [[package]] name = "polyval" version = "0.6.2" @@ -7526,6 +8092,18 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "version_check", +] + [[package]] name = "procfs" version = "0.18.0" @@ -7588,6 +8166,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + [[package]] name = "qlog" version = "0.18.1" @@ -7640,6 +8224,12 @@ dependencies = [ "web-transport-trait", ] +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quick-xml" version = "0.41.0" @@ -7922,6 +8512,17 @@ dependencies = [ "yasna 0.6.0", ] +[[package]] +name = "read-fonts" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046a7d674daf459825b32f5062056d6882db0d2f5a479fbd76ccfc870ac18709" +dependencies = [ + "bytemuck", + "font-types", + "once_cell", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -8627,7 +9228,7 @@ dependencies = [ "ab_glyph", "log", "memmap2", - "smithay-client-toolkit", + "smithay-client-toolkit 0.19.2", "tiny-skia", ] @@ -8775,6 +9376,12 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" +[[package]] +name = "self_cell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab42ca02749e120097e328d91d415325bdf43b1c72c4c8badf37375fe40a813" + [[package]] name = "semver" version = "1.0.28" @@ -9122,6 +9729,16 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" +[[package]] +name = "skrifa" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819ab7d62b1d3e72d9d9dea5650bac30424f9111364bb94928dbf5ecad1baa68" +dependencies = [ + "bytemuck", + "read-fonts", +] + [[package]] name = "slab" version = "0.4.12" @@ -9156,8 +9773,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3457dea1f0eb631b4034d61d4d8c32074caa6cd1ab2d59f2327bd8461e2c0016" dependencies = [ "bitflags 2.13.2", - "calloop", - "calloop-wayland-source", + "calloop 0.13.0", + "calloop-wayland-source 0.3.0", "cursor-icon", "libc", "log", @@ -9174,6 +9791,44 @@ dependencies = [ "xkeysym", ] +[[package]] +name = "smithay-client-toolkit" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0512da38f5e2b31201a93524adb8d3136276fa4fe4aafab4e1f727a82b534cc0" +dependencies = [ + "bitflags 2.13.2", + "calloop 0.14.5", + "calloop-wayland-source 0.4.1", + "cursor-icon", + "libc", + "log", + "memmap2", + "rustix 1.1.5", + "thiserror 2.0.21", + "wayland-backend", + "wayland-client", + "wayland-csd-frame", + "wayland-cursor", + "wayland-protocols", + "wayland-protocols-experimental", + "wayland-protocols-misc", + "wayland-protocols-wlr", + "wayland-scanner", + "xkeysym", +] + +[[package]] +name = "smithay-clipboard" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71704c03f739f7745053bde45fa203a46c58d25bc5c4efba1d9a60e9dba81226" +dependencies = [ + "libc", + "smithay-client-toolkit 0.20.0", + "wayland-backend", +] + [[package]] name = "smol_str" version = "0.2.2" @@ -9822,6 +10477,20 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + [[package]] name = "tikv-jemalloc-ctl" version = "0.7.0" @@ -10407,6 +11076,15 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "type-map" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb30dbbd9036155e74adad6812e9898d03ec374946234fbcebd5dfc7b9187b90" +dependencies = [ + "rustc-hash 2.1.3", +] + [[package]] name = "typenum" version = "1.20.1" @@ -10436,6 +11114,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + [[package]] name = "unicode-ident" version = "1.0.26" @@ -10825,6 +11509,33 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vello_common" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2e9aed918117e8152c9eddfd8362d73c465c23f26c44786aa331707b8a64fa2" +dependencies = [ + "bytemuck", + "fearless_simd", + "guillotiere", + "log", + "peniko", + "smallvec", + "thiserror 2.0.21", +] + +[[package]] +name = "vello_cpu" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac7349e1f55f6b801c7c277958df4ea53e7f20f21e8014910ad888b2ecda93ea" +dependencies = [ + "bytemuck", + "glifo", + "hashbrown 0.17.1", + "vello_common", +] + [[package]] name = "version-compare" version = "0.2.1" @@ -11090,6 +11801,32 @@ dependencies = [ "wayland-scanner", ] +[[package]] +name = "wayland-protocols-experimental" +version = "20250721.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40a1f863128dcaaec790d7b4b396cc9b9a7a079e878e18c47e6c2d2c5a8dcbb1" +dependencies = [ + "bitflags 2.13.2", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-misc" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e9567599ef23e09b8dad6e429e5738d4509dfc46b3b21f32841a304d16b29c8" +dependencies = [ + "bitflags 2.13.2", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + [[package]] name = "wayland-protocols-plasma" version = "0.3.12" @@ -11282,6 +12019,12 @@ dependencies = [ "nom 7.1.3", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "wgpu" version = "30.0.1" @@ -11340,6 +12083,7 @@ dependencies = [ "thiserror 2.0.21", "wgpu-core-deps-apple", "wgpu-core-deps-emscripten", + "wgpu-core-deps-wasm", "wgpu-core-deps-windows-linux-android", "wgpu-hal", "wgpu-naga-bridge", @@ -11364,6 +12108,15 @@ dependencies = [ "wgpu-hal", ] +[[package]] +name = "wgpu-core-deps-wasm" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d260ca2ca8adc654f513a869bf3cac0b47f5b8d3f87803d6942015f8de597aae" +dependencies = [ + "wgpu-hal", +] + [[package]] name = "wgpu-core-deps-windows-linux-android" version = "30.0.1" @@ -11866,7 +12619,7 @@ dependencies = [ "bitflags 2.13.2", "block2 0.5.1", "bytemuck", - "calloop", + "calloop 0.13.0", "cfg_aliases", "concurrent-queue", "core-foundation 0.9.4", @@ -11878,9 +12631,9 @@ dependencies = [ "memmap2", "ndk", "objc2 0.5.2", - "objc2-app-kit", + "objc2-app-kit 0.2.2", "objc2-foundation 0.2.2", - "objc2-ui-kit", + "objc2-ui-kit 0.2.2", "orbclient", "percent-encoding", "pin-project", @@ -11888,7 +12641,7 @@ dependencies = [ "redox_syscall 0.4.1", "rustix 0.38.44", "sctk-adwaita", - "smithay-client-toolkit", + "smithay-client-toolkit 0.19.2", "smol_str", "tracing", "unicode-segmentation", diff --git a/Cargo.toml b/Cargo.toml index 3cd19017fc8..2ac49ce1b72 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,10 +35,12 @@ default-members = [ "rs/moq-rtc", "rs/moq-rtmp", "rs/moq-rtsp", + "rs/moq-scene", "rs/moq-shaper", "rs/moq-sock", "rs/moq-srt", "rs/moq-stats", + "rs/moq-studio", "rs/moq-tokio", "rs/moq-transcode", "rs/moq-uring", @@ -146,6 +148,7 @@ moq-relay = { version = "0.17.2", path = "rs/moq-relay", default-features = fals moq-rtc = { version = "0.3.12", path = "rs/moq-rtc" } moq-rtmp = { version = "0.3.11", path = "rs/moq-rtmp" } moq-rtsp = { version = "0.1.0", path = "rs/moq-rtsp" } +moq-scene = { version = "0.1.0", path = "rs/moq-scene" } moq-sock = { version = "0.2.0", path = "rs/moq-sock" } moq-srt = { version = "0.3.11", path = "rs/moq-srt" } moq-stats = { version = "0.2.12", path = "rs/moq-stats" } diff --git a/rs/justfile b/rs/justfile index 3fc3a8dca88..0538bd4e6e9 100644 --- a/rs/justfile +++ b/rs/justfile @@ -346,7 +346,7 @@ cli-export *args: quic-stress: just rs test -p moq-tokio --lib --run-ignored only -E 'test(/^quic::driver::tests::stress_/)' -# Render CPU frames through wgpu on Mesa's software Vulkan driver, so no GPU is needed. +# Render CPU frames and composite scenes through wgpu on Mesa's software Vulkan driver, so no GPU is needed. render-sw: #!/usr/bin/env bash set -euo pipefail @@ -356,6 +356,8 @@ render-sw: LD_LIBRARY_PATH="$loader/lib" VK_DRIVER_FILES="${icd[0]}" \ just rs test -p moq-video --features render --lib --run-ignored only \ -E 'test(/^render::renderer::tests::(the_declared_color_space_beats_the_size_guess|cpu_frames_survive_the_round_trip|config_size_overrides_the_frame_size)$/)' + LD_LIBRARY_PATH="$loader/lib" VK_DRIVER_FILES="${icd[0]}" \ + just rs test -p moq-scene --lib --run-ignored only -E 'test(/^compositor::tests::/)' # Run the ignored mDNS tests, which multicast on the host's network interfaces. multicast: diff --git a/rs/moq-audio/Cargo.toml b/rs/moq-audio/Cargo.toml index 183de7b7d02..98e13094da7 100644 --- a/rs/moq-audio/Cargo.toml +++ b/rs/moq-audio/Cargo.toml @@ -14,7 +14,7 @@ categories = ["multimedia", "multimedia::audio", "multimedia::encoding"] [features] # On by default: what costs the build nothing beyond cargo, which here is `aac` -# (pure Rust). Off by default: the device features, because on Linux cpal links +# and `bus` (pure Rust). Off by default: the device features, because on Linux cpal links # libasound at build time, the one system build dependency in this crate. macOS # and Windows would pay nothing, but a default is one setting for every host. # The features exist so a consumer can pick what it wants, not to hide code from @@ -23,13 +23,16 @@ categories = ["multimedia", "multimedia::audio", "multimedia::encoding"] # `just rs macos`. Workspace consumers opt in at the root manifest # (`default-features = false`) per crate, which keeps cpal out of the language # bindings. -default = ["aac"] +default = ["aac", "bus"] # AAC-LC decode via symphonia. Pure Rust, so it costs no toolchain: the trade # this crate already refused for Opus. Only a subscriber to an ingest-sourced # broadcast needs it (RTMP, SRT/TS, fmp4, and gstreamer all publish AAC, while # this crate encodes in software is Opus or PCM), so a publish-only build can # drop it and the two crates it pulls. aac = ["dep:symphonia-codec-aac", "dep:symphonia-core"] +# The program mixing bus. Pure Rust: it resamples and de-jitters each source +# through the same `fixed-resample` ring the playback mixer uses. +bus = ["dep:fixed-resample"] # Device capture. Microphones go through cpal (pure-Rust: CoreAudio / WASAPI / # ALSA); macOS system audio goes through ScreenCaptureKit, which is where Apple # puts it. On Linux cpal links libasound, the one system build dependency in diff --git a/rs/moq-audio/src/bus.rs b/rs/moq-audio/src/bus.rs new file mode 100644 index 00000000000..61827b0f883 --- /dev/null +++ b/rs/moq-audio/src/bus.rs @@ -0,0 +1,658 @@ +//! Mix PCM sources into one program feed, read on the caller's clock. +//! +//! A [`Bus`] sums its [`Channel`]s, each with its own [`Strip`] of gain, mute, +//! and level meter, into one interleaved `f32` stream at the bus's +//! [`Config`]. It is what a production switcher's program audio needs: +//! microphones, system audio, and decoded broadcasts mixed into the one feed +//! that gets encoded and published. +//! +//! Unlike `playback::Engine`, which a device callback drives, nothing here owns +//! a clock. Whoever owns the program clock calls [`Bus::read`] with as many +//! frames as that clock has advanced, so the mix stays in step with it exactly. +//! Each channel is resampled from its own rate to the bus's, and a source that +//! runs slightly fast or slow against that clock is corrected by the same +//! ring that absorbs its jitter, so it never drifts out of the mix. +//! +//! ``` +//! # fn example() -> Result<(), moq_audio::Error> { +//! use moq_audio::{bus, encode}; +//! +//! let bus = bus::Bus::new(bus::Config::default())?; +//! let strip = bus::Strip::default(); +//! let mut mic = bus.channel(encode::Input::new(44_100, moq_audio::Layout::Mono), strip.clone())?; +//! strip.set_gain(0.5); +//! +//! // On the program clock: 10 ms of stereo at 48 kHz. +//! let mut program = vec![0.0f32; 480 * 2]; +//! bus.read(&mut program); +//! # Ok(()) +//! # } +//! ``` +//! +//! Enabled by the default-on `bus` feature; [`Bus::capture`] also needs `capture`. + +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, Weak}; +use std::time::Duration; + +use fixed_resample::{ResamplerConfig, ResamplingChannelConfig, ResamplingCons, ResamplingProd, resampling_channel}; + +use crate::encode::Input; +use crate::resample::Remix; +use crate::{Error, Layout, Level}; + +/// Audio held between a [`Channel::write`] and the [`Bus::read`] that mixes it. +/// +/// The jitter a late write absorbs without a dropout, and the delay every +/// source carries into the mix. Matches the playback sink's default, which +/// rides out a stalled network read without being audible as delay. +const LATENCY: Duration = Duration::from_millis(50); + +/// Room above [`LATENCY`] before writes are dropped. A writer that runs ahead +/// (a decoder catching up after a stall) parks its samples here rather than +/// losing them, and the overflow correction then trims back to [`LATENCY`]. +const HEADROOM: Duration = Duration::from_millis(150); + +/// What the program feed is mixed to. +/// +/// `#[non_exhaustive]`: construct via [`Config::default`] and set fields, so new +/// options can be added without breaking callers. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct Config { + /// Samples per second per channel of [`Bus::read`]'s output. + pub sample_rate: u32, + /// Speaker layout of [`Bus::read`]'s output. Must name speaker positions, + /// since every channel is remixed to it. + pub layout: Layout, +} + +impl Default for Config { + fn default() -> Self { + Self { + sample_rate: 48_000, + layout: Layout::Stereo, + } + } +} + +/// One program mix that any number of [`Channel`]s feed. +/// +/// Cheap to clone; every clone reads and feeds the same mix. +#[derive(Clone)] +pub struct Bus { + inner: Arc, +} + +struct Inner { + config: Config, + mixer: Mutex, + next: AtomicU64, + level: Meter, +} + +struct Mixer { + entries: Vec, + /// One channel's share of the block being mixed, reused across reads. + scratch: Vec, +} + +/// A channel as the mixer sees it. +struct Entry { + id: u64, + cons: ResamplingCons, + strip: Strip, + /// Gain actually applied, ramping toward the strip's target across a block. + applied: f32, +} + +impl Bus { + /// An empty mix producing PCM in `config`'s format. + pub fn new(config: Config) -> Result { + if config.sample_rate == 0 { + return Err(Error::Unsupported("bus sample rate must be > 0".into())); + } + if config.layout.speakers().is_none() { + return Err(Error::Unsupported(format!( + "a bus needs speaker positions to remix into (got {:?})", + config.layout + ))); + } + + Ok(Self { + inner: Arc::new(Inner { + config, + mixer: Mutex::new(Mixer { + entries: Vec::new(), + scratch: Vec::new(), + }), + next: AtomicU64::new(0), + level: Meter::default(), + }), + }) + } + + /// The format [`read`](Self::read) produces. + pub fn config(&self) -> &Config { + &self.inner.config + } + + /// Add a source to the mix, taking PCM in the layout `input` describes and + /// controlled by `strip`. + /// + /// The strip is the source's identity to a UI: it outlives the channel, so a + /// source that reopens with another format (a new decoder, a replugged + /// device) keeps its gain, mute, and meter by passing the same strip again. + /// The channel leaves the mix when dropped. + pub fn channel(&self, input: Input, strip: Strip) -> Result { + if input.sample_rate == 0 { + return Err(Error::Unsupported("channel sample rate must be > 0".into())); + } + let output = self.inner.config.layout; + let remix = if input.layout == output { + None + } else { + Some(Remix::new(input.layout, output)?) + }; + + let (prod, cons) = resampling_channel::( + output.channels() as usize, + input.sample_rate, + self.inner.config.sample_rate, + // Only ever pushed interleaved, which skips the planar staging buffer. + true, + ResamplingChannelConfig { + latency_seconds: LATENCY.as_secs_f64(), + capacity_seconds: (LATENCY + HEADROOM).as_secs_f64(), + // Correct drift against the program clock by trimming or padding + // back to the target depth, not by letting the ring run dry or full. + underflow_autocorrect_percent_threshold: Some(25.0), + overflow_autocorrect_percent_threshold: Some(75.0), + resampler_config: ResamplerConfig::default(), + ..Default::default() + }, + ); + + let id = self.inner.next.fetch_add(1, Ordering::Relaxed); + self.inner.mixer.lock().unwrap().entries.push(Entry { + id, + cons, + strip: strip.clone(), + applied: strip.target(), + }); + + Ok(Channel { + id, + prod, + remix, + input, + strip, + bus: Arc::downgrade(&self.inner), + }) + } + + /// Mix the next `output.len() / channels` frames into `output`, interleaved + /// in the bus layout. + /// + /// Call it as the program clock advances, with exactly that much time's worth + /// of frames: the bus has no clock of its own, so this is what paces every + /// channel. A channel with nothing buffered contributes silence. Any samples + /// past the last whole frame are zeroed. + pub fn read(&self, output: &mut [f32]) { + output.fill(0.0); + let channels = self.inner.config.layout.channels() as usize; + let whole = output.len() - output.len() % channels; + let output = &mut output[..whole]; + if output.is_empty() { + return; + } + + let mut mixer = self.inner.mixer.lock().unwrap(); + let Mixer { entries, scratch } = &mut *mixer; + scratch.resize(output.len(), 0.0); + let frames = output.len() / channels; + + for entry in entries.iter_mut() { + entry.cons.read_interleaved(scratch, false); + + // Ramp across the block rather than stepping, so a fader move or a + // mute never clicks. + let target = entry.strip.target(); + let start = entry.applied; + let step = (target - start) / frames as f32; + for (index, frame) in scratch.chunks_exact_mut(channels).enumerate() { + let gain = start + step * (index + 1) as f32; + for sample in frame { + *sample *= gain; + } + } + entry.applied = target; + + entry.strip.shared.level.record(Level::measure(scratch)); + for (out, sample) in output.iter_mut().zip(scratch.iter()) { + *out += *sample; + } + } + drop(mixer); + + self.inner.level.record(Level::measure(output)); + } + + /// The loudest the mix has been since the previous call. + pub fn level(&self) -> Level { + self.inner.level.take() + } + + /// Capture an input device into the mix, described by `config` and + /// controlled by `strip`. + /// + /// Nothing opens until [`Capture::run`] is awaited, and the device's format + /// is whatever it offers: the channel adapts to it each time it opens, under + /// the same strip. + /// + /// Requires the `capture` feature. + #[cfg(feature = "capture")] + pub fn capture(&self, config: crate::capture::Config, strip: Strip) -> Capture { + Capture { + bus: self.clone(), + config, + strip, + } + } +} + +/// One source's PCM on its way into a [`Bus`]. +/// +/// Dropping it takes the source out of the mix. +pub struct Channel { + id: u64, + prod: ResamplingProd, + remix: Option, + input: Input, + strip: Strip, + bus: Weak, +} + +impl Channel { + /// Push PCM in the format, rate, and layout this channel was created with. + /// + /// Never blocks. Audio that arrives before the bus is first read, or that + /// overruns the channel's buffer, is dropped to keep the mix live. + pub fn write(&mut self, data: &[u8]) -> Result<(), Error> { + let pcm = self + .input + .format + .as_interleaved_f32(data, self.input.layout.channels())?; + self.push(&pcm); + Ok(()) + } + + fn push(&mut self, pcm: &[f32]) { + match &self.remix { + Some(remix) => { + let _ = self.prod.push_interleaved(&remix.process(pcm)); + } + None => { + let _ = self.prod.push_interleaved(pcm); + } + } + } + + /// This source's gain, mute, and meter. + pub fn strip(&self) -> &Strip { + &self.strip + } +} + +impl Drop for Channel { + fn drop(&mut self) { + if let Some(bus) = self.bus.upgrade() { + bus.mixer.lock().unwrap().entries.retain(|entry| entry.id != self.id); + } + } +} + +/// The gain, mute, and meter of one source in a [`Bus`]. +/// +/// Cheap to clone and `Send + Sync`: clones control the same source, so a UI +/// can hold one while a task owns the [`Channel`]. Changes apply from the next +/// [`Bus::read`], ramped across it. +#[derive(Clone, Debug, Default)] +pub struct Strip { + shared: Arc, +} + +#[derive(Debug)] +struct StripState { + /// Linear gain, as [`f32::to_bits`] since there is no atomic float. + gain: AtomicU32, + muted: AtomicBool, + level: Meter, +} + +impl Default for StripState { + fn default() -> Self { + Self { + gain: AtomicU32::new(1.0f32.to_bits()), + muted: AtomicBool::new(false), + level: Meter::default(), + } + } +} + +impl Strip { + /// Set the linear gain: `1.0` is unity, `0.0` silent, above `1.0` boosts. + /// + /// A negative or non-finite gain is refused and leaves the gain unchanged. + pub fn set_gain(&self, gain: f32) { + if gain.is_finite() && gain >= 0.0 { + self.shared.gain.store(gain.to_bits(), Ordering::Relaxed); + } + } + + /// The linear gain, independent of [`muted`](Self::muted). + pub fn gain(&self) -> f32 { + f32::from_bits(self.shared.gain.load(Ordering::Relaxed)) + } + + /// Silence this source without forgetting its gain. + pub fn set_muted(&self, muted: bool) { + self.shared.muted.store(muted, Ordering::Relaxed); + } + + /// Whether this source is muted. + pub fn muted(&self) -> bool { + self.shared.muted.load(Ordering::Relaxed) + } + + /// The loudest this source contributed to the mix since the previous call, + /// after its gain and mute. + pub fn level(&self) -> Level { + self.shared.level.take() + } + + fn target(&self) -> f32 { + if self.muted() { 0.0 } else { self.gain() } + } +} + +/// The loudest [`Level`] since it was last taken. +#[derive(Debug, Default)] +struct Meter(Mutex); + +impl Meter { + fn record(&self, level: Level) { + let mut held = self.0.lock().unwrap(); + *held = held.max(level); + } + + fn take(&self) -> Level { + std::mem::take(&mut *self.0.lock().unwrap()) + } +} + +/// Backoff bounds for reopening a capture device, matching the capture +/// publication's: a quick first retry covers a USB device re-enumerating, and +/// the ceiling keeps a missing device from spinning. +#[cfg(feature = "capture")] +const RETRY_MIN: Duration = Duration::from_millis(500); +#[cfg(feature = "capture")] +const RETRY_MAX: Duration = Duration::from_secs(4); + +/// An input device feeding a [`Bus`], built by [`Bus::capture`]. +/// +/// Requires the `capture` feature. +#[cfg(feature = "capture")] +pub struct Capture { + bus: Bus, + config: crate::capture::Config, + strip: Strip, +} + +#[cfg(feature = "capture")] +impl Capture { + /// Capture into the mix until the device fails for good, or until the future + /// is dropped, which releases the device. + /// + /// A device that disappears or stops is reopened with backoff; one that can + /// never work (a denied permission, an unsupported format, no such device) + /// returns its error. On macOS the future is `!Send`, because the permission + /// prompt and ScreenCaptureKit hold ObjC handles across an await, so run it + /// on a local task there. + pub async fn run(self) -> Result<(), Error> { + let mut backoff = RETRY_MIN; + loop { + match crate::capture::open(&self.config).await { + Ok(mut stream) => { + let layout = stream.layout(); + let mut input = Input::new(layout.sample_rate, Layout::from_channels(layout.channels)?); + input.format = crate::Format::F32; + let mut channel = self.bus.channel(input, self.strip.clone())?; + backoff = RETRY_MIN; + + loop { + match stream.read().await { + // A gap needs nothing here: the bus has no timeline of its + // own to re-anchor, so the channel just plays on. + Ok(Some(samples)) => channel.push(&samples.data), + Ok(None) => break, + Err(failure) if failure.is_retryable() => { + tracing::warn!(err = %failure.into_error(), "audio capture stopped; reopening"); + break; + } + Err(failure) => return Err(failure.into_error()), + } + } + } + Err(failure) if failure.is_retryable() => { + tracing::warn!(err = %failure.into_error(), ?backoff, "audio capture unavailable; retrying"); + } + Err(failure) => return Err(failure.into_error()), + } + + moq_clock::sleep(backoff).await; + backoff = (backoff * 2).min(RETRY_MAX); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Format; + + /// Interleaved bytes of `frames` frames of `channels` channels, every sample `value`. + fn constant(value: f32, frames: usize, channels: usize) -> Vec { + std::iter::repeat_n(value, frames * channels) + .flat_map(f32::to_le_bytes) + .collect() + } + + fn input(sample_rate: u32, layout: Layout) -> Input { + let mut input = Input::new(sample_rate, layout); + input.format = Format::F32; + input + } + + /// Read blocks of `frames` until `blocks` have passed, returning the last. + fn settle(bus: &Bus, frames: usize, blocks: usize) -> Vec { + let mut out = vec![0.0; frames * bus.config().layout.channels() as usize]; + for _ in 0..blocks { + bus.read(&mut out); + } + out + } + + /// Writes that land before the first read are dropped, so the first read + /// opens the channel and the test feeds it from there, one block per read the + /// way a live source keeps pace with the program clock. + fn feed(bus: &Bus, channel: &mut Channel, value: f32, blocks: usize) -> Vec { + let frames = 480; + let channels = channel.input.layout.channels() as usize; + let in_frames = frames * channel.input.sample_rate as usize / bus.config().sample_rate as usize; + let mut out = vec![0.0; frames * bus.config().layout.channels() as usize]; + bus.read(&mut out); + for _ in 0..blocks { + channel.write(&constant(value, in_frames, channels)).unwrap(); + bus.read(&mut out); + } + out + } + + #[test] + fn sums_channels_with_their_gain() { + let bus = Bus::new(Config::default()).unwrap(); + let mut a = bus.channel(input(48_000, Layout::Stereo), Strip::default()).unwrap(); + let mut b = bus.channel(input(48_000, Layout::Stereo), Strip::default()).unwrap(); + b.strip().set_gain(0.5); + + let mut out = vec![0.0; 480 * 2]; + bus.read(&mut out); + for _ in 0..20 { + a.write(&constant(0.25, 480, 2)).unwrap(); + b.write(&constant(0.25, 480, 2)).unwrap(); + bus.read(&mut out); + } + + // Past the latency fill and the gain ramp, every sample is 0.25 + 0.125. + for sample in &out { + assert!((sample - 0.375).abs() < 1e-4, "{sample}"); + } + } + + #[test] + fn mute_silences_and_keeps_the_gain() { + let bus = Bus::new(Config::default()).unwrap(); + let mut channel = bus.channel(input(48_000, Layout::Stereo), Strip::default()).unwrap(); + channel.strip().set_gain(0.8); + channel.strip().set_muted(true); + + let out = feed(&bus, &mut channel, 0.5, 20); + assert!(out.iter().all(|sample| *sample == 0.0)); + assert_eq!(channel.strip().gain(), 0.8); + assert_eq!(channel.strip().level(), Level::default()); + + channel.strip().set_muted(false); + let out = feed(&bus, &mut channel, 0.5, 20); + assert!((out.last().unwrap() - 0.4).abs() < 1e-4); + } + + #[test] + fn refuses_an_invalid_gain() { + let strip = Strip::default(); + strip.set_gain(0.3); + strip.set_gain(f32::NAN); + strip.set_gain(-1.0); + strip.set_gain(f32::INFINITY); + assert_eq!(strip.gain(), 0.3); + } + + /// A gain change ramps across one block instead of stepping, which would click. + #[test] + fn gain_changes_ramp() { + let bus = Bus::new(Config::default()).unwrap(); + let mut channel = bus.channel(input(48_000, Layout::Stereo), Strip::default()).unwrap(); + feed(&bus, &mut channel, 0.5, 20); + + channel.strip().set_gain(0.0); + channel.write(&constant(0.5, 480, 2)).unwrap(); + let mut out = vec![0.0; 480 * 2]; + bus.read(&mut out); + + let first = out[0]; + let last = out[out.len() - 1]; + assert!(first > 0.49, "the ramp starts at the old gain: {first}"); + assert!(last.abs() < 1e-6, "the ramp ends at the new gain: {last}"); + assert!(out.windows(2).all(|pair| pair[1] <= pair[0] + 1e-6), "monotonic"); + } + + /// A mono source plays from both speakers of a stereo bus, and a source at + /// another rate is resampled to the bus's rather than played fast or slow. + #[test] + fn converts_layout_and_rate() { + let bus = Bus::new(Config::default()).unwrap(); + let mut channel = bus.channel(input(16_000, Layout::Mono), Strip::default()).unwrap(); + let out = feed(&bus, &mut channel, 0.5, 40); + + for frame in out.as_chunks::<2>().0 { + assert!((frame[0] - 0.5).abs() < 1e-2, "{frame:?}"); + assert!((frame[1] - 0.5).abs() < 1e-2, "{frame:?}"); + } + } + + /// A source that stops writing leaves silence, not a repeat of its last block. + #[test] + fn a_stalled_source_is_silent() { + let bus = Bus::new(Config::default()).unwrap(); + let mut channel = bus.channel(input(48_000, Layout::Stereo), Strip::default()).unwrap(); + feed(&bus, &mut channel, 0.5, 20); + + let out = settle(&bus, 480, 20); + assert!(out.iter().all(|sample| *sample == 0.0)); + } + + #[test] + fn dropping_a_channel_leaves_the_mix() { + let bus = Bus::new(Config::default()).unwrap(); + let mut kept = bus.channel(input(48_000, Layout::Stereo), Strip::default()).unwrap(); + let mut dropped = bus.channel(input(48_000, Layout::Stereo), Strip::default()).unwrap(); + feed(&bus, &mut dropped, 0.5, 5); + drop(dropped); + + let out = feed(&bus, &mut kept, 0.25, 20); + assert!((out.last().unwrap() - 0.25).abs() < 1e-4); + } + + /// The meter holds the loudest reading until it is read, so a UI polling + /// slower than the program clock still sees a peak. + #[test] + fn levels_hold_until_read() { + let bus = Bus::new(Config::default()).unwrap(); + let mut channel = bus.channel(input(48_000, Layout::Stereo), Strip::default()).unwrap(); + feed(&bus, &mut channel, 0.5, 20); + settle(&bus, 480, 5); + + let level = channel.strip().level(); + assert!((level.peak() - 0.5).abs() < 1e-3, "{level:?}"); + assert_eq!(channel.strip().level(), Level::default(), "taken"); + assert!((bus.level().peak() - 0.5).abs() < 1e-3); + } + + /// A source that reopens in another format keeps its controls by passing the + /// same strip again, which is what a decoder restart or a replugged device does. + #[test] + fn a_strip_outlives_its_channel() { + let bus = Bus::new(Config::default()).unwrap(); + let strip = Strip::default(); + strip.set_gain(0.5); + let first = bus.channel(input(48_000, Layout::Stereo), strip.clone()).unwrap(); + drop(first); + + let mut second = bus.channel(input(16_000, Layout::Mono), strip.clone()).unwrap(); + let out = feed(&bus, &mut second, 0.5, 40); + assert!((out.last().unwrap() - 0.25).abs() < 1e-2, "{}", out.last().unwrap()); + assert!(strip.level().peak() > 0.2); + } + + #[test] + fn refuses_a_layout_without_speakers() { + let config = Config { + layout: Layout::Discrete(2), + ..Default::default() + }; + assert!(Bus::new(config).is_err()); + + let bus = Bus::new(Config::default()).unwrap(); + assert!(bus.channel(input(48_000, Layout::Discrete(3)), Strip::default()).is_err()); + } + + /// A partial trailing frame in the output is zeroed rather than left with + /// whatever the caller had there. + #[test] + fn zeroes_a_partial_frame() { + let bus = Bus::new(Config::default()).unwrap(); + let mut out = vec![1.0; 5]; + bus.read(&mut out); + assert!(out.iter().all(|sample| *sample == 0.0)); + } +} diff --git a/rs/moq-audio/src/encode/capture.rs b/rs/moq-audio/src/encode/capture.rs index 8a7c29091bc..25420cb5ca4 100644 --- a/rs/moq-audio/src/encode/capture.rs +++ b/rs/moq-audio/src/encode/capture.rs @@ -14,7 +14,7 @@ use super::producer::Reserved; use super::{Input, Options, Producer}; use crate::capture; use crate::resample::{Remix, Resampler}; -use crate::{Error, Format, Frame, Layout as PcmLayout}; +use crate::{Error, Format, Frame, Layout as PcmLayout, Level}; /// Backoff bounds for reopening a capture source. The quick first retry covers /// a USB device re-enumerating; the ceiling keeps a missing device from spinning. @@ -39,47 +39,6 @@ pub enum Status { Ended, } -/// The post-processing level of the most recently captured buffer. -/// -/// Zero while the input is closed. Read it with [`Control::level`] at -/// whatever rate the meter draws at. -#[derive(Clone, Copy, Debug, Default, PartialEq)] -pub struct Level { - rms: f32, - peak: f32, -} - -impl Level { - /// Root mean square amplitude on a linear `0.0..=1.0` scale. - pub fn rms(&self) -> f32 { - self.rms - } - - /// Peak absolute amplitude on a linear `0.0..=1.0` scale. - pub fn peak(&self) -> f32 { - self.peak - } - - fn measure(samples: &[f32]) -> Self { - if samples.is_empty() { - return Self::default(); - } - - let mut squares = 0.0f64; - let mut peak = 0.0f32; - for &sample in samples { - let sample = sample.clamp(-1.0, 1.0); - squares += f64::from(sample) * f64::from(sample); - peak = peak.max(sample.abs()); - } - - Self { - rms: (squares / samples.len() as f64).sqrt() as f32, - peak, - } - } -} - /// A snapshot of a capture-backed publication's lifecycle. /// /// Levels are deliberately not here: they change every buffer, so they would diff --git a/rs/moq-audio/src/encode/mod.rs b/rs/moq-audio/src/encode/mod.rs index 9357a5ca4ca..173ba56822d 100644 --- a/rs/moq-audio/src/encode/mod.rs +++ b/rs/moq-audio/src/encode/mod.rs @@ -30,4 +30,4 @@ pub use encoder::{Codec, Config, Encoder, Finish, Input, Kind}; pub use producer::{Options, Producer}; #[cfg(feature = "capture")] -pub use capture::{Capture, Control, Driver, Level, State, Status, publish_capture}; +pub use capture::{Capture, Control, Driver, State, Status, publish_capture}; diff --git a/rs/moq-audio/src/level.rs b/rs/moq-audio/src/level.rs new file mode 100644 index 00000000000..66da6b77057 --- /dev/null +++ b/rs/moq-audio/src/level.rs @@ -0,0 +1,54 @@ +/// How loud a stretch of audio was, for drawing a meter. +/// +/// Reported by a capture [`Control`](crate::encode::Control) and by a mixing +/// bus and its strips. Read it at whatever rate the meter draws at. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct Level { + rms: f32, + peak: f32, +} + +impl Level { + /// Root mean square amplitude on a linear `0.0..=1.0` scale. + pub fn rms(&self) -> f32 { + self.rms + } + + /// Peak absolute amplitude on a linear `0.0..=1.0` scale. + pub fn peak(&self) -> f32 { + self.peak + } + + #[cfg_attr( + not(any(feature = "capture", feature = "bus")), + allow(dead_code, reason = "only capture and the bus measure levels") + )] + pub(crate) fn measure(samples: &[f32]) -> Self { + if samples.is_empty() { + return Self::default(); + } + + let mut squares = 0.0f64; + let mut peak = 0.0f32; + for &sample in samples { + let sample = sample.clamp(-1.0, 1.0); + squares += f64::from(sample) * f64::from(sample); + peak = peak.max(sample.abs()); + } + + Self { + rms: (squares / samples.len() as f64).sqrt() as f32, + peak, + } + } + + /// The louder of two readings, channel by channel, so a meter read less often + /// than buffers arrive still shows the loudest of them. + #[cfg_attr(not(feature = "bus"), allow(dead_code, reason = "only the bus holds levels"))] + pub(crate) fn max(self, other: Self) -> Self { + Self { + rms: self.rms.max(other.rms), + peak: self.peak.max(other.peak), + } + } +} diff --git a/rs/moq-audio/src/lib.rs b/rs/moq-audio/src/lib.rs index 076e4391679..d7eab8773ea 100644 --- a/rs/moq-audio/src/lib.rs +++ b/rs/moq-audio/src/lib.rs @@ -32,6 +32,9 @@ //! output device and mixes the `playback::Sink`s registered with it, so one //! device serves every track in a call. Requires the `playback` feature, so //! these names are unlinked here too. +//! - `bus` mixes any number of PCM sources, each with its own gain, mute, and +//! meter, into one program feed that the caller reads on its own clock: a +//! studio's program audio. Behind the default-on `bus` feature. //! - `aec` keeps the speaker out of the microphone, which is what a conference //! on a laptop needs to not send itself back. `playback::Engine::canceller` //! builds cloneable `aec::Control` handles from the mix it is playing and @@ -54,12 +57,15 @@ mod format; mod frame; mod jitter; mod layout; +mod level; mod opus; mod pcm; mod resample; #[cfg(feature = "aec")] pub mod aec; +#[cfg(feature = "bus")] +pub mod bus; #[cfg(feature = "capture")] pub mod capture; pub mod decode; @@ -73,3 +79,4 @@ pub use error::Error; pub use format::Format; pub use frame::Frame; pub use layout::Layout; +pub use level::Level; diff --git a/rs/moq-scene/Cargo.toml b/rs/moq-scene/Cargo.toml new file mode 100644 index 00000000000..247e1842dcf --- /dev/null +++ b/rs/moq-scene/Cargo.toml @@ -0,0 +1,51 @@ +[package] +# Unpublished until the studio settles its API; moq-studio is its only consumer. +publish = false +name = "moq-scene" +description = "Scene graph, live sources, and a wgpu compositor for a MoQ streaming studio" +authors = ["Luke Curley "] +repository = "https://github.com/moq-dev/moq" +license = "MIT OR Apache-2.0" + +version = "0.1.0" +edition = "2024" +rust-version.workspace = true + +keywords = ["quic", "media", "video", "compositing", "live"] +categories = ["multimedia", "multimedia::video", "rendering"] + +[features] +# Camera, display, window, and microphone sources. Off by default for the same +# reason as moq-video's and moq-audio's `capture`: on Linux the microphone links +# libasound at build time, and a workspace build would otherwise unify it on for +# every crate. Without it, those sources report that the build cannot capture. +capture = ["moq-video/capture", "moq-audio/capture"] + +[dependencies] +bytemuck = { version = "1", features = ["derive"] } +hang = { workspace = true } +humantime-serde = { workspace = true } +# PNG and JPEG still images. Already in a studio's graph through eframe. +image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } +kio = { workspace = true } +moq-audio = { workspace = true, features = ["bus"] } +moq-clock = { workspace = true } +moq-mux = { workspace = true } +moq-net = { workspace = true } +# `render` is the compositor's whole input path: it turns every decoded or +# captured frame into a texture on the studio's device. +moq-video = { workspace = true, features = ["render"] } +serde = { workspace = true } +serde_with = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["macros", "time"] } +toml = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +pollster = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt", "test-util"] } + +[lints] +workspace = true diff --git a/rs/moq-scene/src/collection.rs b/rs/moq-scene/src/collection.rs new file mode 100644 index 00000000000..1774bb59efb --- /dev/null +++ b/rs/moq-scene/src/collection.rs @@ -0,0 +1,577 @@ +//! The scene graph, and the TOML file a [`Collection`] saves to. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; +use std::path::PathBuf; +use std::str::FromStr; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use serde_with::{DeserializeFromStr, SerializeDisplay}; + +use crate::Error; + +/// Everything a studio composes: the canvas, the sources, and the scenes that +/// place them. +/// +/// Saved with [`to_toml`](Self::to_toml) and loaded with +/// [`from_toml`](Self::from_toml). The file names devices, files, and broadcast +/// paths, never a relay URL or a token, so it is safe to commit and share. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Collection { + /// The output every scene is drawn onto. + pub canvas: Canvas, + /// How a fade from preview to program behaves. + #[serde(default)] + pub transition: Transition, + /// Every source, by its unique name. Scenes refer to sources by that name, + /// so one camera can appear in many scenes while being opened once. + #[serde(default)] + pub sources: BTreeMap, + /// Gain and mute of each audio-carrying source, by source name. A source + /// without an entry plays at unity. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub audio: BTreeMap, + /// The scenes, in the order the scene list shows them. + #[serde(default)] + pub scenes: Vec, +} + +impl Collection { + /// The scene file format version this build reads and writes. + /// + /// A file names its version, and loading any other one fails rather than + /// guessing at fields whose meaning changed. + pub const VERSION: u32 = 1; + + /// An empty collection with one blank scene on a 1080p, 30 fps canvas. + pub fn new() -> Self { + Self { + canvas: Canvas::default(), + transition: Transition::default(), + sources: BTreeMap::new(), + audio: BTreeMap::new(), + scenes: vec![Scene::new("Scene")], + } + } + + /// Parse and validate a scene file. + pub fn from_toml(text: &str) -> Result { + let mut table: toml::Table = toml::from_str(text).map_err(|err| Error::Format(err.to_string()))?; + let version = match table.remove("version") { + Some(toml::Value::Integer(version)) => version, + Some(other) => return Err(Error::Format(format!("`version` must be an integer, got {other}"))), + None => return Err(Error::Format("missing `version`".into())), + }; + if version != i64::from(Self::VERSION) { + return Err(Error::Version { + found: version, + supported: Self::VERSION, + }); + } + + let collection = Self::deserialize(table).map_err(|err| Error::Format(err.to_string()))?; + collection.validate()?; + Ok(collection) + } + + /// Serialize to the scene file format, `version` first. + pub fn to_toml(&self) -> Result { + self.validate()?; + let body = toml::to_string(self).map_err(|err| Error::Format(err.to_string()))?; + Ok(format!("version = {}\n\n{body}", Self::VERSION)) + } + + /// The scene called `name`. + pub fn scene(&self, name: &str) -> Option<&Scene> { + self.scenes.iter().find(|scene| scene.name == name) + } + + /// The scene called `name`, to edit. + pub fn scene_mut(&mut self, name: &str) -> Option<&mut Scene> { + self.scenes.iter_mut().find(|scene| scene.name == name) + } + + /// Check what the types cannot: the canvas can be encoded, names are unique, + /// and every item and audio entry names a source that exists. + pub fn validate(&self) -> Result<(), Error> { + self.canvas.validate()?; + + let mut names = BTreeSet::new(); + for scene in &self.scenes { + if scene.name.is_empty() { + return Err(Error::Invalid("a scene has an empty name".into())); + } + if !names.insert(scene.name.as_str()) { + return Err(Error::Invalid(format!("two scenes are named `{}`", scene.name))); + } + for item in &scene.items { + if !self.sources.contains_key(&item.source) { + return Err(Error::Invalid(format!( + "scene `{}` places `{}`, which is not a source", + scene.name, item.source + ))); + } + item.rect.validate(&scene.name, &item.source)?; + } + } + + if let Some(name) = self.sources.keys().find(|name| name.is_empty()) { + return Err(Error::Invalid(format!("a source has an empty name: `{name}`"))); + } + + for (name, audio) in &self.audio { + if !self.sources.contains_key(name) { + return Err(Error::Invalid(format!("audio settings for `{name}`, which is not a source"))); + } + if !audio.gain.is_finite() || audio.gain < 0.0 { + return Err(Error::Invalid(format!("`{name}` has gain {}", audio.gain))); + } + } + + Ok(()) + } +} + +impl Default for Collection { + fn default() -> Self { + Self::new() + } +} + +/// The output picture: its size and how often it is drawn. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Canvas { + /// Width in pixels. Even, since the encoder subsamples chroma. + pub width: u32, + /// Height in pixels. Even, since the encoder subsamples chroma. + pub height: u32, + /// Pictures drawn and encoded per second. + pub framerate: u32, +} + +impl Canvas { + /// The canvas size. + pub fn size(&self) -> moq_video::Size { + moq_video::Size::new(self.width, self.height) + } + + /// How often a picture is drawn. + pub fn framerate(&self) -> moq_video::FrameRate { + moq_video::FrameRate::new(self.framerate, 1).expect("validated canvas framerate") + } + + fn validate(&self) -> Result<(), Error> { + if self.width == 0 || self.height == 0 || !self.width.is_multiple_of(2) || !self.height.is_multiple_of(2) { + return Err(Error::Invalid(format!( + "canvas {}x{} must be non-zero and even", + self.width, self.height + ))); + } + moq_video::FrameRate::new(self.framerate, 1) + .map_err(|err| Error::Invalid(format!("canvas framerate {}: {err}", self.framerate)))?; + Ok(()) + } +} + +impl Default for Canvas { + fn default() -> Self { + Self { + width: 1920, + height: 1080, + framerate: 30, + } + } +} + +/// How the program changes when the preview is faded in. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Transition { + /// How long a fade takes. + #[serde(with = "humantime_serde")] + pub fade: Duration, +} + +impl Default for Transition { + fn default() -> Self { + Self { + fade: Duration::from_millis(300), + } + } +} + +/// One arrangement of sources on the canvas. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Scene { + /// Unique among the collection's scenes. + pub name: String, + /// What the scene draws, back to front. + #[serde(default)] + pub items: Vec, +} + +impl Scene { + /// An empty scene. + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + items: Vec::new(), + } + } +} + +/// A source placed in a scene. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Item { + /// The [`Collection::sources`] entry to draw. + pub source: String, + /// Where on the canvas, stretched to fill it. + pub rect: Rect, + /// Whether the item is drawn. A hidden item keeps its place. + #[serde(default = "visible")] + pub visible: bool, +} + +fn visible() -> bool { + true +} + +/// A rectangle in canvas pixels, from the top-left corner. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Rect { + /// Left edge. + pub x: f32, + /// Top edge. + pub y: f32, + /// Width. + pub width: f32, + /// Height. + pub height: f32, +} + +impl Rect { + /// The whole of `canvas`. + pub fn fill(canvas: &Canvas) -> Self { + Self { + x: 0.0, + y: 0.0, + width: canvas.width as f32, + height: canvas.height as f32, + } + } + + fn validate(&self, scene: &str, source: &str) -> Result<(), Error> { + let finite = [self.x, self.y, self.width, self.height].iter().all(|v| v.is_finite()); + if !finite || self.width < 0.0 || self.height < 0.0 { + return Err(Error::Invalid(format!( + "scene `{scene}` places `{source}` at {self:?}, which is not a rectangle" + ))); + } + Ok(()) + } +} + +/// Where a source's picture or sound comes from. +/// +/// Device ids are the ones the matching `moq_video::capture` and +/// `moq_audio::capture` listings report; `None` means the system default. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum Source { + /// A camera. + Camera { + /// The camera's id, or the default camera. + #[serde(default, skip_serializing_if = "Option::is_none")] + id: Option, + }, + /// A whole display. + Display { + /// The display's id, or the main display. + #[serde(default, skip_serializing_if = "Option::is_none")] + id: Option, + }, + /// One window. + Window { + /// The window's id. + id: String, + }, + /// A microphone or other audio input. + Microphone { + /// The input's id, or the default input. + #[serde(default, skip_serializing_if = "Option::is_none")] + id: Option, + }, + /// Everything the machine plays, where the platform can capture it. + SystemAudio, + /// A PNG or JPEG still. + Image { + /// The file, relative to the scene file's directory unless absolute. + path: PathBuf, + }, + /// A solid color. + Color { + /// The color, as `#rrggbb` or `#rrggbbaa`. + color: Rgba, + }, + /// A broadcast subscribed from the relay the studio is connected to. + Broadcast { + /// The broadcast path on that relay. + path: String, + }, +} + +impl Source { + /// Whether the source has a picture to place in a scene. + pub fn has_video(&self) -> bool { + !matches!(self, Self::Microphone { .. } | Self::SystemAudio) + } + + /// Whether the source feeds the program audio. + pub fn has_audio(&self) -> bool { + matches!(self, Self::Microphone { .. } | Self::SystemAudio | Self::Broadcast { .. }) + } +} + +/// The gain and mute of one source in the program audio. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Audio { + /// Linear gain: 1.0 is unity. + pub gain: f32, + /// Whether the source is silenced, keeping its gain. + #[serde(default)] + pub muted: bool, +} + +impl Default for Audio { + fn default() -> Self { + Self { gain: 1.0, muted: false } + } +} + +/// An sRGB color with straight alpha, written `#rrggbb` or `#rrggbbaa`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, SerializeDisplay, DeserializeFromStr)] +pub struct Rgba(pub [u8; 4]); + +impl fmt::Display for Rgba { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let [r, g, b, a] = self.0; + if a == u8::MAX { + write!(f, "#{r:02x}{g:02x}{b:02x}") + } else { + write!(f, "#{r:02x}{g:02x}{b:02x}{a:02x}") + } + } +} + +impl FromStr for Rgba { + type Err = Error; + + fn from_str(text: &str) -> Result { + let invalid = || Error::Invalid(format!("`{text}` is not a #rrggbb or #rrggbbaa color")); + let hex = text.strip_prefix('#').ok_or_else(invalid)?; + if !matches!(hex.len(), 6 | 8) || !hex.is_ascii() { + return Err(invalid()); + } + let mut rgba = [u8::MAX; 4]; + for (channel, pair) in rgba.iter_mut().zip(hex.as_bytes().as_chunks::<2>().0) { + let pair = std::str::from_utf8(pair).map_err(|_| invalid())?; + *channel = u8::from_str_radix(pair, 16).map_err(|_| invalid())?; + } + Ok(Self(rgba)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn example() -> Collection { + let mut collection = Collection::new(); + collection + .sources + .insert("camera".into(), Source::Camera { id: Some("/dev/video0".into()) }); + collection.sources.insert( + "logo".into(), + Source::Image { + path: "logo.png".into(), + }, + ); + collection.sources.insert( + "background".into(), + Source::Color { + color: Rgba([0x10, 0x20, 0x30, 0xff]), + }, + ); + collection.sources.insert("mic".into(), Source::Microphone { id: None }); + collection.sources.insert( + "guest".into(), + Source::Broadcast { + path: "studio/guest".into(), + }, + ); + collection.audio.insert("mic".into(), Audio { gain: 0.5, muted: true }); + + let canvas = collection.canvas; + let scene = &mut collection.scenes[0]; + scene.items.push(Item { + source: "background".into(), + rect: Rect::fill(&canvas), + visible: true, + }); + scene.items.push(Item { + source: "camera".into(), + rect: Rect { + x: 100.0, + y: 50.0, + width: 640.0, + height: 360.0, + }, + visible: false, + }); + collection + } + + #[test] + fn round_trips() { + let collection = example(); + let text = collection.to_toml().unwrap(); + assert!(text.starts_with("version = 1\n"), "{text}"); + assert_eq!(Collection::from_toml(&text).unwrap(), collection); + } + + /// The file is meant to be read and diffed by people, so pin what it looks like. + #[test] + fn reads_a_handwritten_file() { + let text = r##" +version = 1 + +[canvas] +width = 1280 +height = 720 +framerate = 60 + +[transition] +fade = "500ms" + +[sources.cam] +kind = "camera" + +[sources.bg] +kind = "color" +color = "#ff000080" + +[audio.cam2] +gain = 2.0 + +[sources.cam2] +kind = "broadcast" +path = "room/alice" + +[[scenes]] +name = "Main" +items = [ + { source = "bg", rect = { x = 0, y = 0, width = 1280, height = 720 } }, + { source = "cam", rect = { x = 10, y = 10, width = 320, height = 180 }, visible = false }, +] +"##; + let collection = Collection::from_toml(text).unwrap(); + assert_eq!(collection.canvas.framerate, 60); + assert_eq!(collection.transition.fade, Duration::from_millis(500)); + assert_eq!(collection.sources["cam"], Source::Camera { id: None }); + assert_eq!( + collection.sources["bg"], + Source::Color { + color: Rgba([255, 0, 0, 128]) + } + ); + assert_eq!(collection.audio["cam2"], Audio { gain: 2.0, muted: false }); + let main = collection.scene("Main").unwrap(); + assert!(main.items[0].visible); + assert!(!main.items[1].visible); + } + + #[test] + fn refuses_another_version() { + let text = example().to_toml().unwrap().replacen("version = 1", "version = 2", 1); + let err = Collection::from_toml(&text).unwrap_err(); + assert!(matches!(err, Error::Version { found: 2, supported: 1 }), "{err}"); + + let text = example().to_toml().unwrap().replacen("version = 1", "", 1); + assert!(matches!(Collection::from_toml(&text), Err(Error::Format(_)))); + } + + /// A typo or a field from a newer build fails instead of being dropped on the + /// next save. + #[test] + fn refuses_unknown_fields() { + let text = example().to_toml().unwrap().replacen("[canvas]", "[canvas]\ndepth = 3", 1); + assert!(matches!(Collection::from_toml(&text), Err(Error::Format(_)))); + + let text = example() + .to_toml() + .unwrap() + .replacen("kind = \"camera\"", "kind = \"camera\"\nzoom = 2", 1); + assert!(matches!(Collection::from_toml(&text), Err(Error::Format(_)))); + + let text = example() + .to_toml() + .unwrap() + .replacen("kind = \"camera\"", "kind = \"hologram\"", 1); + assert!(matches!(Collection::from_toml(&text), Err(Error::Format(_)))); + } + + #[test] + fn refuses_dangling_references() { + let mut collection = example(); + collection.sources.remove("camera"); + assert!(matches!(collection.to_toml(), Err(Error::Invalid(_)))); + + let mut collection = example(); + collection.audio.insert("nobody".into(), Audio::default()); + assert!(matches!(collection.validate(), Err(Error::Invalid(_)))); + } + + #[test] + fn refuses_duplicate_scenes() { + let mut collection = example(); + collection.scenes.push(Scene::new("Scene")); + assert!(matches!(collection.validate(), Err(Error::Invalid(_)))); + } + + #[test] + fn refuses_a_canvas_the_encoder_cannot_take() { + for (width, height, framerate) in [(1921, 1080, 30), (0, 1080, 30), (1920, 1080, 0)] { + let mut collection = example(); + collection.canvas = Canvas { + width, + height, + framerate, + }; + assert!(matches!(collection.validate(), Err(Error::Invalid(_))), "{width}x{height}@{framerate}"); + } + } + + #[test] + fn refuses_a_degenerate_rect() { + let mut collection = example(); + collection.scenes[0].items[0].rect.width = f32::NAN; + assert!(matches!(collection.validate(), Err(Error::Invalid(_)))); + } + + #[test] + fn colors_parse_and_print() { + assert_eq!("#102030".parse::().unwrap(), Rgba([0x10, 0x20, 0x30, 0xff])); + assert_eq!("#10203040".parse::().unwrap().to_string(), "#10203040"); + assert_eq!(Rgba([1, 2, 3, 255]).to_string(), "#010203"); + for bad in ["102030", "#1020", "#gg0000", "#ééé"] { + assert!(bad.parse::().is_err(), "{bad}"); + } + } +} diff --git a/rs/moq-scene/src/compositor.rs b/rs/moq-scene/src/compositor.rs new file mode 100644 index 00000000000..b1256dcd6c9 --- /dev/null +++ b/rs/moq-scene/src/compositor.rs @@ -0,0 +1,866 @@ +//! Draw scenes onto the canvas on the GPU: a preview, a program, and the fade +//! between them. + +use std::collections::HashMap; +use std::num::NonZeroU64; +use std::time::Duration; + +use moq_clock::Instant; +use moq_video::render::{Renderer, wgpu}; + +use crate::collection::{Canvas, Collection, Rect}; +use crate::source::{Picture, Sources}; +use crate::Error; + +/// The format of every canvas texture: gamma-encoded RGBA, what +/// `moq_video::render` produces and what an encoder reads back. +pub const FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Rgba8Unorm; + +/// One draw's uniform: where on the canvas, in clip space, and how opaque. +#[repr(C)] +#[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)] +struct Uniform { + /// Top-left x, top-left y, bottom-right x, bottom-right y, in clip space. + rect: [f32; 4], + /// Opacity in `x`; the rest pads the struct to WGSL's 16-byte vectors. + opacity: [f32; 4], +} + +const SHADER: &str = r#" +struct Item { + rect: vec4, + opacity: vec4, +} + +@group(0) @binding(0) var item: Item; +@group(1) @binding(0) var image: texture_2d; +@group(1) @binding(1) var image_sampler: sampler; + +struct Varyings { + @builtin(position) position: vec4, + @location(0) uv: vec2, +} + +// A four-vertex triangle strip covering the item's rectangle. +@vertex fn vs_main(@builtin(vertex_index) index: u32) -> Varyings { + let corner = vec2(f32(index & 1u), f32((index >> 1u) & 1u)); + var out: Varyings; + out.position = vec4(mix(item.rect.xy, item.rect.zw, corner), 0.0, 1.0); + out.uv = corner; + return out; +} + +@fragment fn fs_main(in: Varyings) -> @location(0) vec4 { + let color = textureSample(image, image_sampler, in.uv); + return vec4(color.rgb, color.a * item.opacity.x); +} +"#; + +/// Draws a [`Collection`]'s scenes onto its canvas: one texture for the scene +/// in preview, one for the scene on program, and the fade from one to the other. +/// +/// Both textures are plain [`wgpu::Texture`]s on the caller's device, so a UI +/// shows them without a copy and an encoder reads the program from the same +/// memory. [`render`](Self::render) redraws both from the newest picture each +/// source has, so a source that stalls repeats its last picture rather than +/// holding up the canvas. +/// +/// Pixels are blended in the gamma-encoded space sources arrive in, as a +/// broadcast switcher does, so a 50% fade is a 50% mix of the stored values. +pub struct Compositor { + device: wgpu::Device, + queue: wgpu::Queue, + canvas: Canvas, + + pipeline: wgpu::RenderPipeline, + image_layout: wgpu::BindGroupLayout, + sampler: wgpu::Sampler, + + uniform_layout: wgpu::BindGroupLayout, + uniforms: Uniforms, + + preview: Target, + program: Target, + /// The scene fading in, drawn whole so it blends over the program as one + /// picture rather than layer by layer. + incoming: Target, + + layers: HashMap, + preview_scene: Option, + program_scene: Option, + fade: Option, +} + +/// A fade from `from` to the program scene. +struct Fade { + from: Option, + start: Instant, + duration: Duration, +} + +/// A canvas-sized texture drawn into, and the bind group that samples it. +struct Target { + texture: wgpu::Texture, + view: wgpu::TextureView, + bind: wgpu::BindGroup, +} + +/// One source's newest picture on the GPU. +#[derive(Default)] +struct Layer { + /// Converts video frames, created on the first one. + renderer: Option, + /// Samples the newest picture; `None` until the first arrives. + bind: Option, + /// Set once a picture failed to draw, so the failure is logged once. + failed: bool, +} + +/// A growable uniform buffer holding one [`Uniform`] per draw of a render. +struct Uniforms { + buffer: wgpu::Buffer, + bind: wgpu::BindGroup, + /// Bytes between draws, rounded up to the device's dynamic-offset alignment. + stride: u64, + capacity: u64, +} + +/// One textured quad: what to sample, and the uniform that places it. +struct Draw { + bind: wgpu::BindGroup, + uniform: Uniform, +} + +/// One render pass: a target cleared to black and the quads drawn into it. +struct Pass<'a> { + view: &'a wgpu::TextureView, + draws: Vec, +} + +impl Compositor { + /// A compositor drawing `canvas`-sized pictures on `device`. + /// + /// Fails if the device cannot hold a texture that large. + pub fn new(device: &wgpu::Device, queue: &wgpu::Queue, canvas: Canvas) -> Result { + let limit = device.limits().max_texture_dimension_2d; + if canvas.width > limit || canvas.height > limit { + return Err(Error::Invalid(format!( + "canvas {}x{} exceeds this GPU's {limit}px texture limit", + canvas.width, canvas.height + ))); + } + + let image_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("moq-scene image"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }); + let uniform_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("moq-scene item"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX_FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: true, + min_binding_size: NonZeroU64::new(size_of::() as u64), + }, + count: None, + }], + }); + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("moq-scene"), + bind_group_layouts: &[Some(&uniform_layout), Some(&image_layout)], + immediate_size: 0, + }); + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("moq-scene"), + source: wgpu::ShaderSource::Wgsl(SHADER.into()), + }); + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("moq-scene"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + compilation_options: Default::default(), + buffers: &[], + }, + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleStrip, + ..Default::default() + }, + depth_stencil: None, + multisample: Default::default(), + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format: FORMAT, + // Straight alpha over an opaque canvas, which stays opaque. + blend: Some(wgpu::BlendState { + color: wgpu::BlendComponent { + src_factor: wgpu::BlendFactor::SrcAlpha, + dst_factor: wgpu::BlendFactor::OneMinusSrcAlpha, + operation: wgpu::BlendOperation::Add, + }, + alpha: wgpu::BlendComponent { + src_factor: wgpu::BlendFactor::One, + dst_factor: wgpu::BlendFactor::OneMinusSrcAlpha, + operation: wgpu::BlendOperation::Add, + }, + }), + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }); + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("moq-scene"), + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + ..Default::default() + }); + + let alignment = u64::from(device.limits().min_uniform_buffer_offset_alignment); + let stride = (size_of::() as u64).next_multiple_of(alignment); + let uniforms = Uniforms::new(device, &uniform_layout, stride, 16); + + let target = |label| Target::new(device, &image_layout, &sampler, canvas, label); + Ok(Self { + preview: target("moq-scene preview"), + program: target("moq-scene program"), + incoming: target("moq-scene incoming"), + device: device.clone(), + queue: queue.clone(), + canvas, + pipeline, + image_layout, + sampler, + uniform_layout, + uniforms, + layers: HashMap::new(), + preview_scene: None, + program_scene: None, + fade: None, + }) + } + + /// The canvas this compositor draws. + pub fn canvas(&self) -> Canvas { + self.canvas + } + + /// The scene in preview, as of the last [`render`](Self::render). + pub fn preview(&self) -> &wgpu::Texture { + &self.preview.texture + } + + /// The program output, as of the last [`render`](Self::render). Created + /// with `COPY_SRC` so an encoder can read it back. + pub fn program(&self) -> &wgpu::Texture { + &self.program.texture + } + + /// The scene in preview. + pub fn preview_scene(&self) -> Option<&str> { + self.preview_scene.as_deref() + } + + /// The scene on program, or the one fading in. + pub fn program_scene(&self) -> Option<&str> { + self.program_scene.as_deref() + } + + /// Whether a fade is under way. + pub fn fading(&self) -> bool { + self.fade.is_some() + } + + /// Put the scene called `scene` in preview, or nothing. + pub fn set_preview(&mut self, scene: Option) { + self.preview_scene = scene; + } + + /// Put the preview on program at once, ending any fade. + pub fn cut(&mut self) { + self.fade = None; + self.program_scene = self.preview_scene.clone(); + } + + /// Fade the preview onto program over `duration`, starting at `now`. + /// + /// A fade already under way lands on its scene first, so this one starts + /// from what that fade was heading to. + pub fn fade(&mut self, duration: Duration, now: Instant) { + let from = std::mem::replace(&mut self.program_scene, self.preview_scene.clone()); + self.fade = (!duration.is_zero()).then_some(Fade { + from, + start: now, + duration, + }); + } + + /// Draw preview and program as of `now`, from the newest picture each of + /// `sources` has. + /// + /// A source that fails to draw is left out and logged once; this fails only + /// when the canvas itself cannot be drawn. + pub fn render(&mut self, collection: &Collection, sources: &Sources, now: Instant) -> Result<(), Error> { + self.update(collection, sources); + + let progress = self.fade.as_ref().map(|fade| { + let elapsed = now.saturating_duration_since(fade.start); + (elapsed.as_secs_f32() / fade.duration.as_secs_f32()).min(1.0) + }); + if progress.is_some_and(|progress| progress >= 1.0) { + self.fade = None; + } + + let mut passes = vec![Pass { + view: &self.preview.view, + draws: self.scene(collection, self.preview_scene.as_deref()), + }]; + match (&self.fade, progress) { + (Some(fade), Some(progress)) => { + passes.push(Pass { + view: &self.incoming.view, + draws: self.scene(collection, self.program_scene.as_deref()), + }); + let mut draws = self.scene(collection, fade.from.as_deref()); + draws.push(Draw { + bind: self.incoming.bind.clone(), + uniform: uniform(Rect::fill(&self.canvas), self.canvas, progress), + }); + passes.push(Pass { + view: &self.program.view, + draws, + }); + } + _ => passes.push(Pass { + view: &self.program.view, + draws: self.scene(collection, self.program_scene.as_deref()), + }), + } + + let count: usize = passes.iter().map(|pass| pass.draws.len()).sum(); + self.uniforms.reserve(&self.device, &self.uniform_layout, count as u64); + let stride = self.uniforms.stride as usize; + let mut bytes = vec![0u8; count * stride]; + for (index, draw) in passes.iter().flat_map(|pass| &pass.draws).enumerate() { + let offset = index * stride; + bytes[offset..offset + size_of::()].copy_from_slice(bytemuck::bytes_of(&draw.uniform)); + } + if !bytes.is_empty() { + self.queue.write_buffer(&self.uniforms.buffer, 0, &bytes); + } + + let mut encoder = self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("moq-scene"), + }); + let mut index = 0u32; + for pass in &passes { + let mut render = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("moq-scene"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: pass.view, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + render.set_pipeline(&self.pipeline); + for draw in &pass.draws { + render.set_bind_group(0, &self.uniforms.bind, &[index * stride as u32]); + render.set_bind_group(1, &draw.bind, &[]); + render.draw(0..4, 0..1); + index += 1; + } + } + self.queue.submit([encoder.finish()]); + Ok(()) + } + + /// The quads drawing `scene`, back to front. A missing scene, a hidden item, + /// or a source with no picture yet draws nothing. + fn scene(&self, collection: &Collection, scene: Option<&str>) -> Vec { + let Some(scene) = scene.and_then(|name| collection.scene(name)) else { + return Vec::new(); + }; + scene + .items + .iter() + .filter(|item| item.visible) + .filter_map(|item| { + let bind = self.layers.get(&item.source)?.bind.clone()?; + Some(Draw { + bind, + uniform: uniform(item.rect, self.canvas, 1.0), + }) + }) + .collect() + } + + /// Pull the newest picture from every source onto the GPU, and forget the + /// sources the collection no longer has. + fn update(&mut self, collection: &Collection, sources: &Sources) { + self.layers.retain(|name, _| collection.sources.contains_key(name)); + + for name in collection.sources.keys() { + let Some(picture) = sources.get(name).and_then(|feed| feed.take()) else { + continue; + }; + let layer = self.layers.entry(name.clone()).or_default(); + let texture = match picture { + Picture::Frame(frame) => { + let renderer = match layer.renderer.as_mut() { + Some(renderer) => Ok(renderer), + None => Renderer::new(&self.device, &self.queue, Default::default()) + .map(|renderer| layer.renderer.insert(renderer)), + }; + renderer.and_then(|renderer| renderer.render(&frame)).map_err(Error::from) + } + Picture::Rgba { size, pixels } => upload(&self.device, &self.queue, size, &pixels), + }; + + match texture { + Ok(texture) => { + layer.failed = false; + let view = texture.create_view(&Default::default()); + layer.bind = Some(image_bind(&self.device, &self.image_layout, &self.sampler, &view)); + } + Err(err) if !layer.failed => { + tracing::warn!(source = %name, %err, "cannot draw source"); + layer.failed = true; + } + Err(_) => {} + } + } + } +} + +/// Place `rect` (canvas pixels) in clip space. +fn uniform(rect: Rect, canvas: Canvas, opacity: f32) -> Uniform { + let (width, height) = (canvas.width as f32, canvas.height as f32); + let x = |px: f32| px / width * 2.0 - 1.0; + let y = |px: f32| 1.0 - px / height * 2.0; + Uniform { + rect: [x(rect.x), y(rect.y), x(rect.x + rect.width), y(rect.y + rect.height)], + opacity: [opacity, 0.0, 0.0, 0.0], + } +} + +/// A texture holding tightly packed RGBA pixels. +fn upload(device: &wgpu::Device, queue: &wgpu::Queue, size: moq_video::Size, pixels: &[u8]) -> Result { + let limit = device.limits().max_texture_dimension_2d; + if size.width == 0 || size.height == 0 || size.width > limit || size.height > limit { + return Err(Error::Invalid(format!( + "a {size} picture does not fit this GPU's {limit}px texture limit" + ))); + } + + let extent = wgpu::Extent3d { + width: size.width, + height: size.height, + depth_or_array_layers: 1, + }; + let texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("moq-scene picture"), + size: extent, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: FORMAT, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + pixels, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(size.width * 4), + rows_per_image: Some(size.height), + }, + extent, + ); + Ok(texture) +} + +fn image_bind( + device: &wgpu::Device, + layout: &wgpu::BindGroupLayout, + sampler: &wgpu::Sampler, + view: &wgpu::TextureView, +) -> wgpu::BindGroup { + device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("moq-scene image"), + layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(sampler), + }, + ], + }) +} + +impl Target { + fn new( + device: &wgpu::Device, + layout: &wgpu::BindGroupLayout, + sampler: &wgpu::Sampler, + canvas: Canvas, + label: &str, + ) -> Self { + let texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size: wgpu::Extent3d { + width: canvas.width, + height: canvas.height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: FORMAT, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let view = texture.create_view(&Default::default()); + let bind = image_bind(device, layout, sampler, &view); + Self { texture, view, bind } + } +} + +impl Uniforms { + fn new(device: &wgpu::Device, layout: &wgpu::BindGroupLayout, stride: u64, capacity: u64) -> Self { + let buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("moq-scene items"), + size: stride * capacity, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + let bind = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("moq-scene items"), + layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::Buffer(wgpu::BufferBinding { + buffer: &buffer, + offset: 0, + size: NonZeroU64::new(size_of::() as u64), + }), + }], + }); + Self { + buffer, + bind, + stride, + capacity, + } + } + + /// Grow to hold `count` draws, doubling so a growing scene reallocates rarely. + fn reserve(&mut self, device: &wgpu::Device, layout: &wgpu::BindGroupLayout, count: u64) { + if count > self.capacity { + *self = Self::new(device, layout, self.stride, count.next_power_of_two()); + } + } +} + +#[cfg(test)] +mod tests { + use moq_net::Timestamp; + + use super::*; + use crate::collection::{Item, Rgba, Scene, Source}; + use crate::readback::Readback; + use crate::source::test::{context, local, start}; + + const RED: [u8; 4] = [255, 0, 0, 255]; + const BLUE: [u8; 4] = [0, 0, 255, 255]; + const BLACK: [u8; 4] = [0, 0, 0, 255]; + + async fn gpu() -> (wgpu::Device, wgpu::Queue) { + let instance = wgpu::Instance::default(); + let adapter = instance + .request_adapter(&wgpu::RequestAdapterOptions::default()) + .await + .expect("a GPU adapter"); + adapter + .request_device(&wgpu::DeviceDescriptor::default()) + .await + .expect("a GPU device") + } + + /// An 8x8 canvas with a red and a blue source, and scenes "red" (red filling + /// the canvas, blue over its top-right quarter) and "blue" (blue filling it). + fn collection() -> Collection { + let mut collection = Collection::new(); + collection.canvas = Canvas { + width: 8, + height: 8, + framerate: 30, + }; + collection.sources.insert("red".into(), Source::Color { color: Rgba(RED) }); + collection.sources.insert("blue".into(), Source::Color { color: Rgba(BLUE) }); + let item = |source: &str, rect| Item { + source: source.into(), + rect, + visible: true, + }; + let fill = Rect::fill(&collection.canvas); + let corner = Rect { + x: 4.0, + y: 0.0, + width: 4.0, + height: 4.0, + }; + collection.scenes = vec![ + Scene { + name: "red".into(), + items: vec![item("red", fill), item("blue", corner)], + }, + Scene { + name: "blue".into(), + items: vec![item("blue", fill)], + }, + ]; + collection.validate().unwrap(); + collection + } + + /// A compositor over running sources, read back to check pixels. + struct Studio { + collection: Collection, + sources: Sources, + compositor: Compositor, + readback: Readback, + _origin: moq_net::origin::Producer, + _dir: tempfile::TempDir, + } + + impl Studio { + async fn new(collection: Collection) -> Self { + let (device, queue) = gpu().await; + let dir = tempfile::tempdir().unwrap(); + let (context, origin) = context(dir.path()); + let mut sources = Sources::new(context); + let drivers = sources.sync(&collection); + start(&sources, drivers).await; + Self { + compositor: Compositor::new(&device, &queue, collection.canvas).unwrap(), + readback: Readback::new(&device, &queue, collection.canvas), + collection, + sources, + _origin: origin, + _dir: dir, + } + } + + fn render(&mut self, now: Instant) { + self.compositor.render(&self.collection, &self.sources, now).unwrap(); + } + + fn read(&mut self, texture: &wgpu::Texture) -> Vec { + self.readback.copy(texture, Timestamp::ZERO).unwrap().rgba().unwrap() + } + + fn program(&mut self) -> Vec { + let texture = self.compositor.program().clone(); + self.read(&texture) + } + + fn preview(&mut self) -> Vec { + let texture = self.compositor.preview().clone(); + self.read(&texture) + } + } + + fn pixel(rgba: &[u8], x: usize, y: usize) -> [u8; 4] { + let at = (y * 8 + x) * 4; + rgba[at..at + 4].try_into().unwrap() + } + + fn near(actual: [u8; 4], expected: [u8; 4]) -> bool { + actual.iter().zip(expected).all(|(a, e)| a.abs_diff(e) <= 1) + } + + /// Items draw where their rect says, later items over earlier ones, and + /// a hidden item not at all. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn items_draw_in_place_and_in_order() { + local(async { + let mut studio = Studio::new(collection()).await; + studio.compositor.set_preview(Some("red".into())); + studio.compositor.cut(); + studio.render(Instant::now()); + let program = studio.program(); + assert_eq!(pixel(&program, 1, 1), RED); + assert_eq!(pixel(&program, 6, 1), BLUE); + assert_eq!(pixel(&program, 6, 6), RED); + assert_eq!(studio.preview(), program, "preview shows the same scene"); + + studio.collection.scenes[0].items[1].visible = false; + studio.render(Instant::now()); + assert_eq!(pixel(&studio.program(), 6, 1), RED); + }) + .await; + } + + /// Straight alpha blends over what is below, and the canvas stays opaque. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn alpha_blends_over_black() { + local(async { + let mut collection = collection(); + collection.sources.insert( + "red".into(), + Source::Color { + color: Rgba([255, 0, 0, 128]), + }, + ); + collection.scenes[0].items.truncate(1); + let mut studio = Studio::new(collection).await; + studio.compositor.set_preview(Some("red".into())); + studio.compositor.cut(); + studio.render(Instant::now()); + let actual = pixel(&studio.program(), 1, 1); + assert!(near(actual, [128, 0, 0, 255]), "{actual:?}"); + }) + .await; + } + + /// A fade mixes the outgoing and incoming scenes by its progress on the + /// program clock, and lands on the incoming scene; preview never fades. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn a_fade_mixes_by_progress() { + local(async { + let mut studio = Studio::new(collection()).await; + let start = Instant::now(); + studio.compositor.set_preview(Some("red".into())); + studio.compositor.cut(); + studio.compositor.set_preview(Some("blue".into())); + studio.compositor.fade(Duration::from_secs(1), start); + assert_eq!(studio.compositor.program_scene(), Some("blue")); + + studio.render(start + Duration::from_millis(500)); + assert!(studio.compositor.fading()); + let program = studio.program(); + let actual = pixel(&program, 1, 1); + assert!(near(actual, [128, 0, 127, 255]), "{actual:?}"); + assert_eq!(pixel(&program, 6, 1), BLUE, "blue under blue stays blue"); + assert_eq!(pixel(&studio.preview(), 1, 1), BLUE); + + studio.render(start + Duration::from_secs(1)); + assert!(!studio.compositor.fading()); + assert_eq!(pixel(&studio.program(), 1, 1), BLUE); + }) + .await; + } + + /// A cut lands at once, even mid-fade, and an empty program is black. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn a_cut_ends_a_fade() { + local(async { + let mut studio = Studio::new(collection()).await; + let start = Instant::now(); + studio.render(start); + assert_eq!(pixel(&studio.program(), 1, 1), BLACK); + + studio.compositor.set_preview(Some("blue".into())); + studio.compositor.fade(Duration::from_secs(1), start); + studio.render(start + Duration::from_millis(500)); + studio.compositor.set_preview(Some("red".into())); + studio.compositor.cut(); + assert!(!studio.compositor.fading()); + studio.render(start + Duration::from_millis(600)); + assert_eq!(pixel(&studio.program(), 1, 1), RED); + }) + .await; + } + + /// A source with no new picture repeats its last one; a removed source + /// disappears. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn a_stalled_source_repeats_its_last_picture() { + local(async { + let mut studio = Studio::new(collection()).await; + studio.compositor.set_preview(Some("blue".into())); + studio.compositor.cut(); + studio.render(Instant::now()); + assert!(studio.sources.get("blue").unwrap().take().is_none(), "drawn once"); + studio.render(Instant::now()); + assert_eq!(pixel(&studio.program(), 1, 1), BLUE); + + studio.collection.sources.remove("blue"); + studio.collection.scenes.clear(); + studio.render(Instant::now()); + assert_eq!(pixel(&studio.program(), 1, 1), BLACK); + }) + .await; + } + + /// A slow encoder runs out of buffers and drops pictures rather than + /// queuing them, and a finished copy frees its buffer. + #[tokio::test] + #[ignore = "needs a Vulkan device; run by just rs render-sw"] + async fn readback_drops_past_its_depth() { + local(async { + let mut studio = Studio::new(collection()).await; + studio.render(Instant::now()); + let program = studio.compositor.program().clone(); + let mut held: Vec<_> = (0..Readback::DEPTH) + .map(|_| studio.readback.copy(&program, Timestamp::ZERO).unwrap()) + .collect(); + assert!(studio.readback.copy(&program, Timestamp::ZERO).is_none()); + + let frame = held.pop().unwrap().wait().unwrap(); + assert_eq!(frame.size(), moq_video::Size::new(8, 8)); + held.push(studio.readback.copy(&program, Timestamp::ZERO).unwrap()); + assert!(studio.readback.copy(&program, Timestamp::ZERO).is_none()); + }) + .await; + } +} diff --git a/rs/moq-scene/src/lib.rs b/rs/moq-scene/src/lib.rs new file mode 100644 index 00000000000..50e1a5338fc --- /dev/null +++ b/rs/moq-scene/src/lib.rs @@ -0,0 +1,85 @@ +//! Scenes, live sources, and a GPU compositor for a MoQ streaming studio. +//! +//! The UI-free half of `moq-studio`, so the same scenes can be composed by a +//! windowed app, a headless renderer, or a test: +//! +//! - [`Collection`] is the scene graph: a [`Canvas`], named [`Source`]s, and +//! [`Scene`]s placing them. It saves to a versioned TOML file that holds no +//! credentials. +//! - [`source::Sources`] runs each source: a camera, display, window, +//! microphone, or system-audio capture (with the `capture` feature), a still +//! image, a solid color, or a broadcast subscribed over MoQ. Pictures go to the +//! compositor and sound goes to a `moq_audio::bus::Bus`, the program audio. +//! - [`Compositor`] draws the scene in preview and the scene on program into +//! textures on the caller's `wgpu` device, cutting or fading between them. +//! - [`Readback`] copies the program back to the CPU for an encoder that cannot +//! take a GPU surface. +//! +//! Every picture is drawn on the program clock the caller ticks +//! [`Compositor::render`] with, from whatever each source delivered last: a +//! source that stalls repeats its last picture rather than holding up the +//! canvas. + +mod collection; +mod compositor; +mod readback; +pub mod source; + +pub use collection::{Audio, Canvas, Collection, Item, Rect, Rgba, Scene, Source, Transition}; +pub use compositor::{Compositor, FORMAT}; +pub use readback::{Pending, Readback}; + +/// The `wgpu` the compositor draws with: `moq_video`'s, so its textures and +/// the decoder's are the same type. +pub use moq_video::render::wgpu; + +/// Errors from loading scenes, opening sources, and drawing them. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// The scene file is not valid TOML of the expected shape. + #[error("scene file: {0}")] + Format(String), + + /// The scene file is from another version of the format. + #[error("scene file version {found} is not supported; this build reads version {supported}")] + Version { + /// The version the file names. + found: i64, + /// The version this build reads. + supported: u32, + }, + + /// The collection breaks a rule the types cannot express, such as an item + /// naming a source that does not exist. + #[error("invalid scene collection: {0}")] + Invalid(String), + + /// An image source could not be read or decoded. + #[error("image: {0}")] + Image(String), + + /// This build or platform cannot open the source. + #[error("unsupported source: {0}")] + Unsupported(String), + + /// The GPU failed while reading a picture back. + #[error("gpu: {0}")] + Gpu(String), + + /// Capturing, decoding, or drawing video failed. + #[error(transparent)] + Video(#[from] moq_video::Error), + + /// Capturing, decoding, or mixing audio failed. + #[error(transparent)] + Audio(#[from] moq_audio::Error), + + /// Reading a broadcast's catalog or media failed. + #[error(transparent)] + Mux(#[from] moq_mux::Error), + + /// The broadcast subscription failed. + #[error(transparent)] + Net(#[from] moq_net::Error), +} diff --git a/rs/moq-scene/src/readback.rs b/rs/moq-scene/src/readback.rs new file mode 100644 index 00000000000..0ba7c9a7202 --- /dev/null +++ b/rs/moq-scene/src/readback.rs @@ -0,0 +1,190 @@ +//! Copy canvas pictures back to the CPU, for an encoder that cannot take a GPU +//! surface. + +use std::sync::{Arc, Mutex}; + +use moq_net::Timestamp; +use moq_video::render::wgpu; + +use crate::Error; +use crate::collection::Canvas; + +/// Copies of a canvas texture on their way back to the CPU, through a small +/// pool of staging buffers. +/// +/// [`copy`](Self::copy) records the copy and returns at once; the [`Pending`] +/// it hands back waits for the GPU and converts the pixels, on whatever thread +/// the encoder runs on, so the canvas keeps drawing meanwhile. A copy that +/// finds every buffer still in flight is refused rather than queued, so an +/// encoder that falls behind drops pictures instead of adding latency. +pub struct Readback { + device: wgpu::Device, + queue: wgpu::Queue, + canvas: Canvas, + /// Bytes per row in a staging buffer, padded to the copy alignment. + row: u32, + pool: Arc>, +} + +/// Staging buffers not held by a [`Pending`]. +#[derive(Default)] +struct Pool { + free: Vec, + /// Buffers in existence, up to [`Readback::DEPTH`]. + allocated: usize, +} + +impl Readback { + /// Copies in flight at once: one being converted, one being copied, and one + /// spare so a slow conversion does not drop the very next picture. + pub const DEPTH: usize = 3; + + /// Read back `canvas`-sized textures from `device`. + pub fn new(device: &wgpu::Device, queue: &wgpu::Queue, canvas: Canvas) -> Self { + Self { + device: device.clone(), + queue: queue.clone(), + canvas, + row: (canvas.width * 4).next_multiple_of(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT), + pool: Arc::default(), + } + } + + /// Start copying `texture`, a canvas-sized [`FORMAT`](crate::compositor::FORMAT) + /// texture created with `COPY_SRC`, to be encoded at `timestamp`. + /// + /// `None` when every buffer is still held by an earlier [`Pending`]. + pub fn copy(&mut self, texture: &wgpu::Texture, timestamp: Timestamp) -> Option { + let mut pool = self.pool.lock().unwrap(); + let buffer = match pool.free.pop() { + Some(buffer) => buffer, + None if pool.allocated < Self::DEPTH => { + pool.allocated += 1; + self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("moq-scene readback"), + size: u64::from(self.row) * u64::from(self.canvas.height), + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }) + } + None => return None, + }; + drop(pool); + + let mut encoder = self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("moq-scene readback"), + }); + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &buffer, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(self.row), + rows_per_image: Some(self.canvas.height), + }, + }, + wgpu::Extent3d { + width: self.canvas.width, + height: self.canvas.height, + depth_or_array_layers: 1, + }, + ); + let submission = self.queue.submit([encoder.finish()]); + + Some(Pending { + device: self.device.clone(), + buffer: Some(buffer), + submission, + canvas: self.canvas, + row: self.row, + timestamp, + pool: self.pool.clone(), + mapping: false, + }) + } +} + +/// One copy in flight. Dropping it returns its buffer to the pool. +pub struct Pending { + device: wgpu::Device, + buffer: Option, + submission: wgpu::SubmissionIndex, + canvas: Canvas, + row: u32, + timestamp: Timestamp, + pool: Arc>, + /// Set while a mapping was requested and not yet undone, so a wait that + /// failed partway does not hand a mapped buffer to the next copy. + mapping: bool, +} + +impl Pending { + /// Block until the copy lands, then hand back the picture as a frame an + /// encoder takes. + pub fn wait(mut self) -> Result { + let rgba = self.rgba()?; + let surface = moq_video::Surface::rgba(&rgba, self.canvas.size())?; + Ok(moq_video::Frame::new(surface, self.timestamp)) + } + + /// Block until the copy lands, then hand back tightly packed RGBA rows. + pub fn rgba(&mut self) -> Result, Error> { + let buffer = self.buffer.as_ref().expect("held until drop"); + let (send, recv) = std::sync::mpsc::channel(); + self.mapping = true; + buffer.map_async(wgpu::MapMode::Read, .., move |result| { + let _ = send.send(result); + }); + self.device + .poll(wgpu::PollType::Wait { + submission_index: Some(self.submission.clone()), + timeout: None, + }) + .map_err(|err| Error::Gpu(format!("waiting for the readback: {err}")))?; + recv.recv() + .map_err(|_| Error::Gpu("the readback was dropped".into()))? + .map_err(|err| Error::Gpu(format!("mapping the readback: {err}")))?; + + let width = self.canvas.width as usize * 4; + let mut rgba = Vec::with_capacity(width * self.canvas.height as usize); + { + let mapped = buffer + .slice(..) + .get_mapped_range() + .map_err(|err| Error::Gpu(format!("reading the readback: {err}")))?; + for row in mapped.chunks_exact(self.row as usize) { + rgba.extend_from_slice(&row[..width]); + } + } + buffer.unmap(); + self.mapping = false; + Ok(rgba) + } + + /// When the picture is shown. + pub fn timestamp(&self) -> Timestamp { + self.timestamp + } +} + +impl Drop for Pending { + fn drop(&mut self) { + let Some(buffer) = self.buffer.take() else { + return; + }; + let mut pool = self.pool.lock().unwrap(); + // The pool only holds buffers the next copy can write. One a failed wait + // left mapped, or waiting on a mapping, is let go and replaced. + if self.mapping { + pool.allocated -= 1; + } else { + pool.free.push(buffer); + } + } +} diff --git a/rs/moq-scene/src/source.rs b/rs/moq-scene/src/source.rs new file mode 100644 index 00000000000..44fdde93fef --- /dev/null +++ b/rs/moq-scene/src/source.rs @@ -0,0 +1,605 @@ +//! Live sources: what each [`Source`] of a collection shows and plays right now. +//! +//! [`Sources::sync`] keeps one [`Feed`] per source and hands back a [`Driver`] +//! for each one it opens. The caller runs the drivers wherever suits it (a +//! capture driver is `!Send` on macOS); a driver stops once its feed is +//! dropped, which happens when the source is removed or edited. + +use std::collections::BTreeMap; +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; + +use moq_audio::bus::{Bus, Strip}; +use moq_mux::catalog::Stream; + +use crate::Error; +use crate::collection::{Collection, Source}; + +/// A source's newest picture, waiting for the compositor to draw it. +pub(crate) enum Picture { + /// A captured or decoded video frame. + Frame(moq_video::Frame), + /// Tightly packed RGBA with straight alpha, such as a still image. + Rgba { size: moq_video::Size, pixels: Vec }, +} + +/// The one picture a source holds for the compositor. A newer picture replaces +/// an undrawn one, so a slow canvas skips pictures instead of queuing them. +type Mailbox = Arc>>; + +/// One instance of a broadcast being decoded. +type Playing<'a> = Pin> + 'a>>; + +/// What a running source is doing. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum Status { + /// Opening its device, file, or subscription. + Starting, + /// Producing pictures or sound. + Live, + /// Waiting for its broadcast to be announced. + Offline, + /// Stopped, with the reason. A broadcast source keeps waiting for the + /// broadcast to come back; every other source stays stopped until edited. + Failed(String), +} + +/// What every source shares. +#[derive(Clone)] +#[non_exhaustive] +pub struct Context { + /// The program audio that microphones, system audio, and broadcasts feed. + pub bus: Bus, + /// Where broadcast sources are announced: the studio's relay connection. + pub origin: moq_net::origin::Consumer, + /// What a relative image path is relative to: the scene file's directory. + pub dir: PathBuf, +} + +impl Context { + /// Sources feeding `bus`, subscribing through `origin`, with images + /// relative to `dir`. + pub fn new(bus: Bus, origin: moq_net::origin::Consumer, dir: PathBuf) -> Self { + Self { bus, origin, dir } + } +} + +/// The running sources of a collection. +pub struct Sources { + context: Context, + feeds: BTreeMap, +} + +impl Sources { + /// No sources running yet; [`sync`](Self::sync) starts them. + pub fn new(context: Context) -> Self { + Self { + context, + feeds: BTreeMap::new(), + } + } + + /// What every source shares. + pub fn context(&self) -> &Context { + &self.context + } + + /// Run exactly the sources `collection` has: start new and edited ones and + /// stop removed and edited ones. Returns a driver for each source started, + /// which does nothing until run. + /// + /// Also applies the collection's gain and mute to every audio strip, which + /// never restarts a source. + pub fn sync(&mut self, collection: &Collection) -> Vec { + self.feeds + .retain(|name, feed| collection.sources.get(name) == Some(&feed.source)); + + let mut drivers = Vec::new(); + for (name, source) in &collection.sources { + if !self.feeds.contains_key(name) { + let (feed, driver) = open(name, source, &self.context); + self.feeds.insert(name.clone(), feed); + drivers.push(driver); + } + + if let Some(strip) = self.feeds[name].strip() { + let audio = collection.audio.get(name).copied().unwrap_or_default(); + strip.set_gain(audio.gain); + strip.set_muted(audio.muted); + } + } + drivers + } + + /// The running source called `name`. + pub fn get(&self, name: &str) -> Option<&Feed> { + self.feeds.get(name) + } + + /// Every running source, by name. + pub fn iter(&self) -> impl Iterator { + self.feeds.iter().map(|(name, feed)| (name.as_str(), feed)) + } +} + +/// One running source. Dropping it stops its [`Driver`]. +pub struct Feed { + source: Source, + picture: Mailbox, + strip: Option, + status: kio::Consumer, +} + +impl Feed { + /// What this feed was opened for. + pub fn source(&self) -> &Source { + &self.source + } + + /// What the source is doing. + pub fn status(&self) -> Status { + self.status.read().clone() + } + + /// The source's gain, mute, and meter in the program audio, if it has sound. + pub fn strip(&self) -> Option<&Strip> { + self.strip.as_ref() + } + + /// The newest picture not yet drawn. + pub(crate) fn take(&self) -> Option { + self.picture.lock().unwrap().take() + } +} + +/// The work behind one [`Feed`]: opening the source and delivering what it +/// produces. Built by [`Sources::sync`]. +pub struct Driver { + name: String, + source: Source, + context: Context, + picture: Mailbox, + strip: Option, + status: kio::Producer, +} + +impl Driver { + /// The source's name in the collection. + pub fn name(&self) -> &str { + &self.name + } + + /// Run the source until its [`Feed`] is dropped. + /// + /// A failure is reported through [`Feed::status`] rather than returned. On + /// macOS a capture source's future is `!Send`, so run it on a local task. + pub async fn run(self) { + let status = self.status.clone(); + let result = tokio::select! { + _ = status.unused() => return, + result = self.produce() => result, + }; + let status = match result { + Ok(()) => Status::Failed("the source ended".into()), + Err(err) => Status::Failed(err.to_string()), + }; + tracing::warn!(source = %self.name, ?status, "source stopped"); + set(&self.status, status); + } + + async fn produce(&self) -> Result<(), Error> { + match &self.source { + Source::Color { color } => { + self.show(Picture::Rgba { + size: moq_video::Size::new(1, 1), + pixels: color.0.to_vec(), + }); + std::future::pending().await + } + Source::Image { path } => { + let picture = image(&self.context.dir.join(path))?; + self.show(picture); + std::future::pending().await + } + Source::Broadcast { path } => self.broadcast(path).await, + device => self.device(device).await, + } + } + + fn show(&self, picture: Picture) { + *self.picture.lock().unwrap() = Some(picture); + set(&self.status, Status::Live); + } + + /// Capture a camera, display, window, or audio input. + #[cfg(feature = "capture")] + async fn device(&self, source: &Source) -> Result<(), Error> { + use moq_audio::capture::Source as Audio; + use moq_video::capture::Source as Video; + + let video = match source { + Source::Camera { id } => Video::Camera(id.clone()), + Source::Display { id } => Video::Display(id.clone()), + Source::Window { id } => Video::Window(id.clone()), + Source::Microphone { id } => return self.listen(Audio::Microphone(id.clone())).await, + Source::SystemAudio => return self.listen(Audio::System).await, + Source::Color { .. } | Source::Image { .. } | Source::Broadcast { .. } => { + unreachable!("not a device source") + } + }; + + let config = moq_video::capture::Config { + source: video, + ..Default::default() + }; + let mut stream = moq_video::capture::open(&config).await?; + while let Some(frame) = stream.read().await? { + self.show(Picture::Frame(frame)); + } + Ok(()) + } + + #[cfg(feature = "capture")] + async fn listen(&self, source: moq_audio::capture::Source) -> Result<(), Error> { + let strip = self.strip.clone().expect("audio sources have a strip"); + let mut config = moq_audio::capture::Config::default(); + config.source = source; + let capture = self.context.bus.capture(config, strip); + set(&self.status, Status::Live); + Ok(capture.run().await?) + } + + #[cfg(not(feature = "capture"))] + async fn device(&self, _: &Source) -> Result<(), Error> { + Err(Error::Unsupported( + "this build cannot capture devices (enable the `capture` feature)".into(), + )) + } + + /// Follow the broadcast at `path` for as long as the feed lives, playing + /// each instance of it that comes online. + async fn broadcast(&self, path: &str) -> Result<(), Error> { + let mut instances = moq_mux::follow::Instances::new(&self.context.origin, path, None)?; + set(&self.status, Status::Offline); + + let mut playing: Option> = None; + loop { + tokio::select! { + event = instances.next() => match event { + Ok(moq_mux::follow::Event::Start(broadcast)) => { + set(&self.status, Status::Live); + playing = Some(Box::pin(self.play(broadcast, path))); + } + Ok(moq_mux::follow::Event::End) => { + playing = None; + set(&self.status, Status::Offline); + } + Err(err) => return Err(err.into()), + }, + result = async { playing.as_mut().expect("guarded").await }, if playing.is_some() => { + playing = None; + set(&self.status, match result { + Ok(()) => Status::Offline, + Err(err) => Status::Failed(err.to_string()), + }); + } + } + } + } + + /// Decode one instance of a broadcast: the first video and audio renditions + /// its first catalog offers, until both end. + async fn play(&self, broadcast: moq_net::broadcast::Consumer, path: &str) -> Result<(), Error> { + let format = moq_mux::catalog::CatalogFormat::detect(path).unwrap_or_default(); + let mut catalogs = moq_mux::catalog::Consumer::<()>::new(&broadcast, format).await?; + let Some(catalog) = catalogs.next().await? else { + return Ok(()); + }; + + // Renditions served from another broadcast would need that broadcast's + // own subscription; the studio's sources are single broadcasts. + let video = catalog + .video + .enabled() + .find(|(_, config)| config.broadcast.is_none()) + .map(|(name, config)| (name.clone(), config.clone())); + let audio = catalog + .audio + .enabled() + .find(|(_, config)| config.broadcast.is_none()) + .map(|(name, config)| (name.clone(), config.clone())); + + let video = async { + let Some((name, config)) = video else { + return Ok(()); + }; + let mut options = moq_video::decode::Options::new(); + options.start = moq_video::decode::Start::Latest; + let mut consumer = moq_video::decode::Consumer::new(&broadcast, &config, name, options).await?; + while let Some(read) = consumer.read().await? { + if let moq_video::decode::Read::Frame(frame) = read { + self.show(Picture::Frame(frame)); + } + } + Ok::<_, Error>(()) + }; + + let audio = async { + let (Some((name, config)), Some(strip)) = (audio, self.strip.clone()) else { + return Ok(()); + }; + let bus = self.context.bus.config().clone(); + let mut options = moq_audio::decode::Options::new(); + options.start = moq_audio::decode::Start::Latest; + options.output.format = moq_audio::Format::F32; + options.output.sample_rate = Some(bus.sample_rate); + options.output.layout = Some(bus.layout); + let mut consumer = moq_audio::decode::Consumer::new(&broadcast, &config, name, options).await?; + + let mut input = moq_audio::encode::Input::new(bus.sample_rate, bus.layout); + input.format = moq_audio::Format::F32; + let mut channel = self.context.bus.channel(input, strip)?; + // Gaps and encoder boundaries need nothing: the bus plays what arrives + // on its own clock, and silence fills whatever does not. + while let Some(read) = consumer.read().await? { + if let moq_audio::decode::Read::Frame(frame) = read { + channel.write(&frame.data)?; + } + } + Ok::<_, Error>(()) + }; + + tokio::try_join!(video, audio)?; + Ok(()) + } +} + +/// Start the feed for `source`, and the driver that fills it. +fn open(name: &str, source: &Source, context: &Context) -> (Feed, Driver) { + let picture = Mailbox::default(); + let strip = source.has_audio().then(Strip::default); + let status = kio::Producer::new(Status::Starting); + + let feed = Feed { + source: source.clone(), + picture: picture.clone(), + strip: strip.clone(), + status: status.consume(), + }; + let driver = Driver { + name: name.to_string(), + source: source.clone(), + context: context.clone(), + picture, + strip, + status, + }; + (feed, driver) +} + +fn set(status: &kio::Producer, value: Status) { + if let Ok(mut status) = status.write() { + *status = value; + } +} + +/// Decode a still image to straight-alpha RGBA. +fn image(path: &Path) -> Result { + let bytes = std::fs::read(path).map_err(|err| Error::Image(format!("{}: {err}", path.display())))?; + let image = image::load_from_memory(&bytes) + .map_err(|err| Error::Image(format!("{}: {err}", path.display())))? + .into_rgba8(); + Ok(Picture::Rgba { + size: moq_video::Size::new(image.width(), image.height()), + pixels: image.into_raw(), + }) +} + +/// Helpers for tests across the crate. +#[cfg(test)] +pub(crate) mod test { + use super::*; + + /// A context with no relay behind it, so broadcasts stay offline while the + /// returned origin lives. Call from a [`tokio::task::LocalSet`]. + pub(crate) fn context(dir: &Path) -> (Context, moq_net::origin::Producer) { + let (origin, driver) = moq_net::origin::Producer::new(Default::default()); + tokio::task::spawn_local(moq_net::time::run(driver)); + let bus = Bus::new(Default::default()).unwrap(); + (Context::new(bus, origin.consume(), dir.to_path_buf()), origin) + } + + /// Run `drivers` on the local task set, and wait until every source in + /// `sources` has left [`Status::Starting`]. + pub(crate) async fn start(sources: &Sources, drivers: Vec) { + for driver in drivers { + tokio::task::spawn_local(driver.run()); + } + for _ in 0..1000 { + if sources.iter().all(|(_, feed)| feed.status() != Status::Starting) { + return; + } + tokio::task::yield_now().await; + } + panic!("sources never started"); + } + + /// Run `f` on a local task set, as drivers need. + pub(crate) async fn local(f: F) -> F::Output { + tokio::task::LocalSet::new().run_until(f).await + } +} + +#[cfg(test)] +mod tests { + use super::test::{context, local, start}; + use super::*; + use crate::collection::{Audio, Rgba}; + + fn collection() -> Collection { + let mut collection = Collection::new(); + collection.sources.insert( + "red".into(), + Source::Color { + color: Rgba([255, 0, 0, 255]), + }, + ); + collection.sources.insert( + "guest".into(), + Source::Broadcast { + path: "room/guest".into(), + }, + ); + collection + } + + #[tokio::test] + async fn a_color_shows_one_pixel() { + local(async { + let dir = tempfile::tempdir().unwrap(); + let (context, _origin) = context(dir.path()); + let mut sources = Sources::new(context); + let drivers = sources.sync(&collection()); + assert_eq!(drivers.len(), 2); + start(&sources, drivers).await; + + let red = sources.get("red").unwrap(); + assert_eq!(red.status(), Status::Live); + assert!(red.strip().is_none(), "a color is silent"); + let Some(Picture::Rgba { size, pixels }) = red.take() else { + panic!("a color shows a picture"); + }; + assert_eq!(size, moq_video::Size::new(1, 1)); + assert_eq!(pixels, [255, 0, 0, 255]); + assert!(red.take().is_none(), "taken once"); + + let guest = sources.get("guest").unwrap(); + assert_eq!(guest.status(), Status::Offline, "nothing is announced"); + assert!(guest.strip().is_some(), "a broadcast feeds the bus"); + }) + .await; + } + + /// Editing a source restarts it, removing one stops it, and an untouched one + /// keeps running. Gain changes reach the strip without restarting anything. + #[tokio::test] + async fn sync_restarts_only_what_changed() { + local(async { + let dir = tempfile::tempdir().unwrap(); + let (context, _origin) = context(dir.path()); + let mut sources = Sources::new(context); + let mut collection = collection(); + let drivers = sources.sync(&collection); + start(&sources, drivers).await; + let guest = sources.get("guest").unwrap().strip().unwrap().clone(); + + collection.audio.insert("guest".into(), Audio { gain: 0.25, muted: true }); + assert!(sources.sync(&collection).is_empty()); + assert_eq!(guest.gain(), 0.25); + assert!(guest.muted()); + + collection.sources.insert( + "red".into(), + Source::Color { + color: Rgba([0, 0, 255, 255]), + }, + ); + let drivers = sources.sync(&collection); + assert_eq!(drivers.iter().map(Driver::name).collect::>(), ["red"]); + start(&sources, drivers).await; + let Some(Picture::Rgba { pixels, .. }) = sources.get("red").unwrap().take() else { + panic!("the edited color shows"); + }; + assert_eq!(pixels, [0, 0, 255, 255]); + + collection.sources.remove("red"); + assert!(sources.sync(&collection).is_empty()); + assert!(sources.get("red").is_none()); + }) + .await; + } + + /// Dropping a feed stops its driver, which is what releases a device. + #[tokio::test] + async fn a_dropped_feed_stops_its_driver() { + local(async { + let dir = tempfile::tempdir().unwrap(); + let (context, _origin) = context(dir.path()); + let mut sources = Sources::new(context); + let mut drivers = sources.sync(&collection()); + let driver = drivers.pop().unwrap(); + let running = tokio::task::spawn_local(driver.run()); + tokio::task::yield_now().await; + assert!(!running.is_finished()); + + sources.sync(&Collection::new()); + running.await.unwrap(); + }) + .await; + } + + #[tokio::test] + async fn an_image_loads_relative_to_the_scene_file() { + local(async { + let dir = tempfile::tempdir().unwrap(); + let mut png = image::RgbaImage::new(3, 2); + png.put_pixel(2, 1, image::Rgba([1, 2, 3, 4])); + png.save(dir.path().join("logo.png")).unwrap(); + + let mut collection = Collection::new(); + collection.sources.insert( + "logo".into(), + Source::Image { + path: "logo.png".into(), + }, + ); + collection.sources.insert( + "missing".into(), + Source::Image { + path: "missing.png".into(), + }, + ); + let (context, _origin) = context(dir.path()); + let mut sources = Sources::new(context); + let drivers = sources.sync(&collection); + start(&sources, drivers).await; + + let Some(Picture::Rgba { size, pixels }) = sources.get("logo").unwrap().take() else { + panic!("an image shows a picture"); + }; + assert_eq!(size, moq_video::Size::new(3, 2)); + assert_eq!(pixels[pixels.len() - 4..], [1, 2, 3, 4]); + + let Status::Failed(reason) = sources.get("missing").unwrap().status() else { + panic!("a missing image fails"); + }; + assert!(reason.contains("missing.png"), "{reason}"); + }) + .await; + } + + #[cfg(not(feature = "capture"))] + #[tokio::test] + async fn a_build_without_capture_says_so() { + local(async { + let dir = tempfile::tempdir().unwrap(); + let mut collection = Collection::new(); + collection.sources.insert("cam".into(), Source::Camera { id: None }); + collection.sources.insert("mic".into(), Source::Microphone { id: None }); + let (context, _origin) = context(dir.path()); + let mut sources = Sources::new(context); + let drivers = sources.sync(&collection); + start(&sources, drivers).await; + for name in ["cam", "mic"] { + let Status::Failed(reason) = sources.get(name).unwrap().status() else { + panic!("{name} cannot capture"); + }; + assert!(reason.contains("capture"), "{reason}"); + } + }) + .await; + } +} diff --git a/rs/moq-studio/Cargo.toml b/rs/moq-studio/Cargo.toml new file mode 100644 index 00000000000..4fa312caeaf --- /dev/null +++ b/rs/moq-studio/Cargo.toml @@ -0,0 +1,62 @@ +[package] +# Unpublished until the packaging quest decides how the studio ships. +publish = false +name = "moq-studio" +description = "A native streaming studio that composites scenes on the GPU and goes live over MoQ" +authors = ["Luke Curley "] +repository = "https://github.com/moq-dev/moq" +license = "MIT OR Apache-2.0" + +version = "0.1.0" +edition = "2024" +# egui 0.36 and egui_dock 0.21 need 1.92 and 1.95. This is an application, so +# the bump stays here rather than raising the libraries' floor. +rust-version = "1.95" + +keywords = ["quic", "streaming", "studio", "live", "media"] +categories = ["multimedia", "multimedia::video", "gui"] + +[features] +default = ["quic", "websocket", "nvidia"] +quic = ["moq-tokio/quic"] +websocket = ["moq-tokio/websocket"] +# Camera, display, window, microphone, and system-audio sources. Off by default +# like moq-cli's `capture`: on Linux the microphone links libasound at build +# time. Enable with `cargo run -p moq-studio --features capture,openh264`. +capture = ["moq-scene/capture"] +# NVIDIA hardware codecs (NVENC + NVDEC) on Linux, as in moq-cli. +nvidia = ["moq-video/nvidia"] +# The vendored OpenH264 software H.264 fallback, opt-in like moq-video's. +openh264 = ["moq-video/openh264"] +# Intel/AMD VAAPI hardware codecs (Linux); its bindgen needs libclang. +vaapi = ["moq-video/vaapi"] +# Wayland screen capture through the portal and PipeWire cameras and audio. +pipewire = ["capture", "moq-video/pipewire", "moq-audio/pipewire"] + +[dependencies] +anyhow = { workspace = true, features = ["backtrace"] } +# The window, its event loop, and the egui integration. `wgpu` is the renderer +# the compositor shares a device with; no glow, no system font discovery. +eframe = { version = "0.36", default-features = false, features = ["default_fonts", "wayland", "wgpu", "x11"] } +egui = "0.36" +# Docked, rearrangeable panels. 0.21 is the newest release and pins egui 0.36. +egui_dock = "0.21" +hang = { workspace = true } +moq-audio = { workspace = true, features = ["bus"] } +moq-clock = { workspace = true } +moq-mux = { workspace = true } +moq-net = { workspace = true } +moq-scene = { workspace = true } +moq-tokio = { workspace = true, default-features = false, features = ["aws-lc-rs", "net"] } +moq-video = { workspace = true } +pollster = { workspace = true } +tokio = { workspace = true, features = ["full"] } +tracing = { workspace = true } +usage-rs = { workspace = true, features = ["completions", "config"] } + +[dev-dependencies] +moq-relay = { path = "../moq-relay", default-features = false, features = ["test-support"] } +tempfile = { workspace = true } + +[lints] +workspace = true diff --git a/rs/moq-studio/src/main.rs b/rs/moq-studio/src/main.rs new file mode 100644 index 00000000000..f328e4d9d04 --- /dev/null +++ b/rs/moq-studio/src/main.rs @@ -0,0 +1 @@ +fn main() {} From c5aeb63866a03964e130fdc980dbfe038c7c4bfc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 10:11:46 -0700 Subject: [PATCH 03/15] feat(moq-studio): a docked egui studio that publishes its program One wgpu device serves the sources, the compositor, and the window. A program clock draws the canvas and mixes the bus, and the program is read back and encoded only while a viewer watches. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 3 +- rs/justfile | 9 +- rs/moq-scene/src/source.rs | 6 +- rs/moq-studio/Cargo.toml | 3 +- rs/moq-studio/src/engine.rs | 589 +++++++++++++++++++++++++ rs/moq-studio/src/main.rs | 123 +++++- rs/moq-studio/src/ui.rs | 858 ++++++++++++++++++++++++++++++++++++ sh/rs/select.sh | 2 +- 8 files changed, 1582 insertions(+), 11 deletions(-) create mode 100644 rs/moq-studio/src/engine.rs create mode 100644 rs/moq-studio/src/ui.rs diff --git a/Cargo.lock b/Cargo.lock index 6180dbf54a0..6cb0f3c5960 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5829,10 +5829,10 @@ name = "moq-studio" version = "0.1.0" dependencies = [ "anyhow", + "bytes", "eframe", "egui", "egui_dock", - "hang", "moq-audio", "moq-clock", "moq-mux", @@ -5845,6 +5845,7 @@ dependencies = [ "tempfile", "tokio", "tracing", + "url", "usage-rs", ] diff --git a/rs/justfile b/rs/justfile index 0538bd4e6e9..de6fc0ab188 100644 --- a/rs/justfile +++ b/rs/justfile @@ -148,14 +148,14 @@ macos-test: # Compile and lint the capture and playback features, which the default check skips. [positional-arguments] capture *args: - bash sh/local.sh cargo clippy --locked -p moq-video -p moq-audio --all-targets --features moq-video/capture,moq-audio/capture,moq-audio/playback "$@" -- -D warnings + bash sh/local.sh cargo clippy --locked -p moq-video -p moq-audio -p moq-scene -p moq-studio --all-targets --features moq-video/capture,moq-audio/capture,moq-audio/playback,moq-scene/capture,moq-studio/capture "$@" -- -D warnings # The capture and playback tests drive synthetic devices, so they need no hardware. # Test the capture and playback features, which the default test pass skips. [positional-arguments] capture-test *args: - just rs test -p moq-video -p moq-audio --features moq-video/capture,moq-audio/capture,moq-audio/playback "$@" + just rs test -p moq-video -p moq-audio -p moq-scene -p moq-studio --features moq-video/capture,moq-audio/capture,moq-audio/playback,moq-scene/capture,moq-studio/capture "$@" # Compare SHM and GetImage through production capture on the current X11 display. x11-bench: @@ -346,7 +346,8 @@ cli-export *args: quic-stress: just rs test -p moq-tokio --lib --run-ignored only -E 'test(/^quic::driver::tests::stress_/)' -# Render CPU frames and composite scenes through wgpu on Mesa's software Vulkan driver, so no GPU is needed. +# Render CPU frames, composite scenes, and publish a studio program through wgpu on +# Mesa's software Vulkan driver, so no GPU is needed. render-sw: #!/usr/bin/env bash set -euo pipefail @@ -358,6 +359,8 @@ render-sw: -E 'test(/^render::renderer::tests::(the_declared_color_space_beats_the_size_guess|cpu_frames_survive_the_round_trip|config_size_overrides_the_frame_size)$/)' LD_LIBRARY_PATH="$loader/lib" VK_DRIVER_FILES="${icd[0]}" \ just rs test -p moq-scene --lib --run-ignored only -E 'test(/^compositor::tests::/)' + LD_LIBRARY_PATH="$loader/lib" VK_DRIVER_FILES="${icd[0]}" \ + just rs test -p moq-studio --features openh264 --run-ignored only -E 'test(=engine::tests::the_program_reaches_a_relay)' # Run the ignored mDNS tests, which multicast on the host's network interfaces. multicast: diff --git a/rs/moq-scene/src/source.rs b/rs/moq-scene/src/source.rs index 44fdde93fef..f28a6556b75 100644 --- a/rs/moq-scene/src/source.rs +++ b/rs/moq-scene/src/source.rs @@ -231,10 +231,8 @@ impl Driver { } }; - let config = moq_video::capture::Config { - source: video, - ..Default::default() - }; + let mut config = moq_video::capture::Config::default(); + config.source = video; let mut stream = moq_video::capture::open(&config).await?; while let Some(frame) = stream.read().await? { self.show(Picture::Frame(frame)); diff --git a/rs/moq-studio/Cargo.toml b/rs/moq-studio/Cargo.toml index 4fa312caeaf..d8d376c7b61 100644 --- a/rs/moq-studio/Cargo.toml +++ b/rs/moq-studio/Cargo.toml @@ -35,13 +35,13 @@ pipewire = ["capture", "moq-video/pipewire", "moq-audio/pipewire"] [dependencies] anyhow = { workspace = true, features = ["backtrace"] } +bytes = { workspace = true } # The window, its event loop, and the egui integration. `wgpu` is the renderer # the compositor shares a device with; no glow, no system font discovery. eframe = { version = "0.36", default-features = false, features = ["default_fonts", "wayland", "wgpu", "x11"] } egui = "0.36" # Docked, rearrangeable panels. 0.21 is the newest release and pins egui 0.36. egui_dock = "0.21" -hang = { workspace = true } moq-audio = { workspace = true, features = ["bus"] } moq-clock = { workspace = true } moq-mux = { workspace = true } @@ -52,6 +52,7 @@ moq-video = { workspace = true } pollster = { workspace = true } tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } +url = { workspace = true } usage-rs = { workspace = true, features = ["completions", "config"] } [dev-dependencies] diff --git a/rs/moq-studio/src/engine.rs b/rs/moq-studio/src/engine.rs new file mode 100644 index 00000000000..a78dab9c56c --- /dev/null +++ b/rs/moq-studio/src/engine.rs @@ -0,0 +1,589 @@ +//! Everything behind the window: the GPU, the sources, the program clock that +//! draws the canvas and mixes the audio, and the broadcast it publishes. +//! +//! Threads: +//! - **program** draws preview and program once per canvas frame and hands the +//! program to the encoder, stamped with the program clock. +//! - **audio** reads the program bus on the same clock and publishes it. +//! - **encode** reads the program back from the GPU and encodes it. +//! - **source-***: one per running source, since a capture stream may not +//! leave the thread that opened it. +//! - The UI runs on the main thread and edits [`State`] under its lock. + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, mpsc}; +use std::time::Duration; + +use anyhow::Context; +use bytes::Bytes; +use moq_clock::Instant; +use moq_scene::source::{self, Driver, Sources}; +use moq_scene::{Collection, Compositor, Pending, Readback, wgpu}; + +/// The GPU everything shares: sources upload to it, the compositor draws on +/// it, the window presents from it, and the encoder reads back from it. +#[derive(Clone)] +pub struct Gpu { + pub instance: wgpu::Instance, + pub adapter: wgpu::Adapter, + pub device: wgpu::Device, + pub queue: wgpu::Queue, +} + +impl Gpu { + pub async fn new() -> anyhow::Result { + let instance = wgpu::Instance::default(); + let adapter = instance + .request_adapter(&wgpu::RequestAdapterOptions { + power_preference: wgpu::PowerPreference::HighPerformance, + ..Default::default() + }) + .await + .context("no GPU adapter")?; + // Lets a hardware-decoded or captured DMA-BUF reach the compositor + // without a copy, where the driver can import one. + let features = adapter.features() & wgpu::Features::VULKAN_EXTERNAL_MEMORY_DMA_BUF; + let (device, queue) = adapter + .request_device(&wgpu::DeviceDescriptor { + label: Some("moq-studio"), + required_features: features, + ..Default::default() + }) + .await + .context("cannot open the GPU")?; + Ok(Self { + instance, + adapter, + device, + queue, + }) + } +} + +/// What the UI edits and the program clock draws. +pub struct State { + /// The scene file, saved to and reloaded from. + pub path: PathBuf, + pub collection: Collection, + pub sources: Sources, + pub compositor: Compositor, + /// Edited since the sources were last synced. + stale: bool, + /// Edited since the last save or load. + unsaved: bool, +} + +impl State { + /// The collection, for an edit: the sources follow it on the next frame. + pub fn edit(&mut self) -> &mut Collection { + self.stale = true; + self.unsaved = true; + &mut self.collection + } + + pub fn unsaved(&self) -> bool { + self.unsaved + } + + pub fn save(&mut self) -> anyhow::Result<()> { + let text = self.collection.to_toml()?; + std::fs::write(&self.path, text).with_context(|| format!("writing {}", self.path.display()))?; + self.unsaved = false; + Ok(()) + } + + /// Replace the collection with what the scene file holds now. + pub fn reload(&mut self) -> anyhow::Result<()> { + let collection = load(&self.path)?; + anyhow::ensure!( + collection.canvas == self.collection.canvas, + "the canvas changed; restart the studio to use {}x{} at {} fps", + collection.canvas.width, + collection.canvas.height, + collection.canvas.framerate + ); + // A scene in preview or on program that the file no longer has draws + // black until another is picked. + self.collection = collection; + self.stale = true; + self.unsaved = false; + Ok(()) + } +} + +/// Read a scene file, or start a new collection if there is none yet. +pub fn load(path: &Path) -> anyhow::Result { + match std::fs::read_to_string(path) { + Ok(text) => Collection::from_toml(&text).with_context(|| format!("loading {}", path.display())), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(Collection::new()), + Err(err) => Err(err).with_context(|| format!("reading {}", path.display())), + } +} + +/// How the output is doing, for the stream panel. +#[derive(Default)] +pub struct Stats { + /// Canvas frames drawn. + pub rendered: AtomicU64, + /// Program frames copied back to the CPU for the encoder. + pub read_back: AtomicU64, + /// Microseconds the last readback waited on the GPU and converted pixels. + pub readback_us: AtomicU64, + /// Program frames not encoded because the encoder still held every buffer. + pub dropped: AtomicU64, + /// Program frames encoded. + pub encoded: AtomicU64, + /// Microseconds the last encode took. + pub encode_us: AtomicU64, + /// Whether any viewer is subscribed to the video. + pub watched: AtomicBool, + /// The encoder backend, once one opened. + pub encoder: Mutex>, + /// Why the output stopped, if it did. + pub error: Mutex>, +} + +impl Stats { + fn fail(&self, err: impl std::fmt::Display) { + tracing::error!(%err, "output stopped"); + *self.error.lock().unwrap() = Some(err.to_string()); + } +} + +/// Where the program goes. +pub struct Output { + /// The broadcast path on the relay. + pub path: String, + /// The relay connection, which reconnects on its own. + pub connection: moq_tokio::Connection, + publication: moq_net::origin::Publication, +} + +impl Drop for Output { + fn drop(&mut self) { + self.publication.close(); + } +} + +/// The running studio. +pub struct Engine { + pub gpu: Gpu, + pub state: Mutex, + pub stats: Stats, + pub bus: moq_audio::bus::Bus, + /// `None` when running without a relay. + pub output: Option, + /// Keeps broadcast sources' origin open when no connection holds it. + _origin: Option, +} + +/// The audio the program publishes. +const AUDIO_BITRATE: moq_net::bandwidth::Rate = moq_net::bandwidth::Rate::from_kbps(128); + +/// How often the audio thread reads the bus. Short enough that a read is never +/// close to the bus's buffering, long enough to cost nothing. +const AUDIO_PERIOD: Duration = Duration::from_millis(10); + +impl Engine { + /// Open the collection at `path` on `gpu` and start the program clock. + /// With `connect`, publish the program at `broadcast` on that relay. + /// + /// Call inside the tokio runtime that serves the relay connection. + pub async fn start( + gpu: Gpu, + path: PathBuf, + collection: Collection, + connect: Option<(moq_tokio::Client, url::Url)>, + broadcast: &str, + ) -> anyhow::Result> { + let bus = moq_audio::bus::Bus::new(Default::default())?; + let subscribe = moq_tokio::origin::spawn(); + let dir = path.parent().map(Path::to_path_buf).unwrap_or_default(); + let sources = Sources::new(source::Context::new(bus.clone(), subscribe.consume(), dir)); + let mut compositor = Compositor::new(&gpu.device, &gpu.queue, collection.canvas)?; + compositor.set_preview(collection.scenes.first().map(|scene| scene.name.clone())); + compositor.cut(); + let canvas = collection.canvas; + + let (output, origin, publish) = match connect { + Some((client, url)) => { + let (output, publish) = publish(client, url, subscribe, broadcast, canvas).await?; + (Some(output), None, Some(publish)) + } + None => (None, Some(subscribe), None), + }; + + let engine = Arc::new(Self { + state: Mutex::new(State { + path, + collection, + sources, + compositor, + stale: true, + unsaved: false, + }), + gpu, + stats: Stats::default(), + bus, + output, + _origin: origin, + }); + + let (video, audio) = match publish { + Some(Publish { video, audio, config }) => { + let (frames, pending) = mpsc::sync_channel(Readback::DEPTH); + tokio::spawn(watch(video.demand(), engine.clone())); + let encoder = engine.clone(); + std::thread::Builder::new() + .name("encode".into()) + .spawn(move || encode(&encoder, pending, video, config))?; + (Some(frames), Some(audio)) + } + None => (None, None), + }; + + let start = Instant::now(); + let program = engine.clone(); + std::thread::Builder::new() + .name("program".into()) + .spawn(move || clocked(program.run_program(start, video)))?; + let mix = engine.clone(); + std::thread::Builder::new() + .name("audio".into()) + .spawn(move || clocked(mix.run_audio(start, audio)))?; + + Ok(engine) + } + + /// Draw a canvas frame per tick of the program clock, and hand the program + /// to the encoder while anyone watches. + async fn run_program(&self, start: Instant, frames: Option>) { + let canvas = self.state.lock().unwrap().collection.canvas; + let mut readback = Readback::new(&self.gpu.device, &self.gpu.queue, canvas); + let period = Duration::from_secs(1) / canvas.framerate; + let mut clock = moq_clock::interval_at(start, period); + clock.set_missed_tick_behavior(moq_clock::MissedTickBehavior::Skip); + + loop { + let tick = clock.tick().await; + let program = { + let mut guard = self.state.lock().unwrap(); + let state = &mut *guard; + if std::mem::take(&mut state.stale) { + for driver in state.sources.sync(&state.collection) { + spawn(driver); + } + } + if let Err(err) = state.compositor.render(&state.collection, &state.sources, tick) { + tracing::error!(%err, "cannot draw the canvas"); + continue; + } + state.compositor.program().clone() + }; + self.stats.rendered.fetch_add(1, Ordering::Relaxed); + + let Some(frames) = &frames else { continue }; + if !self.stats.watched.load(Ordering::Relaxed) { + continue; + } + let Ok(timestamp) = moq_net::Timestamp::try_from(tick.saturating_duration_since(start)) else { + continue; + }; + match readback.copy(&program, timestamp) { + // The channel holds as many as the pool, so this never blocks. + Some(pending) => { + if frames.send(pending).is_err() { + return; + } + } + None => { + self.stats.dropped.fetch_add(1, Ordering::Relaxed); + } + } + } + } + + /// Read the program bus as the program clock advances, and publish it. + /// + /// Runs without a relay too, so the meters move and sources never back up. + async fn run_audio(&self, start: Instant, mut producer: Option) { + let config = self.bus.config().clone(); + let rate = u128::from(config.sample_rate); + let channels = config.layout.channels() as usize; + let mut clock = moq_clock::interval_at(start, AUDIO_PERIOD); + clock.set_missed_tick_behavior(moq_clock::MissedTickBehavior::Skip); + let mut samples = Vec::new(); + let mut written: u128 = 0; + + loop { + let tick = clock.tick().await; + // Sample-exact against the program clock, so audio and video stay + // aligned however the ticks jitter. + let due = tick.saturating_duration_since(start).as_nanos() * rate / 1_000_000_000; + let frames = (due - written) as usize; + if frames == 0 { + continue; + } + samples.resize(frames * channels, 0.0); + self.bus.read(&mut samples); + let timestamp = moq_net::Timestamp::from_micros((written * 1_000_000 / rate) as u64); + written = due; + + let Some(publish) = producer.as_mut() else { continue }; + let Ok(timestamp) = timestamp else { continue }; + let data: Vec = samples.iter().flat_map(|sample| sample.to_le_bytes()).collect(); + if let Err(err) = publish.write(&moq_audio::Frame::new(Bytes::from(data), timestamp)) { + self.stats.fail(format!("audio: {err}")); + producer = None; + } + } + } +} + +/// The encoders behind an [`Output`], handed to their threads. +struct Publish { + video: moq_video::encode::Producer, + config: moq_video::encode::Config, + audio: moq_audio::encode::Producer, +} + +/// Announce the program broadcast and connect to the relay, which also +/// carries `subscribe`'s broadcast sources. +async fn publish( + client: moq_tokio::Client, + url: url::Url, + subscribe: moq_net::origin::Producer, + path: &str, + canvas: moq_scene::Canvas, +) -> anyhow::Result<(Output, Publish)> { + let origin = moq_tokio::origin::spawn(); + let publication = origin.create_broadcast(path).context("cannot create the broadcast")?; + let mut broadcast = publication.broadcast().clone(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default())?; + + let config = moq_video::encode::Config::new(canvas.width, canvas.height, canvas.framerate()); + // Opens a throwaway encoder now, so a machine that cannot encode the + // canvas fails at startup rather than when the first viewer arrives. + let rendition = config.probe().await.context("cannot encode the canvas")?; + let reserved = catalog + .reserve() + .with_timeline_hint(config.gop.duration(config.framerate)); + let video = moq_video::encode::Producer::new(broadcast.clone(), reserved, rendition)?; + + let bus = moq_audio::bus::Config::default(); + let mut input = moq_audio::encode::Input::new(bus.sample_rate, bus.layout); + input.format = moq_audio::Format::F32; + let mut options = moq_audio::encode::Options::default(); + options.track = Some("audio".into()); + options.config = moq_audio::encode::Config::from_input(moq_audio::encode::Codec::Opus, &input); + options.config.bitrate = Some(AUDIO_BITRATE); + let audio = moq_audio::encode::Producer::new(&mut broadcast, catalog, input, &options)?; + + publication + .announce(moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint())) + .context("cannot announce the broadcast")?; + tracing::info!(url = %moq_tokio::RedactedUrl::new(&url), broadcast = %path, "connecting"); + let connection = client.with_publisher(&origin).with_subscriber(subscribe).connect(url); + + let output = Output { + path: path.to_string(), + connection, + publication, + }; + Ok((output, Publish { video, config, audio })) +} + +/// Track whether anyone watches the video, so the program is only read back +/// and encoded while someone does. +async fn watch(mut demand: moq_net::track::Demand, engine: Arc) { + loop { + match demand.wanted_changed().await { + Ok(wanted) => engine.stats.watched.store(wanted.is_some(), Ordering::Relaxed), + Err(err) => { + engine.stats.watched.store(false, Ordering::Relaxed); + engine.stats.fail(format!("video: {err}")); + return; + } + } + } +} + +/// Read back and encode program frames until the program clock stops. +fn encode( + engine: &Engine, + pending: mpsc::Receiver, + mut producer: moq_video::encode::Producer, + config: moq_video::encode::Config, +) { + let stats = &engine.stats; + let mut encoder: Option = None; + let mut watched = false; + + while let Ok(pending) = pending.recv() { + let start = Instant::now(); + let frame = match pending.wait() { + Ok(frame) => frame, + Err(err) => return stats.fail(format!("readback: {err}")), + }; + stats.readback_us.store(start.elapsed().as_micros() as u64, Ordering::Relaxed); + stats.read_back.fetch_add(1, Ordering::Relaxed); + + let encoder = match &mut encoder { + Some(encoder) => encoder, + None => match moq_video::encode::Encoder::new(&config) { + Ok(opened) => { + *stats.encoder.lock().unwrap() = Some(opened.name().to_string()); + encoder.insert(opened) + } + Err(err) => return stats.fail(format!("video encoder: {err}")), + }, + }; + + // A viewer arriving after a break starts from a keyframe rather than + // waiting out the rest of a group nobody received the start of. + let now = stats.watched.load(Ordering::Relaxed); + if now + && !watched + && let Err(err) = encoder.cut() + { + return stats.fail(format!("video encoder: {err}")); + } + watched = now; + + let start = Instant::now(); + let encoded = match encoder.encode(&frame) { + Ok(encoded) => encoded, + Err(err) => return stats.fail(format!("video encoder: {err}")), + }; + if let Err(err) = producer.publish(&encoded) { + return stats.fail(format!("video: {err}")); + } + stats.encode_us.store(start.elapsed().as_micros() as u64, Ordering::Relaxed); + stats.encoded.fetch_add(1, Ordering::Relaxed); + } +} + +/// Run a source on its own thread until its feed is dropped. +fn spawn(driver: Driver) { + let name = format!("source-{}", driver.name()); + let spawned = std::thread::Builder::new().name(name).spawn(move || clocked(driver.run())); + if let Err(err) = spawned { + tracing::error!(%err, "cannot start a source thread"); + } +} + +/// Run `task` on the calling thread, with its own clock and IO. +fn clocked>(task: F) { + match tokio::runtime::Builder::new_current_thread().enable_all().build() { + Ok(runtime) => runtime.block_on(task), + Err(err) => tracing::error!(%err, "cannot start a runtime"), + } +} + +#[cfg(test)] +mod tests { + use moq_mux::catalog::Stream; + + use super::*; + + /// Bounds each step, so a stall fails rather than hangs. + const TIMEOUT: Duration = Duration::from_secs(30); + + /// The program a studio publishes reaches a relay as the picture it composed, + /// with its audio alongside. + #[tokio::test(flavor = "multi_thread")] + #[ignore = "needs a Vulkan device and an H.264 encoder; run by just rs render-sw"] + async fn the_program_reaches_a_relay() { + let _ = moq_tokio::crypto::install_default(); + let fixture = moq_relay::test_relay().await.expect("test relay"); + let ready = fixture.relay.ready(); + let relay = fixture.relay.cluster().origin.consume(); + tokio::spawn(fixture.relay.run()); + ready.wait().await.expect("relay ready"); + + let mut client = moq_tokio::connect::Config::default(); + client.tls.fingerprint = vec![fixture.fingerprint.clone()]; + let client = client.init(Default::default()).expect("client"); + + let dir = tempfile::tempdir().unwrap(); + let mut collection = Collection::new(); + collection.canvas = moq_scene::Canvas { + width: 320, + height: 180, + framerate: 30, + }; + collection.sources.insert( + "red".into(), + moq_scene::Source::Color { + color: moq_scene::Rgba([255, 0, 0, 255]), + }, + ); + collection.scenes[0].items.push(moq_scene::Item { + source: "red".into(), + rect: moq_scene::Rect::fill(&collection.canvas), + visible: true, + }); + + let gpu = Gpu::new().await.expect("a GPU"); + let path = dir.path().join("show.toml"); + let connect = Some((client, fixture.url.clone())); + let _engine = Engine::start(gpu, path, collection, connect, "studio") + .await + .expect("the studio starts"); + + let broadcast = tokio::time::timeout(TIMEOUT, relay.routed_broadcast("studio")) + .await + .expect("the relay routes the program") + .expect("broadcast"); + let format = moq_mux::catalog::CatalogFormat::default(); + let mut catalogs = moq_mux::catalog::Consumer::<()>::new(&broadcast, format) + .await + .expect("catalog"); + let catalog = catalogs.next().await.expect("catalog").expect("a catalog"); + let (video, config) = catalog.video.enabled().next().expect("a video rendition"); + let mut options = moq_video::decode::Options::new(); + options.start = moq_video::decode::Start::Latest; + let mut decoder = moq_video::decode::Consumer::new(&broadcast, config, video.clone(), options) + .await + .expect("video decoder"); + let frame = tokio::time::timeout(TIMEOUT, async { + loop { + match decoder.read().await.expect("decode") { + Some(moq_video::decode::Read::Frame(frame)) => return frame, + Some(_) => continue, + None => panic!("the video ended"), + } + } + }) + .await + .expect("a picture arrives"); + + assert_eq!(frame.size(), moq_video::Size::new(320, 180)); + let rgba = frame.surface.to_rgba(&Default::default()).expect("rgba"); + let center = (90 * 320 + 160) * 4; + let [r, g, b, _] = rgba.data()[center..center + 4] else { + unreachable!() + }; + // H.264 and its 4:2:0 round trip shift a primary a little. + assert!(r > 200 && g < 60 && b < 60, "{:?}", [r, g, b]); + + let (audio, config) = catalog.audio.enabled().next().expect("an audio rendition"); + let mut options = moq_audio::decode::Options::new(); + options.start = moq_audio::decode::Start::Latest; + let mut decoder = moq_audio::decode::Consumer::new(&broadcast, config, audio.clone(), options) + .await + .expect("audio decoder"); + tokio::time::timeout(TIMEOUT, async { + loop { + match decoder.read().await.expect("decode") { + Some(moq_audio::decode::Read::Frame(_)) => return, + Some(_) => continue, + None => panic!("the audio ended"), + } + } + }) + .await + .expect("sound arrives"); + } +} diff --git a/rs/moq-studio/src/main.rs b/rs/moq-studio/src/main.rs index f328e4d9d04..f10014b2431 100644 --- a/rs/moq-studio/src/main.rs +++ b/rs/moq-studio/src/main.rs @@ -1 +1,122 @@ -fn main() {} +//! MoQ Studio: compose scenes from cameras, screens, images, and other +//! broadcasts on the GPU, and go live over MoQ. +//! +//! The scenes, sources, and compositor live in `moq-scene`; this binary is the +//! window around them and the program clock that publishes the result. + +use std::path::PathBuf; +use std::sync::Arc; + +use anyhow::Context; + +mod engine; +mod ui; + +#[derive(usage_rs::Cli, Clone)] +#[usage(unknown_flags = "error", args_override_self = false)] +#[usage(name = "moq-studio")] +#[usage(completion, settings)] +pub struct Config { + /// The scene collection to open. Created on the first save if it does not exist. + #[usage(value_hint = usage_rs::ValueHint::FilePath, extensions("toml"))] + pub scenes: PathBuf, + + /// The broadcast path the program is published at. + #[usage(long, default = "studio")] + pub broadcast: String, + + /// The MoQ client configuration. Without `--connect`, nothing is published + /// and broadcast sources stay offline. + #[usage(flatten)] + pub client: moq_tokio::connect::Config, + + /// QUIC transport tuning (`--quic-*`). + #[usage(flatten)] + pub quic: moq_tokio::quic::Config, + + /// The log configuration. + #[usage(flatten)] + pub log: moq_tokio::Log, +} + +impl Config { + /// Refuse a config parsed from released spellings, naming what replaced each. + fn check_deprecated(&self) -> anyhow::Result<()> { + let mut deprecated = self.client.deprecated(); + deprecated.extend(self.quic.deprecated()); + anyhow::ensure!(deprecated.is_empty(), "{deprecated}"); + Ok(()) + } +} + +fn main() -> anyhow::Result<()> { + let config = Config::parse(); + config.log.init()?; + config.check_deprecated()?; + moq_tokio::cli::check_env(&[Config::spec()], &[])?; + + let runtime = tokio::runtime::Runtime::new()?; + let gpu = pollster::block_on(engine::Gpu::new())?; + let collection = engine::load(&config.scenes)?; + let connect = match config.client.url.clone() { + Some(url) => Some((config.client.clone().init(config.quic.clone())?, url)), + None => None, + }; + let engine = runtime.block_on(engine::Engine::start( + gpu.clone(), + config.scenes.clone(), + collection, + connect, + &config.broadcast, + ))?; + + run(engine, gpu, runtime.handle().clone()) +} + +/// Show the window until it is closed. +fn run(engine: Arc, gpu: engine::Gpu, runtime: tokio::runtime::Handle) -> anyhow::Result<()> { + let options = eframe::NativeOptions { + viewport: egui::ViewportBuilder::default() + .with_title("MoQ Studio") + .with_app_id("moq-studio") + .with_inner_size([1440.0, 900.0]), + wgpu_options: eframe::egui_wgpu::WgpuConfiguration { + // The window draws on the device the compositor does, so preview and + // program reach the screen without leaving the GPU. + wgpu_setup: eframe::egui_wgpu::WgpuSetup::Existing(eframe::egui_wgpu::WgpuSetupExisting { + instance: gpu.instance, + adapter: gpu.adapter, + device: gpu.device, + queue: gpu.queue, + }), + ..Default::default() + }, + ..Default::default() + }; + eframe::run_native( + "moq-studio", + options, + Box::new(move |cc| Ok(Box::new(ui::Studio::new(cc, engine, runtime)?))), + ) + .map_err(|err| anyhow::anyhow!("{err}")) + .context("the window failed") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_a_scene_file_and_a_relay() { + let config = Config::parse_from(&[ + std::ffi::OsStr::new("show.toml"), + std::ffi::OsStr::new("--connect"), + std::ffi::OsStr::new("http://localhost:4443"), + ]) + .expect("a scene file and a relay should parse"); + config.check_deprecated().unwrap(); + assert_eq!(config.scenes, PathBuf::from("show.toml")); + assert_eq!(config.broadcast, "studio"); + assert_eq!(config.client.url.unwrap().port(), Some(4443)); + } +} diff --git a/rs/moq-studio/src/ui.rs b/rs/moq-studio/src/ui.rs new file mode 100644 index 00000000000..d2fbe72d41e --- /dev/null +++ b/rs/moq-studio/src/ui.rs @@ -0,0 +1,858 @@ +//! The window: docked panels over the engine's state. + +use std::sync::Arc; +use std::sync::atomic::Ordering; +use std::time::Duration; + +use anyhow::Context; +use egui::{Color32, Ui}; +use egui_dock::{DockArea, DockState, NodeIndex, Style, TabViewer}; +use moq_scene::source::Status; +use moq_scene::{Item, Rect, Scene, Source, wgpu}; + +use crate::engine::{Engine, State}; + +/// The panels, each a tab the user can move, split, or stack. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum Tab { + Preview, + Program, + Scenes, + Items, + Sources, + Devices, + Audio, + Output, +} + +impl Tab { + fn title(self) -> &'static str { + match self { + Self::Preview => "Preview", + Self::Program => "Program", + Self::Scenes => "Scenes", + Self::Items => "Items", + Self::Sources => "Sources", + Self::Devices => "Devices", + Self::Audio => "Audio", + Self::Output => "Output", + } + } +} + +/// The studio window. +pub struct Studio { + dock: DockState, + panels: Panels, +} + +impl Studio { + pub fn new( + cc: &eframe::CreationContext<'_>, + engine: Arc, + runtime: tokio::runtime::Handle, + ) -> anyhow::Result { + let render = cc + .wgpu_render_state + .as_ref() + .context("the window is not drawn with wgpu")?; + + // The compositor's textures live as long as the engine, so registering + // them once is enough: egui samples whatever they hold each frame. + let (preview, program) = { + let state = engine.state.lock().unwrap(); + let mut renderer = render.renderer.write(); + let mut register = |texture: &wgpu::Texture| { + let view = texture.create_view(&Default::default()); + renderer.register_native_texture(&render.device, &view, wgpu::FilterMode::Linear) + }; + ( + register(state.compositor.preview()), + register(state.compositor.program()), + ) + }; + + let mut dock = DockState::new(vec![Tab::Preview]); + let tree = dock.main_surface_mut(); + let [top, bottom] = tree.split_below(NodeIndex::root(), 0.62, vec![Tab::Scenes]); + tree.split_right(top, 0.5, vec![Tab::Program]); + let [_, items] = tree.split_right(bottom, 0.18, vec![Tab::Items]); + let [_, sources] = tree.split_right(items, 0.4, vec![Tab::Sources, Tab::Devices]); + tree.split_right(sources, 0.5, vec![Tab::Audio, Tab::Output]); + + Ok(Self { + dock, + panels: Panels { + engine, + runtime, + preview, + program, + new_scene: String::new(), + new_item: None, + add: AddSource::default(), + devices: Devices::default(), + message: None, + }, + }) + } +} + +impl eframe::App for Studio { + fn ui(&mut self, ui: &mut Ui, _frame: &mut eframe::Frame) { + // Redraw at the canvas rate, so preview and program are live. + let framerate = self.panels.engine.state.lock().unwrap().collection.canvas.framerate; + ui.ctx().request_repaint_after(Duration::from_secs(1) / framerate); + + if ui.input_mut(|input| input.consume_key(egui::Modifiers::COMMAND, egui::Key::S)) { + self.panels.save(); + } + + egui::Panel::top("menu").show(ui, |ui| self.panels.menu(ui)); + egui::Panel::bottom("status").show(ui, |ui| self.panels.status(ui)); + DockArea::new(&mut self.dock) + .style(Style::from_egui(ui.style())) + .show_inside(ui, &mut self.panels); + } +} + +/// What every panel draws from, plus the half-typed forms. +struct Panels { + engine: Arc, + runtime: tokio::runtime::Handle, + preview: egui::TextureId, + program: egui::TextureId, + new_scene: String, + new_item: Option, + add: AddSource, + devices: Devices, + /// The outcome of the last action, shown in the status bar. + message: Option>, +} + +impl TabViewer for Panels { + type Tab = Tab; + + fn id(&mut self, tab: &mut Tab) -> egui::Id { + egui::Id::new(*tab) + } + + fn title(&mut self, tab: &mut Tab) -> egui::WidgetText { + tab.title().into() + } + + fn is_closeable(&self, _tab: &Tab) -> bool { + false + } + + fn ui(&mut self, ui: &mut Ui, tab: &mut Tab) { + match tab { + Tab::Preview => self.preview(ui), + Tab::Program => self.program(ui), + Tab::Scenes => self.scenes(ui), + Tab::Items => self.items(ui), + Tab::Sources => self.sources(ui), + Tab::Devices => self.devices(ui), + Tab::Audio => self.audio(ui), + Tab::Output => self.output(ui), + } + } +} + +impl Panels { + fn menu(&mut self, ui: &mut Ui) { + egui::MenuBar::new().ui(ui, |ui| { + ui.menu_button("File", |ui| { + if ui.button("Save").clicked() { + self.save(); + } + if ui.button("Reload").clicked() { + let result = self.engine.state.lock().unwrap().reload(); + self.report(result.map(|()| "reloaded".to_string())); + } + }); + }); + } + + fn status(&mut self, ui: &mut Ui) { + ui.horizontal(|ui| { + let state = self.engine.state.lock().unwrap(); + ui.label(state.path.display().to_string()); + if state.unsaved() { + ui.weak("(unsaved)"); + } + drop(state); + match &self.message { + Some(Ok(message)) => { + ui.separator(); + ui.label(message); + } + Some(Err(message)) => { + ui.separator(); + ui.colored_label(ui.visuals().error_fg_color, message); + } + None => {} + } + }); + } + + fn save(&mut self) { + let result = self.engine.state.lock().unwrap().save(); + self.report(result.map(|()| "saved".to_string())); + } + + fn report(&mut self, result: anyhow::Result) { + self.message = Some(result.map_err(|err| format!("{err:#}"))); + } + + fn preview(&mut self, ui: &mut Ui) { + let state = self.engine.state.lock().unwrap(); + let canvas = state.collection.canvas; + ui.label(format!("Preview: {}", state.compositor.preview_scene().unwrap_or("nothing"))); + drop(state); + picture(ui, self.preview, canvas); + } + + fn program(&mut self, ui: &mut Ui) { + let mut state = self.engine.state.lock().unwrap(); + let canvas = state.collection.canvas; + let fade = state.collection.transition.fade; + ui.horizontal(|ui| { + if ui.button("Cut").clicked() { + state.compositor.cut(); + } + if ui.button(format!("Fade ({} ms)", fade.as_millis())).clicked() { + state.compositor.fade(fade, moq_clock::Instant::now()); + } + ui.label(format!("Program: {}", state.compositor.program_scene().unwrap_or("nothing"))); + if state.compositor.fading() { + ui.weak("fading"); + } + }); + drop(state); + picture(ui, self.program, canvas); + } + + fn scenes(&mut self, ui: &mut Ui) { + let mut state = self.engine.state.lock().unwrap(); + let preview = state.compositor.preview_scene().map(str::to_owned); + let program = state.compositor.program_scene().map(str::to_owned); + let mut pick = None; + let mut remove = None; + + egui::ScrollArea::vertical().show(ui, |ui| { + for (index, scene) in state.collection.scenes.iter().enumerate() { + ui.horizontal(|ui| { + let selected = preview.as_deref() == Some(scene.name.as_str()); + if ui.selectable_label(selected, &scene.name).clicked() { + pick = Some(scene.name.clone()); + } + if program.as_deref() == Some(scene.name.as_str()) { + ui.colored_label(Color32::RED, "on program"); + } + if ui.small_button("Remove").clicked() { + remove = Some(index); + } + }); + } + }); + + ui.separator(); + ui.horizontal(|ui| { + ui.text_edit_singleline(&mut self.new_scene); + if ui.button("Add scene").clicked() { + let name = self.new_scene.trim().to_string(); + if name.is_empty() || state.collection.scene(&name).is_some() { + self.message = Some(Err(format!("a scene needs a new, non-empty name, not `{name}`"))); + } else { + state.edit().scenes.push(Scene::new(name.clone())); + pick = Some(name); + self.new_scene.clear(); + } + } + }); + + if let Some(pick) = pick { + state.compositor.set_preview(Some(pick)); + } + if let Some(index) = remove { + state.edit().scenes.remove(index); + } + } + + /// The items of the scene in preview, back to front. + fn items(&mut self, ui: &mut Ui) { + let mut state = self.engine.state.lock().unwrap(); + let Some(name) = state.compositor.preview_scene().map(str::to_owned) else { + ui.label("Pick a scene to edit what it shows."); + return; + }; + let Some(scene) = state.collection.scene(&name) else { + ui.label(format!("There is no scene `{name}`.")); + return; + }; + let canvas = state.collection.canvas; + let mut items = scene.items.clone(); + let mut edited = false; + let mut moved = None; + let mut remove = None; + + egui::ScrollArea::vertical().show(ui, |ui| { + egui::Grid::new("items").striped(true).show(ui, |ui| { + for heading in ["Shown", "Source", "x", "y", "Width", "Height", ""] { + ui.strong(heading); + } + ui.end_row(); + + let count = items.len(); + for (index, item) in items.iter_mut().enumerate() { + edited |= ui.checkbox(&mut item.visible, "").changed(); + ui.label(&item.source); + let rect = &mut item.rect; + edited |= ui.add(egui::DragValue::new(&mut rect.x)).changed(); + edited |= ui.add(egui::DragValue::new(&mut rect.y)).changed(); + edited |= ui.add(egui::DragValue::new(&mut rect.width).range(0.0..=f32::MAX)).changed(); + edited |= ui.add(egui::DragValue::new(&mut rect.height).range(0.0..=f32::MAX)).changed(); + ui.horizontal(|ui| { + if ui.small_button("Fill").clicked() { + *rect = Rect::fill(&canvas); + edited = true; + } + if ui.add_enabled(index > 0, egui::Button::new("Back").small()).clicked() { + moved = Some((index, index - 1)); + } + if ui.add_enabled(index + 1 < count, egui::Button::new("Front").small()).clicked() { + moved = Some((index, index + 1)); + } + if ui.small_button("Remove").clicked() { + remove = Some(index); + } + }); + ui.end_row(); + } + }); + + ui.separator(); + ui.horizontal(|ui| { + let pictures: Vec<&String> = state + .collection + .sources + .iter() + .filter(|(_, source)| source.has_video()) + .map(|(name, _)| name) + .collect(); + egui::ComboBox::from_id_salt("new item") + .selected_text(self.new_item.as_deref().unwrap_or("Pick a source")) + .show_ui(ui, |ui| { + for name in pictures { + ui.selectable_value(&mut self.new_item, Some(name.clone()), name); + } + }); + if ui.button("Add to scene").clicked() + && let Some(source) = self.new_item.take() + { + items.push(Item { + source, + rect: Rect::fill(&canvas), + visible: true, + }); + edited = true; + } + }); + }); + + if let Some((from, to)) = moved { + items.swap(from, to); + edited = true; + } + if let Some(index) = remove { + items.remove(index); + edited = true; + } + if edited && let Some(scene) = state.edit().scene_mut(&name) { + scene.items = items; + } + } + + fn sources(&mut self, ui: &mut Ui) { + let engine = self.engine.clone(); + let mut state = engine.state.lock().unwrap(); + let mut remove = None; + + egui::ScrollArea::vertical().show(ui, |ui| { + egui::Grid::new("sources").striped(true).show(ui, |ui| { + for (name, feed) in state.sources.iter() { + ui.label(name); + ui.label(describe(feed.source())); + status(ui, &feed.status()); + if ui.small_button("Remove").clicked() { + remove = Some(name.to_string()); + } + ui.end_row(); + } + }); + + ui.separator(); + if let Some(source) = self.add.ui(ui) { + let name = self.add.name.trim().to_string(); + let result = add(&mut state, name, source); + if result.is_ok() { + self.add.name.clear(); + } + self.report(result.map(|name| format!("added `{name}`"))); + } + }); + + if let Some(name) = remove { + let collection = state.edit(); + collection.sources.remove(&name); + collection.audio.remove(&name); + for scene in &mut collection.scenes { + scene.items.retain(|item| item.source != name); + } + } + } + + fn devices(&mut self, ui: &mut Ui) { + self.devices.poll(); + if ui.button("Refresh").clicked() { + self.devices.refresh(&self.runtime); + } + let Some(picked) = self.devices.ui(ui) else { return }; + let mut state = self.engine.state.lock().unwrap(); + // Two cameras of the same model share a name; number the later ones. + let name = std::iter::once(picked.name.clone()) + .chain((2..).map(|n| format!("{} {n}", picked.name))) + .find(|name| !state.collection.sources.contains_key(name)) + .expect("an unused name"); + let result = add(&mut state, name, picked.source); + drop(state); + self.report(result.map(|name| format!("added `{name}`"))); + } + + fn audio(&mut self, ui: &mut Ui) { + ui.horizontal(|ui| { + ui.strong("Program"); + meter(ui, self.engine.bus.level()); + }); + ui.separator(); + + let mut state = self.engine.state.lock().unwrap(); + let mut edits = Vec::new(); + egui::Grid::new("audio").striped(true).show(ui, |ui| { + for (name, feed) in state.sources.iter() { + let Some(strip) = feed.strip() else { continue }; + let mut audio = state.collection.audio.get(name).copied().unwrap_or_default(); + let mut db = decibels(audio.gain); + ui.label(name); + let mut edited = ui.checkbox(&mut audio.muted, "Mute").changed(); + if ui + .add(egui::Slider::new(&mut db, MUTE_DB..=12.0).suffix(" dB")) + .changed() + { + audio.gain = if db <= MUTE_DB { 0.0 } else { 10f32.powf(db / 20.0) }; + edited = true; + } + meter(ui, strip.level()); + ui.end_row(); + if edited { + edits.push((name.to_string(), audio)); + } + } + }); + for (name, audio) in edits { + state.edit().audio.insert(name, audio); + } + } + + fn output(&mut self, ui: &mut Ui) { + let stats = &self.engine.stats; + let count = |value: &std::sync::atomic::AtomicU64| value.load(Ordering::Relaxed); + let millis = |value: &std::sync::atomic::AtomicU64| format!("{:.1} ms", count(value) as f64 / 1000.0); + + egui::Grid::new("output").show(ui, |ui| { + match &self.engine.output { + Some(output) => { + ui.label("Broadcast"); + ui.label(&output.path); + ui.end_row(); + ui.label("Relay"); + ui.label(if output.connection.connected() { + "connected" + } else { + "connecting" + }); + ui.end_row(); + ui.label("Viewers"); + ui.label(if stats.watched.load(Ordering::Relaxed) { + "watching" + } else { + "none; not encoding" + }); + ui.end_row(); + } + None => { + ui.label("Relay"); + ui.label("none; start with --connect to publish"); + ui.end_row(); + } + } + ui.label("Drawn"); + ui.label(format!("{} frames", count(&stats.rendered))); + ui.end_row(); + ui.label("Read back to the CPU"); + ui.label(format!("{} frames, last {}", count(&stats.read_back), millis(&stats.readback_us))); + ui.end_row(); + ui.label("Dropped"); + ui.label(format!("{} frames", count(&stats.dropped))); + ui.end_row(); + ui.label("Encoder"); + ui.label(stats.encoder.lock().unwrap().clone().unwrap_or_else(|| "not open".into())); + ui.end_row(); + ui.label("Encoded"); + ui.label(format!("{} frames, last {}", count(&stats.encoded), millis(&stats.encode_us))); + ui.end_row(); + }); + if let Some(error) = stats.error.lock().unwrap().as_ref() { + ui.colored_label(ui.visuals().error_fg_color, error); + } + } +} + +/// Add `source` as `name`, and place it over the scene in preview if it has a +/// picture. Returns the name. +fn add(state: &mut State, name: String, source: Source) -> anyhow::Result { + anyhow::ensure!(!name.is_empty(), "a source needs a name"); + anyhow::ensure!( + !state.collection.sources.contains_key(&name), + "there is already a source named `{name}`" + ); + let preview = state.compositor.preview_scene().map(str::to_owned); + let canvas = state.collection.canvas; + let video = source.has_video(); + let collection = state.edit(); + collection.sources.insert(name.clone(), source); + if video && let Some(scene) = preview.and_then(|preview| collection.scene_mut(&preview)) { + scene.items.push(Item { + source: name.clone(), + rect: Rect::fill(&canvas), + visible: true, + }); + } + Ok(name) +} + +/// Draw `texture` as large as fits, keeping the canvas's shape. +fn picture(ui: &mut Ui, texture: egui::TextureId, canvas: moq_scene::Canvas) { + let available = ui.available_size(); + let aspect = canvas.width as f32 / canvas.height as f32; + let mut size = egui::vec2(available.x, available.x / aspect); + if size.y > available.y { + size = egui::vec2(available.y * aspect, available.y); + } + ui.vertical_centered(|ui| ui.image((texture, size))); +} + +fn describe(source: &Source) -> String { + let or_default = |id: &Option| id.clone().unwrap_or_else(|| "default".into()); + match source { + Source::Camera { id } => format!("camera: {}", or_default(id)), + Source::Display { id } => format!("display: {}", or_default(id)), + Source::Window { id } => format!("window: {id}"), + Source::Microphone { id } => format!("microphone: {}", or_default(id)), + Source::SystemAudio => "system audio".into(), + Source::Image { path } => format!("image: {}", path.display()), + Source::Color { color } => format!("color: {color}"), + Source::Broadcast { path } => format!("broadcast: {path}"), + } +} + +fn status(ui: &mut Ui, status: &Status) { + match status { + Status::Starting => ui.weak("starting"), + Status::Live => ui.colored_label(Color32::from_rgb(80, 200, 120), "live"), + Status::Offline => ui.colored_label(Color32::from_rgb(220, 180, 60), "offline"), + Status::Failed(reason) => ui + .colored_label(ui.visuals().error_fg_color, "failed") + .on_hover_text(reason), + _ => ui.weak("unknown"), + }; +} + +/// The quietest gain the slider offers, which it treats as silence. +const MUTE_DB: f32 = -60.0; + +fn decibels(gain: f32) -> f32 { + if gain <= 0.0 { + MUTE_DB + } else { + (20.0 * gain.log10()).max(MUTE_DB) + } +} + +/// A level meter: RMS filled, peak as a tick, on a decibel scale. +fn meter(ui: &mut Ui, level: moq_audio::Level) { + let (rect, _) = ui.allocate_exact_size(egui::vec2(160.0, 12.0), egui::Sense::hover()); + let position = |amplitude: f32| (1.0 - decibels(amplitude) / MUTE_DB).clamp(0.0, 1.0); + let painter = ui.painter_at(rect); + painter.rect_filled(rect, 2.0, ui.visuals().extreme_bg_color); + let color = if level.peak() >= 1.0 { + Color32::RED + } else { + Color32::from_rgb(80, 200, 120) + }; + let mut filled = rect; + filled.set_width(rect.width() * position(level.rms())); + painter.rect_filled(filled, 2.0, color); + let x = rect.left() + rect.width() * position(level.peak()); + painter.vline(x, rect.y_range(), egui::Stroke::new(2.0, color)); +} + +/// The form that adds a source by hand. +struct AddSource { + name: String, + kind: Kind, + /// A path, broadcast path, or device id, depending on the kind. + value: String, + color: Color32, +} + +impl Default for AddSource { + fn default() -> Self { + Self { + name: String::new(), + kind: Kind::default(), + value: String::new(), + color: Color32::WHITE, + } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +enum Kind { + #[default] + Color, + Image, + Broadcast, + Camera, + Display, + Window, + Microphone, + SystemAudio, +} + +impl Kind { + const ALL: [Self; 8] = [ + Self::Color, + Self::Image, + Self::Broadcast, + Self::Camera, + Self::Display, + Self::Window, + Self::Microphone, + Self::SystemAudio, + ]; + + fn label(self) -> &'static str { + match self { + Self::Color => "Color", + Self::Image => "Image", + Self::Broadcast => "Broadcast", + Self::Camera => "Camera", + Self::Display => "Display", + Self::Window => "Window", + Self::Microphone => "Microphone", + Self::SystemAudio => "System audio", + } + } + + /// What the value field holds, if this kind takes one. + fn value(self) -> Option<&'static str> { + match self { + Self::Color | Self::SystemAudio => None, + Self::Image => Some("file"), + Self::Broadcast => Some("path"), + Self::Window => Some("window id"), + Self::Camera | Self::Display | Self::Microphone => Some("id (empty for the default)"), + } + } +} + +impl AddSource { + /// Draw the form; the source to add once the user asks for it. + fn ui(&mut self, ui: &mut Ui) -> Option { + let mut add = false; + egui::Grid::new("add source").show(ui, |ui| { + ui.label("Name"); + ui.text_edit_singleline(&mut self.name); + ui.end_row(); + ui.label("Kind"); + egui::ComboBox::from_id_salt("kind") + .selected_text(self.kind.label()) + .show_ui(ui, |ui| { + for kind in Kind::ALL { + ui.selectable_value(&mut self.kind, kind, kind.label()); + } + }); + ui.end_row(); + if let Some(label) = self.kind.value() { + ui.label(label); + ui.text_edit_singleline(&mut self.value); + ui.end_row(); + } + if self.kind == Kind::Color { + ui.label("Color"); + egui::color_picker::color_edit_button_srgba(ui, &mut self.color, egui::color_picker::Alpha::OnlyBlend); + ui.end_row(); + } + add = ui.button("Add source").clicked(); + ui.end_row(); + }); + if !add { + return None; + } + + let value = self.value.trim().to_string(); + let id = (!value.is_empty()).then(|| value.clone()); + Some(match self.kind { + Kind::Color => Source::Color { + color: moq_scene::Rgba(self.color.to_srgba_unmultiplied()), + }, + Kind::Image => Source::Image { path: value.into() }, + Kind::Broadcast => Source::Broadcast { path: value }, + Kind::Camera => Source::Camera { id }, + Kind::Display => Source::Display { id }, + Kind::Window => Source::Window { id: value }, + Kind::Microphone => Source::Microphone { id }, + Kind::SystemAudio => Source::SystemAudio, + }) + } +} + +/// A device the user can add as a source. +struct Pick { + name: String, + source: Source, +} + +/// The capture devices this machine has, listed in the background. +#[derive(Default)] +struct Devices { + listing: Option, + pending: Option>, +} + +/// One listing of every kind of device, each of which can fail on its own. +struct Listing { + kinds: Vec<(&'static str, Result, String>)>, +} + +impl Devices { + fn refresh(&mut self, runtime: &tokio::runtime::Handle) { + let (send, recv) = std::sync::mpsc::channel(); + self.pending = Some(recv); + runtime.spawn(async move { + let _ = send.send(list().await); + }); + } + + fn poll(&mut self) { + if let Some(listing) = self.pending.as_ref().and_then(|pending| pending.try_recv().ok()) { + self.listing = Some(listing); + self.pending = None; + } + } + + /// Draw the listing; the device to add once the user picks one. + fn ui(&mut self, ui: &mut Ui) -> Option { + if self.pending.is_some() { + ui.spinner(); + } + let Some(listing) = &mut self.listing else { + ui.label("Refresh to list this machine's cameras, screens, and inputs."); + return None; + }; + let mut picked = None; + egui::ScrollArea::vertical().show(ui, |ui| { + for (kind, devices) in &mut listing.kinds { + ui.strong(*kind); + match devices { + Ok(devices) if devices.is_empty() => { + ui.weak("none"); + } + Ok(devices) => { + for (index, device) in devices.iter().enumerate() { + ui.horizontal(|ui| { + if ui.small_button("Add").clicked() { + picked = Some(index); + } + ui.label(&device.name); + }); + } + if let Some(index) = picked.take() { + return Some(devices.remove(index)); + } + } + Err(err) => { + ui.colored_label(ui.visuals().error_fg_color, err.as_str()); + } + } + } + None + }) + .inner + } +} + +#[cfg(feature = "capture")] +async fn list() -> Listing { + fn picks( + result: Result, impl std::fmt::Display>, + pick: impl Fn(T) -> Pick, + ) -> Result, String> { + result.map(|devices| devices.into_iter().map(pick).collect()).map_err(|err| err.to_string()) + } + + let cameras = picks(moq_video::capture::cameras().await, |camera| Pick { + name: camera.name, + source: Source::Camera { id: Some(camera.id) }, + }); + let displays = picks(moq_video::capture::displays().await, |display| Pick { + name: display.name, + source: Source::Display { id: Some(display.id) }, + }); + let windows = picks(moq_video::capture::windows().await, |window| Pick { + name: format!("{} ({})", window.title, window.app), + source: Source::Window { id: window.id }, + }); + let mut microphones = picks(moq_audio::capture::devices().await, |device| Pick { + name: format!("{} ({})", device.name, device.host), + source: Source::Microphone { id: Some(device.id) }, + }); + if cfg!(target_os = "macos") + && let Ok(microphones) = &mut microphones + { + microphones.push(Pick { + name: "System audio".into(), + source: Source::SystemAudio, + }); + } + + Listing { + kinds: vec![ + ("Cameras", cameras), + ("Displays", displays), + ("Windows", windows), + ("Audio inputs", microphones), + ], + } +} + +#[cfg(not(feature = "capture"))] +async fn list() -> Listing { + Listing { + kinds: vec![( + "Devices", + Err("this build cannot capture; rebuild with `--features capture`".into()), + )], + } +} diff --git a/sh/rs/select.sh b/sh/rs/select.sh index ae6b88c06c6..43370c5131a 100755 --- a/sh/rs/select.sh +++ b/sh/rs/select.sh @@ -140,7 +140,7 @@ case "$action" in if wants '(moq-wasm|moq-mux|moq-ffi|moq-clock)'; then just rs wasm; fi # Device code behind the off-by-default `capture` feature. check-test # lints it through the test build, as for the default features. - if [[ "$action" != lint ]] && wants '(moq-video|moq-audio)'; then + if [[ "$action" != lint ]] && wants '(moq-video|moq-audio|moq-scene|moq-studio)'; then if [[ "$action" == check-test ]]; then just rs capture-test; else just rs capture; fi fi # moq-cli's media tasks are behind `play`; only check-test runs tests. From ed7df65000461239dd05fedc579067c6ce021a4f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 10:25:46 -0700 Subject: [PATCH 04/15] docs(moq-studio): document the studio and rewrite its quest to what remains Adds doc/bin/studio.md, the repository map and AGENTS.md rows, and the moq-audio migration note for Level. The app quest now holds only the zero-copy program encode and the macOS and Windows runs, and the studio quests that only needed the app are unblocked. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 1 + Cargo.lock | 2 - doc/.vitepress/config.ts | 1 + doc/bin/index.md | 1 + doc/bin/studio.md | 102 ++++++++++++++++++++++++++++++ doc/setup/repository.md | 1 + quest/m3/studio/README.md | 5 +- quest/m3/studio/app.md | 57 +++++++---------- quest/m3/studio/audio-filters.md | 4 -- quest/m3/studio/browser-source.md | 4 -- quest/m3/studio/destinations.md | 1 - quest/m3/studio/guests.md | 4 -- quest/m3/studio/mixes.md | 4 -- quest/m3/studio/multi-canvas.md | 4 -- quest/m3/studio/packaging.md | 2 +- quest/m3/studio/recording.md | 4 -- quest/m3/studio/vcam/linux.md | 4 -- quest/m3/studio/vcam/macos.md | 2 +- quest/m3/studio/vcam/windows.md | 2 +- quest/m3/studio/web/video-web.md | 1 - rs/moq-audio/MIGRATE.md | 4 ++ rs/moq-audio/src/bus.rs | 5 +- rs/moq-scene/Cargo.toml | 2 - rs/moq-scene/src/collection.rs | 41 +++++++++--- rs/moq-scene/src/compositor.rs | 21 ++++-- rs/moq-scene/src/source.rs | 8 ++- rs/moq-studio/src/engine.rs | 12 +++- rs/moq-studio/src/ui.rs | 102 +++++++++++++++++++----------- 28 files changed, 270 insertions(+), 131 deletions(-) create mode 100644 doc/bin/studio.md diff --git a/AGENTS.md b/AGENTS.md index ab25acfde34..87f5f135ae6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -113,6 +113,7 @@ The `quest` binary comes from the kixelated/quest flake input and serves the que | `rs/moq-relay` config/behavior | `doc/bin/relay/` | | `rs/moq-cli` | `doc/bin/cli.md` | | `rs/moq-gst` | `doc/bin/gstreamer.md` | +| `rs/moq-scene`, `rs/moq-studio` | `doc/bin/studio.md` | | `rs/moq-c` C ABI (`moq.h`) | `doc/lib/c` | | `js/{watch,publish}` UI/API | `demo/web` if it consumes the API | diff --git a/Cargo.lock b/Cargo.lock index 6cb0f3c5960..abae2310f09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5739,7 +5739,6 @@ name = "moq-scene" version = "0.1.0" dependencies = [ "bytemuck", - "hang", "humantime-serde", "image", "kio 0.6.2", @@ -5748,7 +5747,6 @@ dependencies = [ "moq-mux", "moq-net", "moq-video", - "pollster", "serde", "serde_with", "tempfile", diff --git a/doc/.vitepress/config.ts b/doc/.vitepress/config.ts index 4d321c6e98b..4492f72371c 100644 --- a/doc/.vitepress/config.ts +++ b/doc/.vitepress/config.ts @@ -145,6 +145,7 @@ export default defineConfig({ link: "/bin/cli", items: [{ text: "Inspect a relay", link: "/bin/inspect" }], }, + { text: "moq-studio", link: "/bin/studio" }, { text: "Gateways", items: [ diff --git a/doc/bin/index.md b/doc/bin/index.md index 6cfcfe7875a..487e71c93b5 100644 --- a/doc/bin/index.md +++ b/doc/bin/index.md @@ -16,6 +16,7 @@ Once connected, [Inspect a relay](/bin/inspect) shows what it is receiving. | --- | --- | | [moq-relay](/bin/relay/) | Route, cache, and fan out broadcasts. Authenticate with JWTs, cluster across regions, publish stats. | | [moq-cli](/bin/cli) | Move media in and out of MoQ: pipe FFmpeg in, play natively, capture a camera or screen, transcode, mint tokens. | +| [moq-studio](/bin/studio) | Compose cameras, screens, images, and other broadcasts into scenes on the GPU and go live. | ## Gateways diff --git a/doc/bin/studio.md b/doc/bin/studio.md new file mode 100644 index 00000000000..540a6d6755a --- /dev/null +++ b/doc/bin/studio.md @@ -0,0 +1,102 @@ +--- +title: Studio +description: Compose scenes on the GPU and go live over MoQ from a native app +--- + +# Studio + +`moq-studio` is a native streaming studio for Linux, macOS, and Windows. It +composes cameras, screens, windows, images, colors, and broadcasts from a relay +into scenes on the GPU, mixes their audio, and publishes the result to a MoQ +relay. It is early and not packaged yet, so build it from source: + +```bash +cargo run --release -p moq-studio --features capture,openh264 -- \ + show.toml --connect https://cdn.moq.dev/anon --broadcast my-show +``` + +- `show.toml` is the scene collection. It is created on the first save. +- Without `--connect`, the studio composes and previews but publishes nothing, + and broadcast sources stay offline. +- `capture` adds camera, display, window, microphone, and system-audio sources. + On Linux it links ALSA; add `pipewire` for Wayland screen capture. `openh264` + is the software H.264 fallback for a machine without a hardware encoder. +- The `--connect-*`, `--quic-*`, and `--log-level` flags are moq-cli's; see + `moq-studio --help`. + +## Panels + +Every panel is a tab you can drag, split, or stack. + +| Panel | Use it to | +| --- | --- | +| Preview, Program | See the scene you are editing and the one that is live. **Cut** puts the preview on program at once; **Fade** blends over the collection's fade time. | +| Scenes | Pick the scene in preview, add one, or remove one. | +| Items | Place, show, hide, and order the sources the preview scene draws, back to front. | +| Sources | Add a source by hand, see whether each is live, and remove one. | +| Devices | List this machine's cameras, displays, windows, and audio inputs, and add one as a source. | +| Audio | Set each audio source's gain and mute, and watch its meter and the program's. | +| Output | See the relay connection, whether anyone is watching, and the encoder's work. | + +A broadcast source subscribes through the same relay connection, plays the +first video and audio its catalog lists, and comes back on its own when the +publisher restarts. Audio from every source is mixed into the program whether +or not its scene is live; mute a source to leave it out. + +## Scene file + +The collection is TOML, so it diffs and reviews like code. It names devices, +files, and broadcast paths, never a relay URL or token, so it is safe to commit +and share. A file from another format version fails to load rather than being +guessed at. + +```toml +version = 1 + +[canvas] +width = 1920 +height = 1080 +framerate = 30 + +[transition] +fade = "300ms" + +[sources.camera] +kind = "camera" + +[sources.logo] +kind = "image" +path = "logo.png" # relative to the scene file + +[sources.guest] +kind = "broadcast" +path = "room/guest" + +[audio.guest] +gain = 0.5 + +[[scenes]] +name = "Main" +items = [ + { source = "camera", rect = { x = 0, y = 0, width = 1920, height = 1080 } }, + { source = "logo", rect = { x = 1600, y = 40, width = 280, height = 120 } }, +] +``` + +Source kinds are `camera`, `display`, and `microphone` (each with an optional +`id`, the default device without one), `window` (with an `id`), `system_audio` +(macOS), `image` (PNG or JPEG), `color` (`#rrggbb` or `#rrggbbaa`), and +`broadcast`. The Devices panel fills in ids. **File > Save** (Ctrl+S) writes +the file and **File > Reload** reads it back; a new canvas size or frame rate +takes a restart. + +## Output + +The program publishes as one H.264 rendition at the canvas size and frame rate, +with Opus stereo audio, under the `--broadcast` path. Timestamps come from the +studio's own clock, so a source that stalls repeats its last picture instead of +holding up the stream. + +Video is only encoded while someone watches it. Each frame is read back from +the GPU to the CPU for the encoder, which the Output panel counts; frames the +encoder cannot keep up with are dropped there rather than delaying the stream. diff --git a/doc/setup/repository.md b/doc/setup/repository.md index a7fbad11d29..fa64a993050 100644 --- a/doc/setup/repository.md +++ b/doc/setup/repository.md @@ -27,6 +27,7 @@ wrap the Rust core. [Layers](/lib/layers) maps the binding namespaces. | Browser capture, playback, and controls | [`js/publish`](https://github.com/moq-dev/moq/tree/main/js/publish), [`js/watch`](https://github.com/moq-dev/moq/tree/main/js/watch), [`js/video`](https://github.com/moq-dev/moq/tree/main/js/video), [`js/ui`](https://github.com/moq-dev/moq/tree/main/js/ui) | [TypeScript](/lib/js/) | | Browser reactive state and cleanup | [`js/signals`](https://github.com/moq-dev/moq/tree/main/js/signals) | [Signals](/lib/js/signals) | | Native capture, codecs, and rendering | [`rs/moq-audio`](https://github.com/moq-dev/moq/tree/main/rs/moq-audio), [`rs/moq-video`](https://github.com/moq-dev/moq/tree/main/rs/moq-video), [`rs/moq-transcode`](https://github.com/moq-dev/moq/tree/main/rs/moq-transcode) | [Audio](/lib/rs/moq-audio), [video](/lib/rs/moq-video) | +| The native studio: scenes, compositing, and its window | [`rs/moq-scene`](https://github.com/moq-dev/moq/tree/main/rs/moq-scene), [`rs/moq-studio`](https://github.com/moq-dev/moq/tree/main/rs/moq-studio) | [Studio](/bin/studio) | | CLI commands and protocol gateways | [`rs/moq-cli`](https://github.com/moq-dev/moq/tree/main/rs/moq-cli), then the corresponding `rs/moq-{rtmp,srt,rtc,rtsp,hls}` crate | [Applications](/bin/) | | Language bindings | [`rs/moq-ffi`](https://github.com/moq-dev/moq/tree/main/rs/moq-ffi), [`rs/moq-c`](https://github.com/moq-dev/moq/tree/main/rs/moq-c), then `py/`, `swift/`, `kt/`, `go/`, `dart/`, or `cpp/moq/` | [Binding layers](/lib/layers) | | Rooms, chat, and application data | [`rs/moq-room`](https://github.com/moq-dev/moq/tree/main/rs/moq-room), [`js/room`](https://github.com/moq-dev/moq/tree/main/js/room), `rs/moq-json`, `js/json` | [Rooms](/lib/js/room), [JSON tracks](/lib/js/json) | diff --git a/quest/m3/studio/README.md b/quest/m3/studio/README.md index 42d2dfa9b66..4572bb2eabd 100644 --- a/quest/m3/studio/README.md +++ b/quest/m3/studio/README.md @@ -52,8 +52,7 @@ Decided 2026-10-22 with the maintainer (/quest-plan): owns, through a web join link or QR code. - **Naming:** quests name no streaming platforms; ingest requirements are written as generic constraints. -- **Docs:** inline. [The app](/quest/m3/studio/app.md) writes - `doc/bin/studio.md`, and each later quest extends it. +- **Docs:** inline. Each quest extends `doc/bin/studio.md`. Dropped from the first scoping round: relay fan-out ("encode once, the relay restreams") and its upload savings. Direct push with shared encoders matches @@ -62,7 +61,7 @@ repository. ## Required -- [The studio app](/quest/m3/studio/app.md) - egui studio with a minimal moq-scene and program bus: one canvas, scenes of capture, image, and MoQ sources, published to MoQ +- [Studio program encode from the GPU](/quest/m3/studio/app.md) - the program reaches the encoder as a GPU surface instead of a CPU readback, and the studio runs on macOS and Windows - [Destinations](/quest/m3/studio/destinations.md) - RTMP, RTMPS, and WHIP push with per-destination codecs and shared encoders, on a UI-free egress library a hosted service can reuse - [Multiple canvases](/quest/m3/studio/multi-canvas.md) - landscape and portrait canvases composite at once from shared sources - [Guests](/quest/m3/studio/guests.md) - remote guests and phone cameras join through a moq-room link or QR code and appear as sources diff --git a/quest/m3/studio/app.md b/quest/m3/studio/app.md index af9f1a10aa0..66aebf2f6ac 100644 --- a/quest/m3/studio/app.md +++ b/quest/m3/studio/app.md @@ -1,45 +1,32 @@ -# [XL] The studio app +# [L] Studio program encode from the GPU ## Goal -`moq-studio` runs on Linux, macOS, and Windows with docked panels: -- scenes and sources -- a preview and program pair with cut and fade -- device pickers -- an audio bus panel with gain and mute +`moq-studio`'s program reaches moq-video's encoder as a GPU `Surface` +wherever the backend accepts one, instead of the CPU readback every frame +takes today, and the studio is run, not only compiled, on macOS and Windows. -Sources are a camera, display or window capture, a microphone, system audio -where moq-audio captures it (macOS today; the Linux and Windows capture quests -add the rest), an image, a solid color, and a subscribed MoQ broadcast. One canvas -publishes to a MoQ relay. A scene collection saves to a diffable file and -reloads. - -`rs/moq-scene` (scene graph, sources, compositing) and moq-audio's program -bus land here, with the app as their consumer. Their API surface is only what -the app uses. +The Output panel already counts readback frames and time; once this lands it +shows readback only for a backend that cannot take a GPU surface. ## Plan -- **Rendering:** - - One wgpu device is shared by moq-video's render import paths (dmabuf, - Metal) and egui-wgpu. - - Preview and program are textures registered with egui, with no readback. - - Program frames reach moq-video's encoder as a GPU `Surface` wherever the - backend accepts one. Any CPU readback fallback shows in stats. -- **Timing:** output timestamps come from the program clock, not from source - timestamps. A stalled source repeats its last frame rather than stalling - the canvas. -- **Scene file:** choose the format in this quest. It must be diffable and - carry a schema version, and loading an unknown version fails loudly. It - never holds credentials. -- **Tests:** compositing tests render on a wgpu fallback adapter (llvmpipe or - WARP), so they run in CI without a GPU. -- **Same-PR updates:** - - `doc/bin/studio.md` - - an AGENTS.md cross-package row for `rs/moq-studio` and `rs/moq-scene` - - the repository map in `doc/setup/repository.md` -- **Splitting:** if this proves too large for one PR, split along moq-scene - versus app, as long as each piece still lands with a consumer. +- **Zero-copy encode:** the compositor's program texture is allocated as an + exportable image on the device the window and sources share, then handed to + the encoder without leaving the GPU: + - NVENC: a `moq_video::frame::vulkan` slot (opaque-fd memory and a timeline + semaphore), the path the NVDEC to NVENC transcode already uses. + - VAAPI: a DMA-BUF export of the same image. + - VideoToolbox: an IOSurface-backed `CVPixelBuffer` the Metal texture aliases. + - Media Foundation: a shared D3D11 texture. + This reaches below `wgpu` (its `as_hal` escape hatch), so keep the export in + moq-video or moq-scene, never in the app. +- **Platforms:** run the studio on macOS (camera, display, microphone, system + audio) and Windows (camera, display, window, microphone), fixing what only + shows up there. +- **Tests:** extend `just rs render-sw`'s end-to-end test (a red canvas + published to a test relay and decoded) to assert no readback where the + backend takes a surface. ## Related diff --git a/quest/m3/studio/audio-filters.md b/quest/m3/studio/audio-filters.md index a819d02ae60..99a5e7ed53c 100644 --- a/quest/m3/studio/audio-filters.md +++ b/quest/m3/studio/audio-filters.md @@ -11,7 +11,3 @@ from the studio and saved with the scene collection. Prefer maintained crates, such as `nnnoiseless` for RNNoise suppression, over hand-rolled DSP. Benchmark the chain per source so its cost per added source shows up as a slope. - -## Required - -- [The studio app](/quest/m3/studio/app.md) - the sources and bus the chain sits between diff --git a/quest/m3/studio/browser-source.md b/quest/m3/studio/browser-source.md index 42d10be5f2c..db1ae0da708 100644 --- a/quest/m3/studio/browser-source.md +++ b/quest/m3/studio/browser-source.md @@ -21,7 +21,3 @@ Decided 2026-10-22: use CEF through `cef-rs`, desktop only. - **Process:** CEF runs out of process. - **First check:** whether `cef-rs` exposes accelerated offscreen painting. If it does not, upstream that before building on it. - -## Required - -- [The studio app](/quest/m3/studio/app.md) - the compositor and sources this adds a source type to diff --git a/quest/m3/studio/destinations.md b/quest/m3/studio/destinations.md index 55ffcb55f4b..84892afdba5 100644 --- a/quest/m3/studio/destinations.md +++ b/quest/m3/studio/destinations.md @@ -64,6 +64,5 @@ Decided 2026-10-22 with the maintainer: ## Required -- [The studio app](/quest/m3/studio/app.md) - the canvas and MoQ output these destinations join - [AudioToolbox AAC encode](/quest/m2/audio-encode-audiotoolbox.md) - AAC on macOS - [Media Foundation AAC encode](/quest/m2/audio-encode-mediafoundation.md) - AAC on Windows diff --git a/quest/m3/studio/guests.md b/quest/m3/studio/guests.md index 1e84f3da4f4..32c91b8832e 100644 --- a/quest/m3/studio/guests.md +++ b/quest/m3/studio/guests.md @@ -26,10 +26,6 @@ remove a participant from the scene. inside the studio was considered and dropped, because it brings NAT traversal and TURN for no capability the relay lacks. -## Required - -- [The studio app](/quest/m3/studio/app.md) - the MoQ broadcast source guests reuse - ## Related - [Hang contribution references](/quest/m2/contribution-ref.md) - a composed broadcast naming each guest feed diff --git a/quest/m3/studio/mixes.md b/quest/m3/studio/mixes.md index 45956de0c9d..a9aef7338ba 100644 --- a/quest/m3/studio/mixes.md +++ b/quest/m3/studio/mixes.md @@ -15,7 +15,3 @@ output that falls behind drops its own audio, not the bus's. Benchmark source count against mix count, including sparse routing. A mix must cost only what is routed into it, never a scan of every source. - -## Required - -- [The studio app](/quest/m3/studio/app.md) - the single program bus this extends diff --git a/quest/m3/studio/multi-canvas.md b/quest/m3/studio/multi-canvas.md index e1d3004b5c0..296b7395c45 100644 --- a/quest/m3/studio/multi-canvas.md +++ b/quest/m3/studio/multi-canvas.md @@ -16,7 +16,3 @@ canvas together unless the creator unlinks one. Benchmark source count against canvas count, including sparse reuse. Per-canvas cost must grow only with the sources that canvas uses, never with the whole source table. - -## Required - -- [The studio app](/quest/m3/studio/app.md) - the single canvas this generalizes diff --git a/quest/m3/studio/packaging.md b/quest/m3/studio/packaging.md index a2ee938c65e..505d73be09d 100644 --- a/quest/m3/studio/packaging.md +++ b/quest/m3/studio/packaging.md @@ -9,7 +9,7 @@ is verified to launch and list devices on its platform. ## Required - [Signing identities are available](/quest/m3/studio/signing.md) - the credentials signing needs -- [The studio app](/quest/m3/studio/app.md) - the binary being packaged +- [Studio program encode from the GPU](/quest/m3/studio/app.md) - the studio running on every platform it packages ## Related diff --git a/quest/m3/studio/recording.md b/quest/m3/studio/recording.md index 707024ee2e2..5ccb1138252 100644 --- a/quest/m3/studio/recording.md +++ b/quest/m3/studio/recording.md @@ -12,7 +12,3 @@ Write fMP4 through moq-mux. A recording shares a live destination's encoder only when both have the same input (the same canvas or source, and the same audio mix) and identical encode settings. An isolated source track never shares the program's encoder. - -## Required - -- [The studio app](/quest/m3/studio/app.md) - the program and encoders a recording reuses diff --git a/quest/m3/studio/vcam/linux.md b/quest/m3/studio/vcam/linux.md index 5183dc39059..c5c22c6c6fd 100644 --- a/quest/m3/studio/vcam/linux.md +++ b/quest/m3/studio/vcam/linux.md @@ -5,7 +5,3 @@ The studio publishes a canvas's program as a PipeWire video node that PipeWire-aware apps (browsers and conferencing apps) list as a camera. It falls back to an existing v4l2loopback device when one is present. - -## Required - -- [The studio app](/quest/m3/studio/app.md) - the program this exposes diff --git a/quest/m3/studio/vcam/macos.md b/quest/m3/studio/vcam/macos.md index 314ae7c259f..cb7c76bb4d9 100644 --- a/quest/m3/studio/vcam/macos.md +++ b/quest/m3/studio/vcam/macos.md @@ -9,4 +9,4 @@ update installs the new extension without a reboot. ## Required - [Signing identities are available](/quest/m3/studio/signing.md) - system extensions only load when signed -- [The studio app](/quest/m3/studio/app.md) - the program this exposes +- [Studio program encode from the GPU](/quest/m3/studio/app.md) - the studio running on macOS diff --git a/quest/m3/studio/vcam/windows.md b/quest/m3/studio/vcam/windows.md index d57c2025dc4..a56f2762e63 100644 --- a/quest/m3/studio/vcam/windows.md +++ b/quest/m3/studio/vcam/windows.md @@ -8,4 +8,4 @@ Apps list it as a camera while the studio runs. ## Required -- [The studio app](/quest/m3/studio/app.md) - the program this exposes +- [Studio program encode from the GPU](/quest/m3/studio/app.md) - the studio running on Windows diff --git a/quest/m3/studio/web/video-web.md b/quest/m3/studio/web/video-web.md index 5fe8b8e5490..154e61493e8 100644 --- a/quest/m3/studio/web/video-web.md +++ b/quest/m3/studio/web/video-web.md @@ -34,5 +34,4 @@ A CI `cargo check` for wasm32 keeps them building. ## Required -- [The studio app](/quest/m3/studio/app.md) - the app the page builds to wasm - [A runtime trait replaces web-async](/quest/m2/runtime-trait.md) - spawn and time on the browser instead of tokio's runtime diff --git a/rs/moq-audio/MIGRATE.md b/rs/moq-audio/MIGRATE.md index 250129f83dd..51df70a4bf0 100644 --- a/rs/moq-audio/MIGRATE.md +++ b/rs/moq-audio/MIGRATE.md @@ -4,6 +4,10 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `Level` is at the crate root + +`encode::Level` is `moq_audio::Level`, since the program bus's meters report it too. + ### Decoded reads carry gaps and encoder ends (#5687, #6475) `decode::Consumer::read()` returns `decode::Read` events instead of bare frames, the same shape as `moq_video::decode::Read`: `Frame(frame)`, `Gap { timestamp, duration }` for lost media, and `End { end }` where the publisher stopped its encoder; `None` still means the track is finished. diff --git a/rs/moq-audio/src/bus.rs b/rs/moq-audio/src/bus.rs index 61827b0f883..389d11e2925 100644 --- a/rs/moq-audio/src/bus.rs +++ b/rs/moq-audio/src/bus.rs @@ -643,7 +643,10 @@ mod tests { assert!(Bus::new(config).is_err()); let bus = Bus::new(Config::default()).unwrap(); - assert!(bus.channel(input(48_000, Layout::Discrete(3)), Strip::default()).is_err()); + assert!( + bus.channel(input(48_000, Layout::Discrete(3)), Strip::default()) + .is_err() + ); } /// A partial trailing frame in the output is zeroed rather than left with diff --git a/rs/moq-scene/Cargo.toml b/rs/moq-scene/Cargo.toml index 247e1842dcf..8002caec96d 100644 --- a/rs/moq-scene/Cargo.toml +++ b/rs/moq-scene/Cargo.toml @@ -23,7 +23,6 @@ capture = ["moq-video/capture", "moq-audio/capture"] [dependencies] bytemuck = { version = "1", features = ["derive"] } -hang = { workspace = true } humantime-serde = { workspace = true } # PNG and JPEG still images. Already in a studio's graph through eframe. image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } @@ -43,7 +42,6 @@ toml = { workspace = true } tracing = { workspace = true } [dev-dependencies] -pollster = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true, features = ["macros", "rt", "test-util"] } diff --git a/rs/moq-scene/src/collection.rs b/rs/moq-scene/src/collection.rs index 1774bb59efb..77a91829e83 100644 --- a/rs/moq-scene/src/collection.rs +++ b/rs/moq-scene/src/collection.rs @@ -123,7 +123,9 @@ impl Collection { for (name, audio) in &self.audio { if !self.sources.contains_key(name) { - return Err(Error::Invalid(format!("audio settings for `{name}`, which is not a source"))); + return Err(Error::Invalid(format!( + "audio settings for `{name}`, which is not a source" + ))); } if !audio.gain.is_finite() || audio.gain < 0.0 { return Err(Error::Invalid(format!("`{name}` has gain {}", audio.gain))); @@ -334,7 +336,10 @@ impl Source { /// Whether the source feeds the program audio. pub fn has_audio(&self) -> bool { - matches!(self, Self::Microphone { .. } | Self::SystemAudio | Self::Broadcast { .. }) + matches!( + self, + Self::Microphone { .. } | Self::SystemAudio | Self::Broadcast { .. } + ) } } @@ -351,7 +356,10 @@ pub struct Audio { impl Default for Audio { fn default() -> Self { - Self { gain: 1.0, muted: false } + Self { + gain: 1.0, + muted: false, + } } } @@ -394,9 +402,12 @@ mod tests { fn example() -> Collection { let mut collection = Collection::new(); - collection - .sources - .insert("camera".into(), Source::Camera { id: Some("/dev/video0".into()) }); + collection.sources.insert( + "camera".into(), + Source::Camera { + id: Some("/dev/video0".into()), + }, + ); collection.sources.insert( "logo".into(), Source::Image { @@ -491,7 +502,13 @@ items = [ color: Rgba([255, 0, 0, 128]) } ); - assert_eq!(collection.audio["cam2"], Audio { gain: 2.0, muted: false }); + assert_eq!( + collection.audio["cam2"], + Audio { + gain: 2.0, + muted: false + } + ); let main = collection.scene("Main").unwrap(); assert!(main.items[0].visible); assert!(!main.items[1].visible); @@ -511,7 +528,10 @@ items = [ /// next save. #[test] fn refuses_unknown_fields() { - let text = example().to_toml().unwrap().replacen("[canvas]", "[canvas]\ndepth = 3", 1); + let text = example() + .to_toml() + .unwrap() + .replacen("[canvas]", "[canvas]\ndepth = 3", 1); assert!(matches!(Collection::from_toml(&text), Err(Error::Format(_)))); let text = example() @@ -554,7 +574,10 @@ items = [ height, framerate, }; - assert!(matches!(collection.validate(), Err(Error::Invalid(_))), "{width}x{height}@{framerate}"); + assert!( + matches!(collection.validate(), Err(Error::Invalid(_))), + "{width}x{height}@{framerate}" + ); } } diff --git a/rs/moq-scene/src/compositor.rs b/rs/moq-scene/src/compositor.rs index b1256dcd6c9..0b40404b96c 100644 --- a/rs/moq-scene/src/compositor.rs +++ b/rs/moq-scene/src/compositor.rs @@ -8,9 +8,9 @@ use std::time::Duration; use moq_clock::Instant; use moq_video::render::{Renderer, wgpu}; +use crate::Error; use crate::collection::{Canvas, Collection, Rect}; use crate::source::{Picture, Sources}; -use crate::Error; /// The format of every canvas texture: gamma-encoded RGBA, what /// `moq_video::render` produces and what an encoder reads back. @@ -443,7 +443,9 @@ impl Compositor { None => Renderer::new(&self.device, &self.queue, Default::default()) .map(|renderer| layer.renderer.insert(renderer)), }; - renderer.and_then(|renderer| renderer.render(&frame)).map_err(Error::from) + renderer + .and_then(|renderer| renderer.render(&frame)) + .map_err(Error::from) } Picture::Rgba { size, pixels } => upload(&self.device, &self.queue, size, &pixels), }; @@ -476,7 +478,12 @@ fn uniform(rect: Rect, canvas: Canvas, opacity: f32) -> Uniform { } /// A texture holding tightly packed RGBA pixels. -fn upload(device: &wgpu::Device, queue: &wgpu::Queue, size: moq_video::Size, pixels: &[u8]) -> Result { +fn upload( + device: &wgpu::Device, + queue: &wgpu::Queue, + size: moq_video::Size, + pixels: &[u8], +) -> Result { let limit = device.limits().max_texture_dimension_2d; if size.width == 0 || size.height == 0 || size.width > limit || size.height > limit { return Err(Error::Invalid(format!( @@ -639,8 +646,12 @@ mod tests { height: 8, framerate: 30, }; - collection.sources.insert("red".into(), Source::Color { color: Rgba(RED) }); - collection.sources.insert("blue".into(), Source::Color { color: Rgba(BLUE) }); + collection + .sources + .insert("red".into(), Source::Color { color: Rgba(RED) }); + collection + .sources + .insert("blue".into(), Source::Color { color: Rgba(BLUE) }); let item = |source: &str, rect| Item { source: source.into(), rect, diff --git a/rs/moq-scene/src/source.rs b/rs/moq-scene/src/source.rs index f28a6556b75..f4472dbb0ce 100644 --- a/rs/moq-scene/src/source.rs +++ b/rs/moq-scene/src/source.rs @@ -494,7 +494,13 @@ mod tests { start(&sources, drivers).await; let guest = sources.get("guest").unwrap().strip().unwrap().clone(); - collection.audio.insert("guest".into(), Audio { gain: 0.25, muted: true }); + collection.audio.insert( + "guest".into(), + Audio { + gain: 0.25, + muted: true, + }, + ); assert!(sources.sync(&collection).is_empty()); assert_eq!(guest.gain(), 0.25); assert!(guest.muted()); diff --git a/rs/moq-studio/src/engine.rs b/rs/moq-studio/src/engine.rs index a78dab9c56c..4821cab1a34 100644 --- a/rs/moq-studio/src/engine.rs +++ b/rs/moq-studio/src/engine.rs @@ -426,7 +426,9 @@ fn encode( Ok(frame) => frame, Err(err) => return stats.fail(format!("readback: {err}")), }; - stats.readback_us.store(start.elapsed().as_micros() as u64, Ordering::Relaxed); + stats + .readback_us + .store(start.elapsed().as_micros() as u64, Ordering::Relaxed); stats.read_back.fetch_add(1, Ordering::Relaxed); let encoder = match &mut encoder { @@ -459,7 +461,9 @@ fn encode( if let Err(err) = producer.publish(&encoded) { return stats.fail(format!("video: {err}")); } - stats.encode_us.store(start.elapsed().as_micros() as u64, Ordering::Relaxed); + stats + .encode_us + .store(start.elapsed().as_micros() as u64, Ordering::Relaxed); stats.encoded.fetch_add(1, Ordering::Relaxed); } } @@ -467,7 +471,9 @@ fn encode( /// Run a source on its own thread until its feed is dropped. fn spawn(driver: Driver) { let name = format!("source-{}", driver.name()); - let spawned = std::thread::Builder::new().name(name).spawn(move || clocked(driver.run())); + let spawned = std::thread::Builder::new() + .name(name) + .spawn(move || clocked(driver.run())); if let Err(err) = spawned { tracing::error!(%err, "cannot start a source thread"); } diff --git a/rs/moq-studio/src/ui.rs b/rs/moq-studio/src/ui.rs index d2fbe72d41e..18d21d53b63 100644 --- a/rs/moq-studio/src/ui.rs +++ b/rs/moq-studio/src/ui.rs @@ -207,7 +207,10 @@ impl Panels { fn preview(&mut self, ui: &mut Ui) { let state = self.engine.state.lock().unwrap(); let canvas = state.collection.canvas; - ui.label(format!("Preview: {}", state.compositor.preview_scene().unwrap_or("nothing"))); + ui.label(format!( + "Preview: {}", + state.compositor.preview_scene().unwrap_or("nothing") + )); drop(state); picture(ui, self.preview, canvas); } @@ -223,7 +226,10 @@ impl Panels { if ui.button(format!("Fade ({} ms)", fade.as_millis())).clicked() { state.compositor.fade(fade, moq_clock::Instant::now()); } - ui.label(format!("Program: {}", state.compositor.program_scene().unwrap_or("nothing"))); + ui.label(format!( + "Program: {}", + state.compositor.program_scene().unwrap_or("nothing") + )); if state.compositor.fading() { ui.weak("fading"); } @@ -310,8 +316,12 @@ impl Panels { let rect = &mut item.rect; edited |= ui.add(egui::DragValue::new(&mut rect.x)).changed(); edited |= ui.add(egui::DragValue::new(&mut rect.y)).changed(); - edited |= ui.add(egui::DragValue::new(&mut rect.width).range(0.0..=f32::MAX)).changed(); - edited |= ui.add(egui::DragValue::new(&mut rect.height).range(0.0..=f32::MAX)).changed(); + edited |= ui + .add(egui::DragValue::new(&mut rect.width).range(0.0..=f32::MAX)) + .changed(); + edited |= ui + .add(egui::DragValue::new(&mut rect.height).range(0.0..=f32::MAX)) + .changed(); ui.horizontal(|ui| { if ui.small_button("Fill").clicked() { *rect = Rect::fill(&canvas); @@ -320,7 +330,10 @@ impl Panels { if ui.add_enabled(index > 0, egui::Button::new("Back").small()).clicked() { moved = Some((index, index - 1)); } - if ui.add_enabled(index + 1 < count, egui::Button::new("Front").small()).clicked() { + if ui + .add_enabled(index + 1 < count, egui::Button::new("Front").small()) + .clicked() + { moved = Some((index, index + 1)); } if ui.small_button("Remove").clicked() { @@ -500,16 +513,31 @@ impl Panels { ui.label(format!("{} frames", count(&stats.rendered))); ui.end_row(); ui.label("Read back to the CPU"); - ui.label(format!("{} frames, last {}", count(&stats.read_back), millis(&stats.readback_us))); + ui.label(format!( + "{} frames, last {}", + count(&stats.read_back), + millis(&stats.readback_us) + )); ui.end_row(); ui.label("Dropped"); ui.label(format!("{} frames", count(&stats.dropped))); ui.end_row(); ui.label("Encoder"); - ui.label(stats.encoder.lock().unwrap().clone().unwrap_or_else(|| "not open".into())); + ui.label( + stats + .encoder + .lock() + .unwrap() + .clone() + .unwrap_or_else(|| "not open".into()), + ); ui.end_row(); ui.label("Encoded"); - ui.label(format!("{} frames, last {}", count(&stats.encoded), millis(&stats.encode_us))); + ui.label(format!( + "{} frames, last {}", + count(&stats.encoded), + millis(&stats.encode_us) + )); ui.end_row(); }); if let Some(error) = stats.error.lock().unwrap().as_ref() { @@ -772,44 +800,44 @@ impl Devices { return None; }; let mut picked = None; - egui::ScrollArea::vertical().show(ui, |ui| { - for (kind, devices) in &mut listing.kinds { - ui.strong(*kind); - match devices { - Ok(devices) if devices.is_empty() => { - ui.weak("none"); - } - Ok(devices) => { - for (index, device) in devices.iter().enumerate() { - ui.horizontal(|ui| { - if ui.small_button("Add").clicked() { - picked = Some(index); - } - ui.label(&device.name); - }); + egui::ScrollArea::vertical() + .show(ui, |ui| { + for (kind, devices) in &mut listing.kinds { + ui.strong(*kind); + match devices { + Ok(devices) if devices.is_empty() => { + ui.weak("none"); } - if let Some(index) = picked.take() { - return Some(devices.remove(index)); + Ok(devices) => { + for (index, device) in devices.iter().enumerate() { + ui.horizontal(|ui| { + if ui.small_button("Add").clicked() { + picked = Some(index); + } + ui.label(&device.name); + }); + } + if let Some(index) = picked.take() { + return Some(devices.remove(index)); + } + } + Err(err) => { + ui.colored_label(ui.visuals().error_fg_color, err.as_str()); } - } - Err(err) => { - ui.colored_label(ui.visuals().error_fg_color, err.as_str()); } } - } - None - }) - .inner + None + }) + .inner } } #[cfg(feature = "capture")] async fn list() -> Listing { - fn picks( - result: Result, impl std::fmt::Display>, - pick: impl Fn(T) -> Pick, - ) -> Result, String> { - result.map(|devices| devices.into_iter().map(pick).collect()).map_err(|err| err.to_string()) + fn picks(result: Result, impl std::fmt::Display>, pick: impl Fn(T) -> Pick) -> Result, String> { + result + .map(|devices| devices.into_iter().map(pick).collect()) + .map_err(|err| err.to_string()) } let cameras = picks(moq_video::capture::cameras().await, |camera| Pick { From ee2db4ae0c487d7409fe3376a1a920820d226f2a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 10:34:36 -0700 Subject: [PATCH 05/15] fix(moq-audio): unlink capture-only items from default-feature docs Co-Authored-By: Claude Opus 5.5 --- rs/moq-audio/src/bus.rs | 2 +- rs/moq-audio/src/level.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/moq-audio/src/bus.rs b/rs/moq-audio/src/bus.rs index 389d11e2925..bb32650197c 100644 --- a/rs/moq-audio/src/bus.rs +++ b/rs/moq-audio/src/bus.rs @@ -29,7 +29,7 @@ //! # } //! ``` //! -//! Enabled by the default-on `bus` feature; [`Bus::capture`] also needs `capture`. +//! Enabled by the default-on `bus` feature; `Bus::capture` also needs `capture`. use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; use std::sync::{Arc, Mutex, Weak}; diff --git a/rs/moq-audio/src/level.rs b/rs/moq-audio/src/level.rs index 66da6b77057..f9797fc4afc 100644 --- a/rs/moq-audio/src/level.rs +++ b/rs/moq-audio/src/level.rs @@ -1,6 +1,6 @@ /// How loud a stretch of audio was, for drawing a meter. /// -/// Reported by a capture [`Control`](crate::encode::Control) and by a mixing +/// Reported by a capture `encode::Control` and by a mixing /// bus and its strips. Read it at whatever rate the meter draws at. #[derive(Clone, Copy, Debug, Default, PartialEq)] pub struct Level { From 153c2b9987e6e3fbdf3aeabce985cac0c76f2a78 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 10:39:37 -0700 Subject: [PATCH 06/15] fix(moq-studio): mark a pause in the video and cut a keyframe on resume Co-Authored-By: Claude Opus 5.5 --- rs/moq-studio/src/engine.rs | 51 +++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/rs/moq-studio/src/engine.rs b/rs/moq-studio/src/engine.rs index 4821cab1a34..71373f06393 100644 --- a/rs/moq-studio/src/engine.rs +++ b/rs/moq-studio/src/engine.rs @@ -232,7 +232,8 @@ impl Engine { let (video, audio) = match publish { Some(Publish { video, audio, config }) => { - let (frames, pending) = mpsc::sync_channel(Readback::DEPTH); + // One more than the readback pool, for a pause behind a full pool. + let (frames, pending) = mpsc::sync_channel(Readback::DEPTH + 1); tokio::spawn(watch(video.demand(), engine.clone())); let encoder = engine.clone(); std::thread::Builder::new() @@ -258,12 +259,13 @@ impl Engine { /// Draw a canvas frame per tick of the program clock, and hand the program /// to the encoder while anyone watches. - async fn run_program(&self, start: Instant, frames: Option>) { + async fn run_program(&self, start: Instant, frames: Option>) { let canvas = self.state.lock().unwrap().collection.canvas; let mut readback = Readback::new(&self.gpu.device, &self.gpu.queue, canvas); let period = Duration::from_secs(1) / canvas.framerate; let mut clock = moq_clock::interval_at(start, period); clock.set_missed_tick_behavior(moq_clock::MissedTickBehavior::Skip); + let mut watched = false; loop { let tick = clock.tick().await; @@ -284,16 +286,22 @@ impl Engine { self.stats.rendered.fetch_add(1, Ordering::Relaxed); let Some(frames) = &frames else { continue }; - if !self.stats.watched.load(Ordering::Relaxed) { + let was = std::mem::replace(&mut watched, self.stats.watched.load(Ordering::Relaxed)); + if !watched { + // Behind every frame already sent, so the break lands after them. + if was && frames.send(Video::Pause).is_err() { + return; + } continue; } let Ok(timestamp) = moq_net::Timestamp::try_from(tick.saturating_duration_since(start)) else { continue; }; match readback.copy(&program, timestamp) { - // The channel holds as many as the pool, so this never blocks. + // The pool bounds the frames in flight, so this waits only behind + // a pause the encoder has not reached yet. Some(pending) => { - if frames.send(pending).is_err() { + if frames.send(Video::Frame(pending)).is_err() { return; } } @@ -409,18 +417,38 @@ async fn watch(mut demand: moq_net::track::Demand, engine: Arc) { } } +/// What the program clock hands the encoder. +enum Video { + /// A program frame on its way back from the GPU. + Frame(Pending), + /// The last viewer left, so frames stop until one returns. + Pause, +} + /// Read back and encode program frames until the program clock stops. fn encode( engine: &Engine, - pending: mpsc::Receiver, + video: mpsc::Receiver