From 024b5e9225db314e54d3e1403845dbe06cb4cc25 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 23:10:18 -0700 Subject: [PATCH 01/15] quest: claim quest/m1/p2p/webrtc Co-Authored-By: Claude Opus 5.5 From d1b56541dab7e099b4ffe72a4ce15bce9525f46e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 23:47:31 -0700 Subject: [PATCH 02/15] feat(tokio): WebRTC data channel transport over str0m Add a `webrtc` feature to moq-tokio: a full ICE agent (host candidates on every interface, a dual-stack socket, STUN server-reflexive gathering, mDNS resolution of `.local` candidates) and qmux-01 over one ordered data channel, matching @moq/p2p. Signaling stays with the caller through async methods on `webrtc::Peer`; `Client::connect_webrtc` and `Listener::accept_webrtc` run the MoQ session over it. moq-rtc now drives its sessions with the shared `webrtc::ice::Agent` instead of its own run loop, socket reader, and candidate pick. moq-cli enables the feature by default. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 3 + Cargo.toml | 6 + doc/concept/transport.md | 6 + rs/moq-cli/Cargo.toml | 4 +- rs/moq-relay/Cargo.toml | 2 +- rs/moq-rtc/Cargo.toml | 7 +- rs/moq-rtc/src/client/whep.rs | 4 +- rs/moq-rtc/src/client/whip.rs | 4 +- rs/moq-rtc/src/error.rs | 8 - rs/moq-rtc/src/lib.rs | 1 - rs/moq-rtc/src/net.rs | 73 --- rs/moq-rtc/src/server/mux.rs | 23 +- rs/moq-rtc/src/server/whep.rs | 2 +- rs/moq-rtc/src/server/whip.rs | 2 +- rs/moq-rtc/src/session.rs | 294 ++-------- rs/moq-rtc/src/wire.rs | 38 +- rs/moq-tokio/Cargo.toml | 5 + rs/moq-tokio/src/client.rs | 56 +- rs/moq-tokio/src/error.rs | 12 + rs/moq-tokio/src/lib.rs | 7 +- rs/moq-tokio/src/server.rs | 125 ++++- rs/moq-tokio/src/transport.rs | 3 + rs/moq-tokio/src/webrtc.rs | 860 +++++++++++++++++++++++++++++ rs/moq-tokio/src/webrtc/channel.rs | 62 +++ rs/moq-tokio/src/webrtc/ice.rs | 505 +++++++++++++++++ rs/moq-tokio/src/webrtc/tests.rs | 242 ++++++++ sh/rs/tokio-features.sh | 2 +- 27 files changed, 1931 insertions(+), 425 deletions(-) delete mode 100644 rs/moq-rtc/src/net.rs create mode 100644 rs/moq-tokio/src/webrtc.rs create mode 100644 rs/moq-tokio/src/webrtc/channel.rs create mode 100644 rs/moq-tokio/src/webrtc/ice.rs create mode 100644 rs/moq-tokio/src/webrtc/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 05337201151..67de5b682c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5006,6 +5006,7 @@ dependencies = [ "hex", "hmac 0.13.0", "humantime", + "if-addrs", "jni 0.22.4", "libc", "mdns-sd", @@ -5024,6 +5025,8 @@ dependencies = [ "serde_with", "sha2 0.11.0", "socket2 0.6.5", + "str0m", + "stun-types", "subtle", "tempfile", "thiserror 2.0.21", diff --git a/Cargo.toml b/Cargo.toml index 19a3faf7eed..5917d5cdf74 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -133,6 +133,8 @@ hmac = "0.13" humantime = "2.3" humantime-serde = "1.1.1" hyper-util = { version = "0.1.21", features = ["tokio"] } +# Interface addresses for WebRTC host candidates (moq-tokio's `webrtc` feature). +if-addrs = "0.15" io-uring = "0.7.15" jsonwebtoken = "11" kio = { version = "0.6.2", path = "rs/kio" } @@ -222,6 +224,10 @@ serde_with = "3" sha2 = "0.11" slab = "0.4.12" socket2 = "0.6" +# Sans-IO WebRTC. The default `examples` feature pulls in a web server, and aws-lc-rs +# is the provider rustls already brings. +str0m = { version = "0.24.1", default-features = false, features = ["aws-lc-rs"] } +stun-types = "2.0.3" subtle = "2.6" tempfile = "3" thiserror = "2" diff --git a/doc/concept/transport.md b/doc/concept/transport.md index cc0b6d6475a..dd41787939f 100644 --- a/doc/concept/transport.md +++ b/doc/concept/transport.md @@ -71,6 +71,12 @@ records, with no datagrams. The application supplies signaling and ICE and keeps ownership of the peer connection; closing the transport closes only its channel. SCTP ordering means one lost packet can hold back every stream. +Native peers use `moq-tokio`'s `webrtc` feature, which runs its own ICE agent: +host candidates on every interface, server-reflexive candidates from the STUN +servers it is given, and mDNS resolution of a browser's `.local` candidates. +The application still carries the offer, answer, and candidates; the offering +peer connects as the MoQ client and the answering peer accepts. + ## iroh (peer-to-peer, experimental) [iroh](https://www.iroh.computer/) swaps "dial a hostname" for "dial a public diff --git a/rs/moq-cli/Cargo.toml b/rs/moq-cli/Cargo.toml index 9e8d1066930..ab7e81aa362 100644 --- a/rs/moq-cli/Cargo.toml +++ b/rs/moq-cli/Cargo.toml @@ -26,7 +26,7 @@ path = "src/main.rs" doc = false [features] -default = ["cluster-lan", "noq", "websocket", "nvidia", "s3", "gcs", "azure"] +default = ["cluster-lan", "noq", "websocket", "webrtc", "nvidia", "s3", "gcs", "azure"] # Experimental transport with a heavy dependency tree, enabled only on request. iroh = ["moq-tokio/iroh"] # LAN discovery and meshing (`--cluster-lan`). @@ -38,6 +38,8 @@ noq = ["moq-tokio/noq"] # congestion-control testing. qlog = ["moq-tokio/qlog"] websocket = ["moq-tokio/websocket"] +# Native WebRTC data channels, for holding sessions with browser peers. +webrtc = ["moq-tokio/webrtc"] # The cloud object stores `export archive` and `import archive` reach (`s3://`, # `gs://`, `az://`). Local `file://` URLs need none of them. s3 = ["object_store/aws"] diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index 8b1a73eb743..1a80772f970 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -72,7 +72,7 @@ rustls = { version = "0.23", features = ["aws-lc-rs"], default-features = false serde = { workspace = true } serde_json = { workspace = true } serde_with = { workspace = true, features = ["json", "base64"] } -stun-types = "2.0.3" +stun-types = { workspace = true } sysinfo = { version = "0.39", default-features = false, features = ["system"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["full"] } diff --git a/rs/moq-rtc/Cargo.toml b/rs/moq-rtc/Cargo.toml index 50162b504a2..b296ed18008 100644 --- a/rs/moq-rtc/Cargo.toml +++ b/rs/moq-rtc/Cargo.toml @@ -28,10 +28,11 @@ bytes = { workspace = true } hang = { workspace = true } moq-mux = { workspace = true } moq-net = { workspace = true } -# `RedactedUrl` keeps a dialed URL's query token and userinfo out of the logs. -moq-tokio = { workspace = true } +# `RedactedUrl` keeps a dialed URL's query token and userinfo out of the logs, and +# `webrtc::ice` drives each str0m connection over its UDP socket. +moq-tokio = { workspace = true, features = ["webrtc"] } reqwest = { workspace = true, features = ["rustls"] } -str0m = "0.24" +str0m = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } diff --git a/rs/moq-rtc/src/client/whep.rs b/rs/moq-rtc/src/client/whep.rs index 27e11bd4fd1..07c8aa63b47 100644 --- a/rs/moq-rtc/src/client/whep.rs +++ b/rs/moq-rtc/src/client/whep.rs @@ -66,8 +66,8 @@ pub(crate) async fn dial( // 1:1 socket (no demux on the client): pump its datagrams into the session. // The session tags each datagram with the advertised candidate matching its // family (str0m matches the destination against a host candidate, not the bind). - let inbound = session::spawn_socket_reader(socket.clone()); - let mut session = session::Session::ingest(rtc, socket, candidates, inbound, sink); + let inbound = moq_tokio::webrtc::ice::reader(socket.clone()); + let mut session = session::Session::ingest(rtc, socket, candidates, inbound, sink)?; for mid in mids { session.open_media(mid)?; } diff --git a/rs/moq-rtc/src/client/whip.rs b/rs/moq-rtc/src/client/whip.rs index 055992ed14c..9df6023d32c 100644 --- a/rs/moq-rtc/src/client/whip.rs +++ b/rs/moq-rtc/src/client/whip.rs @@ -78,8 +78,8 @@ pub(crate) async fn dial( // 1:1 socket (no demux on the client): pump its datagrams into the session. // The session tags each datagram with the advertised candidate matching its // family (str0m matches the destination against a host candidate, not the bind). - let inbound = session::spawn_socket_reader(socket.clone()); - let mut session = session::Session::egress(rtc, socket, candidates, inbound, source); + let inbound = moq_tokio::webrtc::ice::reader(socket.clone()); + let mut session = session::Session::egress(rtc, socket, candidates, inbound, source)?; for mid in mids { session.open_media(mid)?; } diff --git a/rs/moq-rtc/src/error.rs b/rs/moq-rtc/src/error.rs index e36cf3a7131..a8c4f54fbb2 100644 --- a/rs/moq-rtc/src/error.rs +++ b/rs/moq-rtc/src/error.rs @@ -62,10 +62,6 @@ pub enum Error { #[error("rtc error: {0}")] Rtc(String), - /// Error feeding a received UDP datagram into the WebRTC engine. - #[error("rtc input error: {0}")] - RtcInput(String), - /// An internal video bridge could not be initialized. #[error("video bridge initialization failed")] BridgeFailed, @@ -83,10 +79,6 @@ impl Error { pub(crate) fn rtc(err: impl std::fmt::Display) -> Self { Self::Rtc(err.to_string()) } - - pub(crate) fn rtc_input(err: impl std::fmt::Display) -> Self { - Self::RtcInput(err.to_string()) - } } /// Convenience alias for results from the WebRTC <-> MoQ gateway. diff --git a/rs/moq-rtc/src/lib.rs b/rs/moq-rtc/src/lib.rs index 7075db3947b..e7f05fc51db 100644 --- a/rs/moq-rtc/src/lib.rs +++ b/rs/moq-rtc/src/lib.rs @@ -55,7 +55,6 @@ mod codec; mod egress; mod error; mod ingest; -mod net; mod sdp; mod session; diff --git a/rs/moq-rtc/src/net.rs b/rs/moq-rtc/src/net.rs deleted file mode 100644 index 47f2924e412..00000000000 --- a/rs/moq-rtc/src/net.rs +++ /dev/null @@ -1,73 +0,0 @@ -//! Address-family adaptation at the UDP socket boundary. -//! -//! A dual-stack listener (`[::]`, which Linux serves both families from unless -//! `bindv6only` is set) reports an IPv4 peer as an IPv4-mapped IPv6 address -//! (`::ffff:a.b.c.d`), and refuses a send to a real `V4` destination. Neither -//! form should leak past this boundary: -//! -//! - **Inbound**: [`canonical`] unmaps, so an IPv4 peer looks like the `V4` it -//! is. ICE pairing keys off the address family (see -//! [`pick_local`](crate::session::pick_local)), so without this an IPv4 peer -//! on a dual-stack socket would be paired against the IPv6 host candidate. -//! - **Outbound**: [`to_family`] re-maps, because the destination str0m hands -//! back is the canonical address we fed it. - -use std::net::{IpAddr, SocketAddr}; - -/// Unmap an IPv4-mapped peer address to the `V4` it really is; other addresses -/// pass through unchanged. -pub(crate) fn canonical(addr: SocketAddr) -> SocketAddr { - SocketAddr::new(addr.ip().to_canonical(), addr.port()) -} - -/// Adapt a destination to what `socket_is_v6`'s socket will accept: an AF_INET6 -/// socket can only send to an IPv6 address, so an IPv4 destination has to go -/// back to its mapped form. -pub(crate) fn to_family(dst: SocketAddr, socket_is_v6: bool) -> SocketAddr { - match (socket_is_v6, dst.ip()) { - (true, IpAddr::V4(ip)) => SocketAddr::new(IpAddr::V6(ip.to_ipv6_mapped()), dst.port()), - _ => dst, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn canonical_unmaps_v4_mapped_peers() { - // What a dual-stack socket reports for an IPv4 peer. - let mapped: SocketAddr = "[::ffff:1.2.3.4]:5000".parse().unwrap(); - assert_eq!(canonical(mapped), "1.2.3.4:5000".parse().unwrap()); - } - - #[test] - fn canonical_passes_through_native_addresses() { - let v4: SocketAddr = "1.2.3.4:5000".parse().unwrap(); - let v6: SocketAddr = "[2001:db8::1]:5000".parse().unwrap(); - assert_eq!(canonical(v4), v4); - assert_eq!(canonical(v6), v6); - } - - #[test] - fn to_family_remaps_v4_for_a_dual_stack_socket() { - let v4: SocketAddr = "1.2.3.4:5000".parse().unwrap(); - assert_eq!(to_family(v4, true), "[::ffff:1.2.3.4]:5000".parse().unwrap()); - } - - #[test] - fn to_family_leaves_matching_families_alone() { - let v4: SocketAddr = "1.2.3.4:5000".parse().unwrap(); - let v6: SocketAddr = "[2001:db8::1]:5000".parse().unwrap(); - assert_eq!(to_family(v4, false), v4); - assert_eq!(to_family(v6, true), v6); - } - - /// The round trip a dual-stack server does per IPv4 peer: canonicalize the - /// source on the way in, then re-map str0m's destination on the way out. - #[test] - fn canonical_and_to_family_round_trip() { - let mapped: SocketAddr = "[::ffff:1.2.3.4]:5000".parse().unwrap(); - assert_eq!(to_family(canonical(mapped), true), mapped); - } -} diff --git a/rs/moq-rtc/src/server/mux.rs b/rs/moq-rtc/src/server/mux.rs index b7d68527702..3eb0e8baf41 100644 --- a/rs/moq-rtc/src/server/mux.rs +++ b/rs/moq-rtc/src/server/mux.rs @@ -23,12 +23,13 @@ use std::collections::HashMap; use std::net::SocketAddr; use std::sync::{Arc, Mutex}; +use moq_tokio::webrtc::ice::{self, Inbound, Packet}; use str0m::ice::{IceCreds, StunMessage}; use tokio::net::UdpSocket; use tokio::sync::mpsc; use crate::Result; -use crate::session::{Inbound, Packet, SESSION_INBOX, advertised_candidates}; +use crate::session::advertised_candidates; /// Per-session routing table, shared between the demux task and the accept path. #[derive(Default)] @@ -44,7 +45,7 @@ struct Registry { struct Entry { ufrag: String, sender: mpsc::Sender, - wire: crate::wire::Source, + wire: ice::Wire, } /// The shared media socket plus its demux routing table. @@ -124,8 +125,8 @@ impl Mux { /// [`Registration`] guard the session task must hold for its lifetime. pub(crate) fn register(&self) -> (IceCreds, Inbound, Registration) { let creds = IceCreds::new(); - let (tx, rx) = mpsc::channel(SESSION_INBOX); - let wire = crate::wire::Source::new(); + let (tx, rx) = mpsc::channel(ice::INBOX); + let wire = ice::Wire::default(); let entry = Arc::new(Entry { ufrag: creds.ufrag.clone(), sender: tx, @@ -170,8 +171,8 @@ async fn demux(socket: Arc, registry: Arc>) { } }; // A dual-stack bind reports IPv4 peers as `::ffff:a.b.c.d`; unmap before - // this address becomes a registry key or reaches ICE (see crate::net). - let src = crate::net::canonical(src); + // this address becomes a registry key or reaches ICE (see `ice::canonical`). + let src = ice::canonical(src); let data = &buf[..len]; // Cached peers need one routing lock; only a new address parses STUN. @@ -254,7 +255,7 @@ mod tests { let address_b = "127.0.0.1:10002".parse().unwrap(); let mut registry = mux.registry.lock().unwrap(); assert!(registry.route(100, address_a, Some("unknown")).is_none()); - for _ in 0..SESSION_INBOX { + for _ in 0..ice::INBOX { let entry = registry.route(11, address_a, Some(&a.ufrag)).unwrap(); entry.sender.try_send((vec![0; 11], address_a)).unwrap(); } @@ -267,12 +268,12 @@ mod tests { drop(registry); assert_eq!( source_a.stats(), - crate::wire::Stats { + ice::Stats { tx: 0, - rx: 11 * SESSION_INBOX as u64 + 23 + rx: 11 * ice::INBOX as u64 + 23 } ); - assert_eq!(source_b.stats(), crate::wire::Stats { tx: 0, rx: 37 }); + assert_eq!(source_b.stats(), ice::Stats { tx: 0, rx: 37 }); // Unregister even while an inbox is still held: cached addresses cannot // keep attributing traffic after the peer ends. @@ -299,7 +300,7 @@ mod tests { let mut addresses = Vec::new(); for peer in 0..peers { let (sender, receiver) = mpsc::channel(1); - let wire = crate::wire::Source::new(); + let wire = ice::Wire::default(); let address = SocketAddr::from(([127, 0, 0, 1], peer as u16 + 1)); let entry = Arc::new(Entry { ufrag: peer.to_string(), diff --git a/rs/moq-rtc/src/server/whep.rs b/rs/moq-rtc/src/server/whep.rs index c309b1d9c72..ca69a2c872f 100644 --- a/rs/moq-rtc/src/server/whep.rs +++ b/rs/moq-rtc/src/server/whep.rs @@ -159,7 +159,7 @@ pub async fn accept( let answer = rtc.sdp_api().accept_offer(offer).map_err(Error::rtc)?; let resource_id = sdp::new_resource_id(); - let session = session::Session::egress(rtc, mux.socket(), mux.candidates().to_vec(), inbound, source); + let session = session::Session::egress(rtc, mux.socket(), mux.candidates().to_vec(), inbound, source)?; // Register before returning so a DELETE that races startup still finds the // session; Response::run unregisters itself when it ends. diff --git a/rs/moq-rtc/src/server/whip.rs b/rs/moq-rtc/src/server/whip.rs index 1663ffc748e..5f0241af3a5 100644 --- a/rs/moq-rtc/src/server/whip.rs +++ b/rs/moq-rtc/src/server/whip.rs @@ -153,7 +153,7 @@ pub async fn accept( handle.announce(moq_net::origin::Route::default().with_epoch(epoch.clone()))?; let resource_id = sdp::new_resource_id(); - let session = session::Session::ingest(rtc, mux.socket(), mux.candidates().to_vec(), inbound, sink); + let session = session::Session::ingest(rtc, mux.socket(), mux.candidates().to_vec(), inbound, sink)?; // Register before returning so a DELETE that races the first packet still // finds the session; Response::run unregisters itself when it ends. diff --git a/rs/moq-rtc/src/session.rs b/rs/moq-rtc/src/session.rs index 90c39fb3310..49059316d8b 100644 --- a/rs/moq-rtc/src/session.rs +++ b/rs/moq-rtc/src/session.rs @@ -1,9 +1,7 @@ //! str0m session driver shared by every HTTP role / media direction. //! -//! str0m is sans-IO, so we drive the [`str0m::Rtc`] instance from a tokio -//! task that owns a UDP socket. [`Session::run`] alternates between -//! [`Rtc::poll_output`] (drain pending transmits / events) and -//! [`Rtc::handle_input`] (feed UDP packets or timeouts). +//! [`moq_tokio::webrtc::ice::Agent`] drives the sans-IO [`str0m::Rtc`] over UDP; +//! [`Session::run`] feeds it egress writes and handles the media events it surfaces. //! //! The session itself doesn't care whether the [`Rtc`] was populated by //! accepting an SDP offer (server side) or by minting one and posting it @@ -13,53 +11,22 @@ use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::sync::Arc; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; -use str0m::{Candidate, Event, IceConnectionState, Input, Output, Rtc, net::Receive}; +use moq_tokio::webrtc::ice; +use str0m::{Candidate, Event, Rtc}; use tokio::net::UdpSocket; use tokio::sync::mpsc; use crate::egress::{EgressClock, EgressSource, WriteRequest}; use crate::{Error, Result, codec}; -/// One inbound UDP datagram plus its source address, the unit fed to a session. -/// The [`server`](crate::server) paths get these from the shared-socket demux -/// (`crate::server::mux`); the client paths get them from a 1:1 reader -/// ([`spawn_socket_reader`]). -pub(crate) type Packet = (Vec, SocketAddr); - -pub(crate) struct Inbound { - pub packets: mpsc::Receiver, - pub wire: crate::wire::Source, -} - -/// Bound on a session's inbound datagram queue, sized like a socket buffer: -/// past this, datagrams are dropped rather than buffered (WebRTC tolerates loss -/// and a stalled session must not grow memory without limit). -pub(crate) const SESSION_INBOX: usize = 256; - /// str0m's outbound video buffer depth (packets), which also backs NACK resends. /// Raised above the str0m default (1000) so a late-joining peer can recover a /// large keyframe and the rest of the current group via NACK; see /// [`rtc_config_with_codecs`]. const EGRESS_SEND_BUFFER_VIDEO: usize = 3000; -/// Backstop deadline for a session to reach a connected ICE state, covering the one -/// case str0m's ICE agent deliberately never times out: a peer that answers the SDP -/// but provides NO remote candidates and sends nothing (an abandoned WHIP/WHEP -/// offer, or a probe that only exercises signalling). str0m DOES end a connection -/// whose candidate pairs were tried and exhausted -- the agent goes to -/// `IceConnectionState::Disconnected` (handled in `handle_event`) after -/// ~`StunTiming::timeout()` (~21s at the defaults). But when `remote_candidates` -/// stays empty the agent treats the session as "still possible" forever (trickle -/// ICE: more candidates could arrive), so it sits in `Checking` indefinitely, -/// pinning this task, its broadcast announcement, and its mux registration. Nothing -/// upstream ends it, so we do. Set ABOVE str0m's ~21s pair-exhaustion so a -/// connection that actually started checks is ended by str0m's native path (and a -/// slow-but-real TURN/lossy peer isn't clipped); this only fires for the -/// never-any-candidate case. -const ICE_ESTABLISH_TIMEOUT: Duration = Duration::from_secs(30); - /// Receives `MediaData` events from str0m and dispatches to the right codec /// [`Bridge`](codec::Bridge). Used as the per-session sink in [`Session::run`] /// for any flow where RTP arrives from the peer (`server publish` / WHIP @@ -114,23 +81,7 @@ impl MediaFlow { /// `inbound` rather than being read off the socket directly, so several /// sessions can share one socket behind the `crate::server::mux`. pub struct Session { - rtc: Rtc, - /// Send side. Shared across sessions on the server (the mux socket); owned - /// 1:1 on the client. Receiving happens via `inbound`, not this socket. - socket: Arc, - /// The local ICE candidates we advertised. Each inbound datagram is tagged - /// (for str0m) with the candidate whose address family matches the packet's - /// source, so a dual-stack peer reaching us over IPv6 isn't told the packet - /// arrived on an IPv4 host candidate. MUST be the advertised candidates, not - /// the socket's bind address: str0m drops a STUN binding request whose - /// destination doesn't match a host candidate ("unknown interface"), and the - /// shared mux socket binds a wildcard (`0.0.0.0`) while advertising concrete - /// IPs. Never empty (falls back to the bound address). - locals: Vec, - /// Inbound datagrams routed to this session (demux on the server, a 1:1 - /// reader on the client). `None` from `recv` means every sender dropped, so - /// the session is done. - inbound: Inbound, + agent: ice::Agent, role: MediaRole, opened_media: HashSet, /// Egress write requests. `Some` only for [`MediaRole::Egress`] @@ -147,52 +98,48 @@ pub struct Session { impl Session { /// Convenience for the ingest case (WHIP server, WHEP client). `locals` are the - /// advertised ICE candidates (see the field docs), not the socket bind. + /// advertised ICE candidates, not the socket bind (see [`ice::Agent::new`]). + /// `inbound` comes from the server mux, or [`ice::reader`] for a 1:1 client socket. pub fn ingest( rtc: Rtc, socket: Arc, locals: Vec, - inbound: Inbound, + inbound: ice::Inbound, sink: Box, - ) -> Self { - Self { - rtc, - socket, - locals, - inbound, + ) -> Result { + Ok(Self { + agent: ice::Agent::new(rtc, socket, locals, inbound)?, role: MediaRole::Ingest(sink), opened_media: HashSet::new(), writes_rx: None, ingest_clock: IngestClock::default(), egress_clock: EgressClock::default(), - } + }) } /// Convenience for the egress case (WHEP server, WHIP client). `locals` are the - /// advertised ICE candidates (see the field docs), not the socket bind. + /// advertised ICE candidates, not the socket bind (see [`ice::Agent::new`]). + /// `inbound` comes from the server mux, or [`ice::reader`] for a 1:1 client socket. pub fn egress( rtc: Rtc, socket: Arc, locals: Vec, - inbound: Inbound, + inbound: ice::Inbound, mut source: EgressSource, - ) -> Self { + ) -> Result { let writes_rx = source.take_writes(); - Self { - rtc, - socket, - locals, - inbound, + Ok(Self { + agent: ice::Agent::new(rtc, socket, locals, inbound)?, role: MediaRole::Egress(Box::new(source)), opened_media: HashSet::new(), writes_rx: Some(writes_rx), ingest_clock: IngestClock::default(), egress_clock: EgressClock::default(), - } + }) } pub(crate) fn wire(&self) -> crate::wire::Source { - self.inbound.wire.clone() + crate::wire::Source::new(self.agent.wire().clone()) } /// Open a locally offered media line after its answer has been applied. @@ -215,96 +162,29 @@ impl Session { } async fn run_loop(&mut self) -> Result<()> { - let started = Instant::now(); - let mut connected = false; - // str0m hands back the canonical destination we fed it, so a dual-stack - // socket needs IPv4 re-mapped before each send (see crate::net). - let socket_v6 = self.socket.local_addr().map_err(Error::Io)?.is_ipv6(); loop { - // A dead Rtc (DTLS/SDP failure, explicit disconnect) makes poll_output - // return a never-firing timeout instead of erroring, which would hang - // this task forever holding the broadcast announcement + mux - // registration. Bail so those release. - if !self.rtc.is_alive() { - return Err(Error::SessionClosed); - } - - // Abort a session that never finishes connecting (see - // ICE_ESTABLISH_TIMEOUT); once connected, str0m's own timeouts take over. - if !connected && started.elapsed() >= ICE_ESTABLISH_TIMEOUT { - return Err(Error::IceTimeout); - } - - let timeout = match self.rtc.poll_output().map_err(Error::rtc)? { - Output::Timeout(t) => t, - Output::Transmit(t) => { - let dst = crate::net::to_family(t.destination, socket_v6); - if let Err(err) = transmit(&self.socket, &self.inbound.wire, &t.contents, dst).await { - tracing::warn!(%err, %dst, "send failed"); - } - continue; - } - Output::Event(event) => { - if let Event::IceConnectionStateChange(state) = &event { - connected |= state.is_connected(); - } - self.handle_event(event)?; - continue; - } - }; - - let now = Instant::now(); - let mut duration = timeout.saturating_duration_since(now); - // While still connecting, never sleep past the establishment deadline, so - // the check above fires on time even if str0m scheduled a far-off timeout. - if !connected { - duration = duration.min(ICE_ESTABLISH_TIMEOUT.saturating_sub(started.elapsed())); - } - if duration.is_zero() { - self.rtc.handle_input(Input::Timeout(now)).map_err(Error::rtc)?; - continue; - } - - // Wait for the earliest of: an inbound UDP packet, an egress - // write request (if egress), or the str0m-requested timeout. tokio::select! { + // Egress writes get drained promptly; `Agent::next` is cancel safe. biased; - // Egress writes get drained promptly. Without `biased` an - // idle socket select could starve them. Some(req) = async { match self.writes_rx.as_mut() { Some(rx) => rx.recv().await, None => std::future::pending::>().await, } } => { - let now = Instant::now(); - let wallclock = self.egress_clock.wallclock(req.time, now); - crate::egress::dispatch(&mut self.rtc, req, wallclock); - } - - packet = self.inbound.packets.recv() => { - match packet { - Some((data, src)) => { - let now = Instant::now(); - // Tag the packet with the advertised candidate matching its - // address family, not the socket bind (see the `locals` docs). - let local = pick_local(&self.locals, src); - let recv = Receive::new(str0m::net::Protocol::Udp, src, local, &data) - .map_err(Error::rtc_input)?; - self.rtc.handle_input(Input::Receive(now, recv)).map_err(Error::rtc)?; - } - // Every sender dropped: the demux unregistered us (or the - // 1:1 reader stopped). Nothing more will arrive, so end. - None => return Err(Error::SessionClosed), - } + let wallclock = self.egress_clock.wallclock(req.time, Instant::now()); + crate::egress::dispatch(self.agent.rtc(), req, wallclock); } - _ = tokio::time::sleep(duration) => { - self.rtc - .handle_input(Input::Timeout(Instant::now())) - .map_err(Error::rtc)?; - } + event = self.agent.next() => match event { + Ok(ice::Event::Rtc(event)) => self.handle_event(event)?, + // No STUN servers are configured, so there is nothing to trickle. + Ok(_) => {} + Err(ice::Error::Closed) => return Err(Error::SessionClosed), + Err(ice::Error::Timeout) => return Err(Error::IceTimeout), + Err(err) => return Err(Error::rtc(err)), + }, } } } @@ -313,9 +193,6 @@ impl Session { match event { Event::IceConnectionStateChange(state) => { tracing::debug!(?state, "ice state"); - if state == IceConnectionState::Disconnected { - return Err(Error::SessionClosed); - } } Event::MediaAdded(added) => self.handle_media(added.mid)?, Event::MediaData(data) => { @@ -356,7 +233,7 @@ impl Session { // str0m's CodecConfig is the negotiated set; pick the first // codec advertised for this `mid`. - let Some(media) = self.rtc.media(mid) else { + let Some(media) = self.agent.rtc().media(mid) else { tracing::warn!(?mid, "negotiated media is missing; ignoring"); return Ok(()); }; @@ -369,7 +246,15 @@ impl Session { return Ok(()); } let pt = media.remote_pts().first().copied(); - let params = pt.and_then(|pt| self.rtc.codec_config().params().iter().find(|p| p.pt() == pt).copied()); + let params = pt.and_then(|pt| { + self.agent + .rtc() + .codec_config() + .params() + .iter() + .find(|p| p.pt() == pt) + .copied() + }); let params = match params { Some(p) => p, None => { @@ -398,12 +283,6 @@ impl Session { } } -async fn transmit(socket: &UdpSocket, wire: &crate::wire::Source, data: &[u8], dst: SocketAddr) -> std::io::Result<()> { - let len = socket.send_to(data, dst).await?; - wire.sent(len); - Ok(()) -} - /// Per-session clock that rebases each ingested track's raw RTP timestamps onto /// one timeline so audio and video stay in sync. /// @@ -550,18 +429,6 @@ pub(crate) fn log_session_end(role: &str, result: &Result<()>) { } } -/// Pick the advertised local candidate to tag an inbound packet with: the first -/// one whose address family matches `src`, falling back to the first candidate -/// (the list is never empty). Keeps a dual-stack peer's packets tagged with a -/// same-family host candidate so str0m's ICE pairing stays consistent. -fn pick_local(locals: &[SocketAddr], src: SocketAddr) -> SocketAddr { - locals - .iter() - .find(|l| l.is_ipv4() == src.is_ipv4()) - .copied() - .unwrap_or(locals[0]) -} - /// Convert a str0m [`MediaTime`](str0m::media::MediaTime) to microseconds. fn media_time_to_micros(time: &str0m::media::MediaTime) -> u64 { // MediaTime stores `numer / denom` seconds; cast through i128 so the @@ -649,7 +516,7 @@ pub fn rtc_with_codecs(codecs: &[str0m::format::Codec]) -> Rtc { } /// Bind an ephemeral UDP socket for a single client session and return it -/// (shared with its [reader task](spawn_socket_reader)) plus the ICE candidates +/// (to share with its [`ice::reader`]) plus the ICE candidates /// to advertise. /// /// The client paths are 1:1 (one socket per dialed session, no demux); the @@ -690,41 +557,6 @@ pub(crate) fn advertised_candidates(advertise: &[SocketAddr], local: SocketAddr) Ok(candidates) } -/// Spawn a 1:1 reader pumping every datagram from `socket` into a channel, for -/// the client paths (one socket per session, so no demux is needed). Mirrors the -/// inbound side of `crate::server::mux` for a single session. -pub fn spawn_socket_reader(socket: Arc) -> Inbound { - let (tx, rx) = mpsc::channel(SESSION_INBOX); - let wire = crate::wire::Source::new(); - let source = wire.clone(); - tokio::spawn(async move { - let mut buf = vec![0u8; 65_535]; - loop { - let packet = tokio::select! { - biased; - _ = tx.closed() => break, - packet = socket.recv_from(&mut buf) => packet, - }; - match packet { - // Bounded like a socket buffer: drop on full, stop once the - // session's receiver is gone. - Ok((len, src)) => { - source.received(len); - let src = crate::net::canonical(src); - if let Err(mpsc::error::TrySendError::Closed(_)) = tx.try_send((buf[..len].to_vec(), src)) { - break; - } - } - Err(err) => { - tracing::warn!(%err, "webrtc client socket recv failed"); - break; - } - } - } - }); - Inbound { packets: rx, wire } -} - #[cfg(test)] mod tests { use std::time::{Duration, UNIX_EPOCH}; @@ -735,33 +567,6 @@ mod tests { use super::*; - #[tokio::test] - async fn wire_tx_counts_only_successful_udp_payload() { - let socket = UdpSocket::bind("127.0.0.1:0").await.unwrap(); - let peer = UdpSocket::bind("127.0.0.1:0").await.unwrap(); - let wire = crate::wire::Source::new(); - let packet = [42; 73]; - transmit(&socket, &wire, &packet, peer.local_addr().unwrap()) - .await - .unwrap(); - let mut buffer = [0; 100]; - let (len, _) = tokio::time::timeout(Duration::from_secs(5), peer.recv_from(&mut buffer)) - .await - .unwrap() - .unwrap(); - assert_eq!(&buffer[..len], &packet); - assert_eq!(wire.stats(), crate::wire::Stats { tx: 73, rx: 0 }); - // An oversized datagram is refused by UDP, never counted as sent. - assert!( - transmit(&socket, &wire, &vec![0; 65_536], peer.local_addr().unwrap()) - .await - .is_err() - ); - assert_eq!(wire.stats().tx, 73); - drop(socket); - assert_eq!(wire.stats().tx, 73); - } - #[test] fn media_flow_respects_negotiated_direction() { use str0m::media::Direction::{Inactive, RecvOnly, SendOnly, SendRecv}; @@ -815,19 +620,6 @@ mod tests { assert!(advertised_candidates(&advertised, local).is_err()); } - #[test] - fn pick_local_matches_address_family() { - let v4: SocketAddr = "1.2.3.4:5000".parse().unwrap(); - let v6: SocketAddr = "[2001:db8::1]:5000".parse().unwrap(); - let locals = vec![v4, v6]; - let src_v4: SocketAddr = "9.9.9.9:1".parse().unwrap(); - let src_v6: SocketAddr = "[2001:db8::2]:1".parse().unwrap(); - assert_eq!(pick_local(&locals, src_v4), v4); - assert_eq!(pick_local(&locals, src_v6), v6); - // No same-family candidate falls back to the first. - assert_eq!(pick_local(&[v4], src_v6), v4); - } - #[test] fn ingest_clock_rebases_first_frame_to_zero() { let mut clock = IngestClock::default(); diff --git a/rs/moq-rtc/src/wire.rs b/rs/moq-rtc/src/wire.rs index d8246dc6bcc..2cf816028db 100644 --- a/rs/moq-rtc/src/wire.rs +++ b/rs/moq-rtc/src/wire.rs @@ -1,47 +1,21 @@ //! Cumulative UDP payload bytes for one WebRTC peer. -use std::sync::{ - Arc, - atomic::{AtomicU64, Ordering}, -}; +use moq_tokio::webrtc::ice; /// A peer's cumulative datagram bytes, excluding IP and UDP headers. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct Stats { - /// Bytes successfully handed to the UDP socket, including control and retransmissions. - pub tx: u64, - /// Bytes routed to this peer, including control and packets dropped by its full inbox. - pub rx: u64, -} +pub use ice::Stats; /// A read-only source whose counters survive the WebRTC session ending. #[derive(Clone, Debug)] -pub struct Source(Arc); - -#[derive(Debug, Default)] -struct Counters { - tx: AtomicU64, - rx: AtomicU64, -} +pub struct Source(ice::Wire); impl Source { - pub(crate) fn new() -> Self { - Self(Arc::default()) + pub(crate) fn new(wire: ice::Wire) -> Self { + Self(wire) } /// Read cumulative sent and received UDP payload bytes; neither read consumes them. pub fn stats(&self) -> Stats { - Stats { - tx: self.0.tx.load(Ordering::Relaxed), - rx: self.0.rx.load(Ordering::Relaxed), - } - } - - pub(crate) fn sent(&self, bytes: usize) { - self.0.tx.fetch_add(bytes as u64, Ordering::Relaxed); - } - - pub(crate) fn received(&self, bytes: usize) { - self.0.rx.fetch_add(bytes as u64, Ordering::Relaxed); + self.0.stats() } } diff --git a/rs/moq-tokio/Cargo.toml b/rs/moq-tokio/Cargo.toml index 45ad002eabc..24a0b9ac20b 100644 --- a/rs/moq-tokio/Cargo.toml +++ b/rs/moq-tokio/Cargo.toml @@ -44,6 +44,8 @@ tcp = ["_transport", "dep:qmux", "dep:dns-lookup"] # Unix-domain-socket qmux transport (`unix://`), unix-only. Adds peer-credential # inspection on accept. Pulls in `tcp` (shares qmux's stream transport). uds = ["tcp", "qmux/uds"] +# Native WebRTC data channels over str0m, with a full ICE agent (the `webrtc` module). +webrtc = ["_transport", "dep:qmux", "dep:str0m", "dep:stun-types", "dep:if-addrs", "dep:mdns-sd"] ring = ["_certs", "rustls/ring", "rcgen/ring", "web-transport-moq?/ring"] [dependencies] @@ -55,6 +57,7 @@ futures = "0.3" hex = { workspace = true } hmac = { workspace = true, optional = true } humantime = "2.3" +if-addrs = { workspace = true, optional = true } mdns-sd = { workspace = true, optional = true } moq-net = { workspace = true } @@ -75,6 +78,8 @@ rustls-webpki = { version = "0.103", features = ["aws-lc-rs"], optional = true } serde = { version = "1", features = ["derive"] } serde_with = { version = "3", features = ["hex"] } sha2 = { workspace = true, optional = true } +str0m = { workspace = true, optional = true } +stun-types = { workspace = true, optional = true } subtle = { workspace = true, optional = true } thiserror = "2" tikv-jemalloc-ctl = { version = "0.6", optional = true } diff --git a/rs/moq-tokio/src/client.rs b/rs/moq-tokio/src/client.rs index 44cec6bca74..056b1cbb0ac 100644 --- a/rs/moq-tokio/src/client.rs +++ b/rs/moq-tokio/src/client.rs @@ -46,13 +46,7 @@ pub struct Client { /// every transport (passed into the QUIC backend's `connect` and used directly for /// raw TCP/UDS qmux and WebSocket). Resolved once in [`Client::new`] so the ALPN list /// can't diverge between transports. - #[cfg(any( - feature = "noq", - feature = "iroh", - feature = "websocket", - feature = "tcp", - feature = "uds" - ))] + #[cfg(feature = "_transport")] versions: moq_net::Versions, /// The URL from [`connect.url`](crate::connect::Config::url), dialed by [`Client::publish`] / [`Client::consume`]. connect: Option, @@ -144,13 +138,6 @@ impl Client { moq: moq_net::Client::new() .with_versions(versions.clone()) .with_extensions(config.extensions), - #[cfg(any( - feature = "noq", - feature = "iroh", - feature = "websocket", - feature = "tcp", - feature = "uds" - ))] versions, connect: config.url, timeout, @@ -311,6 +298,39 @@ impl Client { Some(self.with_subscriber(origin).connect(url)) } + /// Run a MoQ client session over the data channel an offering [`crate::webrtc::Peer`] opened. + /// + /// Waits for ICE and the channel to come up, bounded by the connect timeout. A + /// peer link cannot be redialed, so there is no reconnect: the session ends with + /// the peer connection. + #[cfg(feature = "webrtc")] + pub async fn connect_webrtc(&self, peer: &crate::webrtc::Peer) -> crate::Result { + let protocol = peer.protocol()?; + if !self.versions.alpns().contains(&protocol) { + return Err(crate::webrtc::Error::Protocol(protocol.to_string()).into()); + } + + let attempt = async { + let opened = peer.channel().await?; + let config = opened.config(); + let session = qmux::Session::connect(opened.channel, config) + .await + .map_err(crate::webrtc::Error::from)?; + // A data channel has no request URI, so a path would ride the SETUP; a peer link has none. + let transport = crate::transport::Session::new(session, moq_net::transport::Binding::InBand); + Ok::<_, Error>(connect_session(&self.moq, transport).await?) + }; + + let session = match self.timeout.is_zero() { + true => attempt.await?, + false => tokio::time::timeout(self.timeout, attempt) + .await + .map_err(|_| Error::ConnectTimeout(self.timeout))??, + }; + tracing::info!(version = %session.version(), transport = %crate::Transport::WebRtc, "connected"); + Ok(session) + } + /// Dial the given URL and complete the MoQ handshake. /// /// Errors if no transport feature is compiled in. @@ -763,13 +783,7 @@ fn race_error(quic: Error, websocket: Error) -> Error { } } -#[cfg(any( - feature = "noq", - feature = "iroh", - feature = "websocket", - feature = "tcp", - all(feature = "uds", unix) -))] +#[cfg(feature = "_transport")] async fn connect_session( client: &moq_net::Client, transport: S, diff --git a/rs/moq-tokio/src/error.rs b/rs/moq-tokio/src/error.rs index e1dbb140643..f73311e42a2 100644 --- a/rs/moq-tokio/src/error.rs +++ b/rs/moq-tokio/src/error.rs @@ -230,6 +230,11 @@ pub enum Error { #[cfg(all(feature = "uds", unix))] #[error(transparent)] Unix(Arc), + + /// The WebRTC data channel transport failed. + #[cfg(feature = "webrtc")] + #[error(transparent)] + WebRtc(Arc), } impl Error { @@ -391,6 +396,13 @@ impl From for Error { } } +#[cfg(feature = "webrtc")] +impl From for Error { + fn from(err: crate::webrtc::Error) -> Self { + Self::WebRtc(Arc::new(err)) + } +} + /// Convenience alias for results produced by this crate. pub type Result = std::result::Result; diff --git a/rs/moq-tokio/src/lib.rs b/rs/moq-tokio/src/lib.rs index 29a9b356e95..9ea3afd48fc 100644 --- a/rs/moq-tokio/src/lib.rs +++ b/rs/moq-tokio/src/lib.rs @@ -7,6 +7,7 @@ //! - TCP via the `tcp://` (plaintext) and `tls://` schemes (qmux; requires `tcp` feature) //! - Unix domain socket via the `unix://` scheme (qmux, peer-credential aware; requires `uds` feature, unix-only) //! - Iroh P2P (requires `iroh` feature) +//! - A WebRTC data channel with a browser peer, signaled by the caller (qmux; requires `webrtc` feature) //! //! See [`Client`] for connecting to relays and [`Server`] for accepting //! connections. The `mdns` feature finds peers to connect to on the local network. @@ -64,15 +65,17 @@ pub mod worker; mod util; #[cfg(feature = "watch")] pub mod watch; +#[cfg(feature = "webrtc")] +pub mod webrtc; #[cfg(feature = "websocket")] pub mod websocket; -/// Session settings for every qmux transport (WebSocket, `tcp://`, `tls://`, `unix://`). +/// Session settings for every qmux transport (WebSocket, `tcp://`, `tls://`, `unix://`, WebRTC). /// /// moq-lite holds a stream open per subscription, so a qmux session grants the same /// stream count as QUIC rather than qmux's default of 100, past which a peer's opens /// wait on credit forever. -#[cfg(any(feature = "websocket", feature = "tcp"))] +#[cfg(any(feature = "websocket", feature = "tcp", feature = "webrtc"))] fn qmux_config() -> qmux::Config { let mut config = qmux::Config::default(); config.max_streams_bidi = quic::DEFAULT_MAX_STREAMS; diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 40675a64696..127ed360df5 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -18,9 +18,19 @@ use url::Url; // the stream listeners hand back an already-built `Request`. #[cfg(any(feature = "noq", feature = "iroh", feature = "websocket"))] use futures::FutureExt; -use futures::future::BoxFuture; -use futures::stream::FuturesUnordered; -use futures::stream::StreamExt; +// The pending handshakes of the transports `Server` accepts itself; WebRTC +// signaling is the caller's, so a WebRTC-only build has none. +#[cfg(any( + feature = "noq", + feature = "iroh", + feature = "websocket", + feature = "tcp", + all(feature = "uds", unix) +))] +use futures::{ + future::BoxFuture, + stream::{FuturesUnordered, StreamExt}, +}; impl crate::listen::Config { /// Build the [`Server`] this config describes, binding its listeners. @@ -227,6 +237,13 @@ pub struct Server { versions: moq_net::Versions, /// From [`crate::listen::Config::resolved_timeout`]. timeout: Option, + #[cfg(any( + feature = "noq", + feature = "iroh", + feature = "websocket", + feature = "tcp", + all(feature = "uds", unix) + ))] accept: FuturesUnordered>>, #[cfg(any(feature = "tcp", all(feature = "uds", unix)))] streams: StreamListeners, @@ -429,6 +446,13 @@ impl Server { Ok(Server { activity, + #[cfg(any( + feature = "noq", + feature = "iroh", + feature = "websocket", + feature = "tcp", + all(feature = "uds", unix) + ))] accept: Default::default(), moq, versions, @@ -538,11 +562,11 @@ impl Server { feature = "tcp", all(feature = "uds", unix) )))] - /// Returns the next partially established session. - /// - /// Panics: no transport feature is compiled in, so nothing can be accepted. + /// Returns the next partially established session: none, since no compiled + /// transport is one this server listens on (WebRTC sessions arrive through + /// [`Listener::accept_webrtc`]). async fn accept_next(&mut self) -> Option { - unreachable!("no transport compiled; enable a QUIC backend, websocket, tcp, or uds feature"); + None } /// The accept-loop health of every listener this server owns that performs a real @@ -879,6 +903,56 @@ impl Listener { self.server.accept_next().await } + /// Read the MoQ SETUP from the data channel the browser opened on an answering + /// [`crate::webrtc::Peer`], leaving admission to the returned [`Request`]. + /// + /// Waits for ICE and the channel to come up, bounded by the handshake timeout. + /// The channel's protocol must be a MoQ ALPN this server speaks. + #[cfg(feature = "webrtc")] + pub async fn accept_webrtc(&self, peer: &crate::webrtc::Peer) -> crate::Result { + use crate::webrtc::Error as WebRtcError; + + if !peer.answering() { + return Err(WebRtcError::Role("only the answering peer adopts a data channel").into()); + } + let server = &self.server; + let active = server + .activity + .enter() + .ok_or(Error::NoBackend("the listener is closing"))?; + let deadline = server + .timeout + .and_then(|timeout| tokio::time::Instant::now().checked_add(timeout)); + + let handshake = async { + let opened = peer.channel().await?; + if !server.versions.alpns().contains(&opened.protocol.as_str()) { + return Err(WebRtcError::Protocol(opened.protocol)); + } + let config = opened.config(); + let protocol = opened.protocol; + let session = qmux::Session::accept(opened.channel, config).await?; + Ok((session, protocol)) + }; + let (session, protocol) = within(deadline, handshake).await?; + + let (request, deadline) = + accept_request(&server.moq, session, moq_net::transport::Binding::InBand, deadline).await?; + Ok(Request { + transport: Transport::WebRtc, + url: None, + authority: None, + identity: None, + link: Link { + alpn: Some(protocol), + ..Default::default() + }, + kind: RequestKind::Qmux(Box::new(request)), + deadline, + active, + }) + } + /// Close every listener and connection, once each QUIC peer has been sent the /// close. /// @@ -979,7 +1053,8 @@ impl Setup { feature = "iroh", feature = "websocket", feature = "tcp", - all(feature = "uds", unix) + all(feature = "uds", unix), + feature = "webrtc" ))] /// Read the peer's MoQ SETUP by `deadline`, closing the session with a timeout /// code if it hasn't arrived. The deadline comes back too, since answering the @@ -1042,7 +1117,8 @@ impl Deadline { feature = "iroh", feature = "websocket", feature = "tcp", - all(feature = "uds", unix) + all(feature = "uds", unix), + feature = "webrtc" ))] fn new(at: tokio::time::Instant, session: S) -> Self where @@ -1085,7 +1161,8 @@ impl Deadline { feature = "iroh", feature = "websocket", feature = "tcp", - all(feature = "uds", unix) + all(feature = "uds", unix), + feature = "webrtc" ))] async fn within( deadline: Option, @@ -1533,7 +1610,12 @@ pub(crate) enum RequestKind { Noq(Box>>), #[cfg(feature = "iroh")] Iroh(Box>>), - #[cfg(any(feature = "tcp", all(feature = "uds", unix), feature = "websocket"))] + #[cfg(any( + feature = "tcp", + all(feature = "uds", unix), + feature = "websocket", + feature = "webrtc" + ))] Qmux(Box>>), } @@ -1613,7 +1695,12 @@ macro_rules! request_ref { RequestKind::Noq($r) => $body, #[cfg(feature = "iroh")] RequestKind::Iroh($r) => $body, - #[cfg(any(feature = "tcp", all(feature = "uds", unix), feature = "websocket"))] + #[cfg(any( + feature = "tcp", + all(feature = "uds", unix), + feature = "websocket", + feature = "webrtc" + ))] RequestKind::Qmux($r) => $body, } }; @@ -1627,7 +1714,12 @@ macro_rules! request_into { RequestKind::Noq($r) => $body, #[cfg(feature = "iroh")] RequestKind::Iroh($r) => $body, - #[cfg(any(feature = "tcp", all(feature = "uds", unix), feature = "websocket"))] + #[cfg(any( + feature = "tcp", + all(feature = "uds", unix), + feature = "websocket", + feature = "webrtc" + ))] RequestKind::Qmux($r) => $body, } }; @@ -1641,7 +1733,12 @@ macro_rules! request_map { RequestKind::Noq($r) => RequestKind::Noq(Box::new($body)), #[cfg(feature = "iroh")] RequestKind::Iroh($r) => RequestKind::Iroh(Box::new($body)), - #[cfg(any(feature = "tcp", all(feature = "uds", unix), feature = "websocket"))] + #[cfg(any( + feature = "tcp", + all(feature = "uds", unix), + feature = "websocket", + feature = "webrtc" + ))] RequestKind::Qmux($r) => RequestKind::Qmux(Box::new($body)), } }; diff --git a/rs/moq-tokio/src/transport.rs b/rs/moq-tokio/src/transport.rs index a87ec2f1c12..8016f4ba1a2 100644 --- a/rs/moq-tokio/src/transport.rs +++ b/rs/moq-tokio/src/transport.rs @@ -35,6 +35,8 @@ pub enum Transport { Unix, /// WebTransport over HTTP/3 on QUIC. WebTransport, + /// A WebRTC data channel using qmux framing. + WebRtc, } impl Transport { @@ -47,6 +49,7 @@ impl Transport { Self::Tcp => "tcp", Self::Unix => "unix", Self::WebTransport => "webtransport", + Self::WebRtc => "webrtc", } } } diff --git a/rs/moq-tokio/src/webrtc.rs b/rs/moq-tokio/src/webrtc.rs new file mode 100644 index 00000000000..2089999435c --- /dev/null +++ b/rs/moq-tokio/src/webrtc.rs @@ -0,0 +1,860 @@ +//! MoQ over one reliable, ordered WebRTC data channel, for sessions with browser peers. +//! +//! A [`Peer`] is one str0m peer connection with a full ICE agent: a dual-stack UDP +//! socket, a host candidate per non-loopback interface address, a server-reflexive +//! candidate from each configured STUN server, and remote `.local` candidates +//! resolved over mDNS. Signaling is the caller's: send the [`Offer`] or [`Answer`] +//! and the local [`Candidate`]s to the remote peer, and feed its replies back. +//! +//! The offering peer opens the data channel, labelled `moq` and carrying the +//! agreed MoQ ALPN as its protocol, and runs the MoQ client over it with +//! [`crate::Client::connect_webrtc`]. The answering peer adopts that channel and +//! runs the MoQ server with [`crate::Listener::accept_webrtc`]. This matches +//! `@moq/p2p`, where `Transport.connect` creates the channel and +//! `Transport.accept` adopts it. +//! +//! The channel carries qmux-01, one record per SCTP message, capped at 16 KiB and +//! the peer's SCTP message limit. Ordered delivery means one lost chunk stalls +//! every stream until SCTP retransmits it. + +use std::net::{IpAddr, SocketAddr}; +use std::str::FromStr; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use bytes::Bytes; +use str0m::Rtc; +use str0m::change::{SdpAnswer, SdpOffer, SdpPendingOffer}; +use str0m::channel::{ChannelConfig, ChannelId, Reliability}; +use tokio::net::UdpSocket; +use tokio::sync::{mpsc, oneshot}; + +mod channel; +pub mod ice; + +/// The data channel label, as `@moq/p2p` creates it. +const LABEL: &str = "moq"; + +/// The largest qmux record, as `@moq/p2p` sends: SCTP flow control is per +/// message, so a smaller record keeps one stream from holding the association. +const RECORD_SIZE: u64 = 16 * 1024; + +/// qmux-01 does not allow a peer to advertise a smaller record limit. +const MIN_RECORD_SIZE: u64 = 16_382; + +/// SCTP's message limit when a description omits `a=max-message-size` (RFC 8841 section 6.1). +const DEFAULT_MAX_MESSAGE_SIZE: u64 = 64 * 1024; + +/// Bytes the channel may hold unsent before qmux's writer waits, as `@moq/p2p` allows. +const SEND_WINDOW: usize = RECORD_SIZE as usize * 4; + +/// How long to wait for a `.local` name to resolve: mdns-sd's first query plus its +/// retransmissions at one and two seconds (RFC 6762 section 5.2). +const MDNS_TIMEOUT: Duration = Duration::from_secs(5); + +/// The default STUN port (RFC 8489 section 18.10). +const STUN_PORT: u16 = 3478; + +/// Errors from a WebRTC peer connection or its data channel. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// Binding the UDP socket or resolving a STUN server failed. + #[error(transparent)] + Io(#[from] std::io::Error), + + /// The peer connection ended. + #[error(transparent)] + Ice(#[from] ice::Error), + + /// A session description failed to parse or apply. + #[error("invalid session description: {0}")] + Sdp(String), + + /// An ICE candidate failed to parse. + #[error("invalid ICE candidate: {0}")] + Candidate(String), + + /// An ICE server URL is malformed or not a `stun:` URL. + #[error("unsupported ICE server: {0}")] + Server(String), + + /// No interface has an address to offer as a host candidate. + #[error("no network interface address to offer as a host candidate")] + NoHost, + + /// A `.local` candidate could not be resolved. + #[error("mDNS resolution failed: {0}")] + Mdns(String), + + /// The call needs the other role: only the offering peer opens the data channel + /// and applies an answer, and only the answering peer adopts a channel. + #[error("wrong role: {0}")] + Role(&'static str), + + /// The data channel was already taken by a session. + #[error("the data channel is already in use")] + Taken, + + /// The channel's protocol is not a MoQ ALPN this endpoint speaks. + #[error("unsupported MoQ protocol: {0:?}")] + Protocol(String), + + /// The peer opened an unordered or partially reliable channel, which qmux cannot run over. + #[error("the data channel must be reliable and ordered")] + Unreliable, + + /// The peer's SCTP message limit is below qmux's minimum record size. + #[error("SCTP message limit {0} is smaller than qmux's minimum record size")] + MessageSize(u64), + + /// The qmux handshake over the channel failed. + #[error("qmux handshake failed: {0}")] + Qmux(String), +} + +crate::error::from_message!(qmux::Error => Qmux); + +/// How a [`Peer`] gathers its local candidates. +#[derive(Clone, Debug, Default)] +#[non_exhaustive] +pub struct Config { + /// STUN servers asked for this host's public address. Empty means LAN only: + /// host candidates and nothing else. + pub stun: Vec, +} + +/// A STUN server, written the way an `RTCIceServer` URL names one: `stun:host[:port]`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Stun { + host: String, + port: u16, +} + +impl FromStr for Stun { + type Err = Error; + + fn from_str(url: &str) -> Result { + let invalid = || Error::Server(url.to_string()); + let (scheme, rest) = url.split_once(':').ok_or_else(invalid)?; + if !scheme.eq_ignore_ascii_case("stun") { + // TURN is out of scope: the relay already carries what a TURN server would. + return Err(invalid()); + } + + let (host, port) = match rest.strip_prefix('[') { + Some(rest) => { + let (host, port) = rest.split_once(']').ok_or_else(invalid)?; + let port = match port { + "" => None, + port => Some(port.strip_prefix(':').ok_or_else(invalid)?), + }; + ( + host.parse::().map_err(|_| invalid())?.to_string(), + port, + ) + } + None => { + let (host, port) = match rest.split_once(':') { + Some((host, port)) => (host, Some(port)), + None => (rest, None), + }; + if host.is_empty() || host.contains(['/', '?', '@', '[', ']']) { + return Err(invalid()); + } + (host.to_string(), port) + } + }; + let port = match port { + Some(port) => port.parse().map_err(|_| invalid())?, + None => STUN_PORT, + }; + Ok(Self { host, port }) + } +} + +impl std::fmt::Display for Stun { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self.host.contains(':') { + true => write!(f, "stun:[{}]:{}", self.host, self.port), + false => write!(f, "stun:{}:{}", self.host, self.port), + } + } +} + +/// An SDP offer, as `RTCSessionDescription.sdp` carries it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Offer(String); + +/// An SDP answer, as `RTCSessionDescription.sdp` carries it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Answer(String); + +/// An ICE candidate, as `RTCIceCandidate.candidate` carries it: `candidate:...`. +/// +/// The connection has one media section, so a browser takes it with `sdpMLineIndex: 0`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Candidate(String); + +macro_rules! text { + ($ty:ident) => { + impl $ty { + /// Wrap the text a browser produced. + pub fn new(text: impl Into) -> Self { + Self(text.into()) + } + + /// The text to hand a browser. + pub fn as_str(&self) -> &str { + &self.0 + } + } + + impl std::fmt::Display for $ty { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } + } + }; +} + +text!(Offer); +text!(Answer); +text!(Candidate); + +/// One side of a WebRTC peer connection, for signaling. +/// +/// A background task drives the connection from creation, so ICE checks and STUN +/// replies are answered while signaling is still in progress. It runs until both +/// this handle and any session over its data channel are gone. Dropping the handle +/// stops signaling but leaves a running session alone. +pub struct Peer { + commands: mpsc::UnboundedSender, + candidates: mpsc::UnboundedReceiver, + /// The ALPN the offerer's channel carries, or `None` for the answering peer. + protocol: Option, +} + +impl Peer { + /// Start the offering side, opening a data channel for MoQ `protocol`, the ALPN + /// the peers agreed through signaling. + pub async fn offer(config: &Config, protocol: &str) -> Result<(Self, Offer), Error> { + Self::offer_from(config, protocol, hosts()?).await + } + + async fn offer_from(config: &Config, protocol: &str, hosts: Vec) -> Result<(Self, Offer), Error> { + if protocol.is_empty() || protocol.contains(char::is_whitespace) { + return Err(Error::Protocol(protocol.to_string())); + } + let (socket, locals) = bind(&hosts)?; + let mut rtc = rtc(&locals)?; + + let mut api = rtc.sdp_api(); + let id = api.add_channel_with_config(ChannelConfig { + label: LABEL.to_string(), + ordered: true, + reliability: Reliability::Reliable, + negotiated: None, + protocol: protocol.to_string(), + }); + let (offer, pending) = api.apply().expect("adding a channel is an SDP change"); + + let role = Role::Offerer { + channel: id, + pending: Some(pending), + }; + let peer = Self::spawn(config, rtc, socket, locals, role, None).await?; + Ok(( + Self { + protocol: Some(protocol.to_string()), + ..peer + }, + Offer(offer.to_sdp_string()), + )) + } + + /// Start the answering side for a browser's offer. + pub async fn answer(config: &Config, offer: &Offer) -> Result<(Self, Answer), Error> { + Self::answer_from(config, offer, hosts()?).await + } + + async fn answer_from(config: &Config, offer: &Offer, hosts: Vec) -> Result<(Self, Answer), Error> { + let (sdp, mdns) = split_mdns(offer.as_str()); + let max_message = max_message_size(offer.as_str())?; + let parsed = SdpOffer::from_sdp_string(&sdp).map_err(|err| Error::Sdp(err.to_string()))?; + + let (socket, locals) = bind(&hosts)?; + let mut rtc = rtc(&locals)?; + let answer = rtc + .sdp_api() + .accept_offer(parsed) + .map_err(|err| Error::Sdp(err.to_string()))?; + + let peer = Self::spawn(config, rtc, socket, locals, Role::Answerer, Some(max_message)).await?; + peer.resolve(mdns); + Ok((peer, Answer(answer.to_sdp_string()))) + } + + async fn spawn( + config: &Config, + rtc: Rtc, + socket: Arc, + locals: Vec, + role: Role, + max_message: Option, + ) -> Result { + let stun = resolve_stun(&config.stun).await; + let inbound = ice::reader(socket.clone()); + let agent = ice::Agent::new(rtc, socket, locals, inbound)?.with_stun(&stun); + + let (commands, rx) = mpsc::unbounded_channel(); + let (candidates_tx, candidates) = mpsc::unbounded_channel(); + let driver = Driver { + agent, + commands: rx, + candidates: Some(candidates_tx), + role, + max_message, + channel: None, + waiter: None, + taken: false, + }; + use tracing::Instrument; + tokio::spawn(driver.run().instrument(tracing::Span::current())); + + Ok(Self { + commands, + candidates, + protocol: None, + }) + } + + /// Apply the browser's answer to this peer's offer. + pub async fn accept(&self, answer: &Answer) -> Result<(), Error> { + if self.protocol.is_none() { + return Err(Error::Role("only the offering peer applies an answer")); + } + let (sdp, mdns) = split_mdns(answer.as_str()); + let max_message = max_message_size(answer.as_str())?; + let parsed = SdpAnswer::from_sdp_string(&sdp).map_err(|err| Error::Sdp(err.to_string()))?; + + let (tx, rx) = oneshot::channel(); + self.send(Command::Answer(parsed, max_message, tx))?; + rx.await.map_err(|_| ice::Error::Closed)??; + self.resolve(mdns); + Ok(()) + } + + /// The next local candidate to send the remote peer, or `None` once gathering is done. + /// + /// Host candidates ride in the description; these are the server-reflexive ones + /// that STUN finds afterwards. + pub async fn candidate(&mut self) -> Option { + self.candidates.recv().await + } + + /// Add a candidate the remote peer sent, resolving a `.local` address over mDNS first. + /// + /// An empty candidate, which a browser sends to mark the end of its candidates, is ignored. + pub async fn add_candidate(&self, candidate: &Candidate) -> Result<(), Error> { + let line = candidate.as_str().trim(); + let line = line.strip_prefix("a=").unwrap_or(line); + if line.is_empty() { + return Ok(()); + } + let candidate = parse_candidate(line).await?; + self.send(Command::Candidate(candidate)) + } + + /// The MoQ ALPN this peer's data channel carries, for the offering side. + pub(crate) fn protocol(&self) -> Result<&str, Error> { + self.protocol + .as_deref() + .ok_or(Error::Role("only the offering peer opens the data channel")) + } + + /// Whether this is the answering side, which adopts the remote peer's channel. + pub(crate) fn answering(&self) -> bool { + self.protocol.is_none() + } + + /// Wait for the data channel to open, and take it. + pub(crate) async fn channel(&self) -> Result { + let (tx, rx) = oneshot::channel(); + self.send(Command::Channel(tx))?; + let mut opened = rx.await.map_err(|_| ice::Error::Closed)??; + // The session keeps the driver alive once this handle is gone. + opened.channel.writer._driver = Some(self.commands.clone()); + Ok(opened) + } + + fn send(&self, command: Command) -> Result<(), Error> { + self.commands.send(command).map_err(|_| ice::Error::Closed.into()) + } + + /// Add the `.local` candidates a description carried once mDNS resolves them. + fn resolve(&self, lines: Vec) { + for line in lines { + let commands = self.commands.clone(); + tokio::spawn(async move { + match parse_candidate(&line).await { + Ok(candidate) => { + let _ = commands.send(Command::Candidate(candidate)); + } + Err(err) => tracing::debug!(%err, "dropped a remote candidate"), + } + }); + } + } +} + +/// A data channel that opened, ready to carry qmux. +pub(crate) struct Opened { + pub channel: channel::Channel, + /// The MoQ ALPN the channel carries. + pub protocol: String, + /// The record size to advertise: 16 KiB, capped by the peer's SCTP message limit. + pub max_record_size: u64, +} + +impl Opened { + /// The qmux session settings for this channel. + pub fn config(&self) -> qmux::Config { + let mut config = crate::qmux_config(); + config.version = qmux::Version::QMux01; + config.protocol = qmux::Protocol::Negotiated(self.protocol.clone()); + config.max_record_size = self.max_record_size; + // The binding has no datagrams. + config.max_datagram_frame_size = 0; + config + } +} + +pub(crate) enum Command { + Answer(SdpAnswer, u64, oneshot::Sender>), + Candidate(str0m::Candidate), + Channel(oneshot::Sender>), +} + +enum Role { + Offerer { + channel: ChannelId, + /// Until the answer is applied. + pending: Option, + }, + Answerer, +} + +/// The open data channel, from the driver's side. +struct Open { + id: ChannelId, + protocol: String, + reads: mpsc::UnboundedSender>, + /// `None` once the session's writer closed. + writes: Option>, + max_record_size: u64, + /// The session's halves, until a session takes them. + halves: Option, +} + +/// Owns the connection and runs it on its own task. +struct Driver { + agent: ice::Agent, + commands: mpsc::UnboundedReceiver, + /// Dropped once gathering finishes, ending [`Peer::candidate`]. + candidates: Option>, + role: Role, + /// The remote peer's SCTP message limit, once its description is applied. + max_message: Option, + channel: Option>, + /// The session waiting for the channel to open. + waiter: Option>>, + /// Whether a session already asked for the channel, which only one may carry. + taken: bool, +} + +impl Driver { + async fn run(mut self) { + let err = loop { + let writable = self.writable(); + let writes = match &mut self.channel { + Some(Ok(Open { + writes: Some(writes), .. + })) => Some(writes), + _ => None, + }; + + tokio::select! { + event = self.agent.next() => match event { + Ok(event) => if let Err(err) = self.event(event) { + break err; + }, + Err(err) => break err.into(), + }, + command = self.commands.recv() => match command { + Some(command) => self.command(command), + // Neither the signaling handle nor a session is left to use the connection. + None => return, + }, + data = next_write(writes), if writable => { + if let Err(err) = self.write(data) { + break err; + } + } + } + }; + + match &err { + Error::Ice(ice::Error::Closed) => tracing::debug!("peer connection closed"), + err => tracing::debug!(%err, "peer connection failed"), + } + if let Some(waiter) = self.waiter.take() { + let _ = waiter.send(Err(err)); + } + } + + /// Whether the channel can take another record without passing [`SEND_WINDOW`]. + fn writable(&mut self) -> bool { + let Some(Ok(Open { + id, writes: Some(_), .. + })) = &self.channel + else { + return false; + }; + let id = *id; + self.agent + .rtc() + .channel(id) + .is_some_and(|mut channel| channel.buffered_amount() < SEND_WINDOW) + } + + /// Hand a record from the session to SCTP, or close the channel when the session's writer closed. + fn write(&mut self, data: Option) -> Result<(), Error> { + let Some(Ok(open)) = &mut self.channel else { + return Ok(()); + }; + let id = open.id; + let Some(data) = data else { + open.writes = None; + self.agent.rtc().direct_api().close_data_channel(id); + return Ok(()); + }; + let Some(mut channel) = self.agent.rtc().channel(id) else { + return Err(ice::Error::Closed.into()); + }; + // The send window keeps the buffer well below str0m's, so a refusal is a bug, not backpressure. + match channel.write(true, &data) { + Ok(true) => Ok(()), + Ok(false) => Err(ice::Error::Rtc("SCTP refused a record within the send window".into()).into()), + Err(err) => Err(ice::Error::Rtc(err.to_string()).into()), + } + } + + fn command(&mut self, command: Command) { + match command { + Command::Answer(answer, max_message, reply) => { + let result = match &mut self.role { + Role::Offerer { pending, .. } => match pending.take() { + Some(pending) => self + .agent + .rtc() + .sdp_api() + .accept_answer(pending, answer) + .map(|()| self.max_message = Some(max_message)) + .map_err(|err| Error::Sdp(err.to_string())), + None => Err(Error::Sdp("the answer was already applied".into())), + }, + Role::Answerer => Err(Error::Role("only the offering peer applies an answer")), + }; + let _ = reply.send(result); + } + Command::Candidate(candidate) => self.agent.rtc().add_remote_candidate(candidate), + Command::Channel(reply) => { + if self.taken { + let _ = reply.send(Err(Error::Taken)); + return; + } + self.taken = true; + self.waiter = Some(reply); + self.deliver(); + } + } + } + + fn event(&mut self, event: ice::Event) -> Result<(), Error> { + let event = match event { + ice::Event::Candidate(candidate) => { + if let Some(candidates) = &self.candidates { + let _ = candidates.send(Candidate(candidate.to_sdp_string())); + } + return Ok(()); + } + ice::Event::Gathered => { + self.candidates = None; + return Ok(()); + } + ice::Event::Rtc(event) => event, + }; + + match event { + str0m::Event::ChannelOpen(id, _) => self.open(id), + str0m::Event::ChannelData(data) => { + let Some(Ok(open)) = &self.channel else { + return Ok(()); + }; + if data.id != open.id { + return Ok(()); + } + let record = if !data.binary { + Err(qmux::Error::ProtocolViolation) + } else if data.data.len() as u64 > open.max_record_size { + Err(qmux::Error::FrameTooLarge) + } else { + Ok(Bytes::from(data.data)) + }; + let _ = open.reads.send(record); + } + str0m::Event::ChannelClose(id) => { + if let Some(Ok(open)) = &self.channel + && open.id == id + { + // The session's reader sees the end of its records. + self.channel = Some(Err(ice::Error::Closed.into())); + } + } + _ => {} + } + Ok(()) + } + + /// Take a channel that opened as the session's, if it is the one this role expects. + fn open(&mut self, id: ChannelId) { + if self.channel.is_some() { + tracing::debug!(?id, "closing an extra data channel"); + self.agent.rtc().direct_api().close_data_channel(id); + return; + } + if let Role::Offerer { channel, .. } = &self.role + && *channel != id + { + self.agent.rtc().direct_api().close_data_channel(id); + return; + } + + let opened = self.adopt(id); + if opened.is_err() { + self.agent.rtc().direct_api().close_data_channel(id); + } + self.channel = Some(opened); + self.deliver(); + } + + fn adopt(&mut self, id: ChannelId) -> Result { + let max_message = self.max_message.unwrap_or(DEFAULT_MAX_MESSAGE_SIZE); + let max_record_size = RECORD_SIZE.min(max_message); + if max_record_size < MIN_RECORD_SIZE { + return Err(Error::MessageSize(max_message)); + } + + let mut channel = self.agent.rtc().channel(id).ok_or(ice::Error::Closed)?; + let config = channel.config().ok_or(Error::Unreliable)?; + if !config.ordered || config.reliability != Reliability::Reliable { + return Err(Error::Unreliable); + } + let protocol = config.protocol.clone(); + channel.set_buffered_amount_low_threshold(SEND_WINDOW / 2); + + let (reads, rx) = mpsc::unbounded_channel(); + let (tx, writes) = mpsc::channel(1); + Ok(Open { + id, + protocol, + reads, + writes: Some(writes), + max_record_size, + halves: Some(channel::Channel { + writer: channel::Writer { + tx: Some(tx), + max: usize::try_from(max_message).unwrap_or(usize::MAX), + _driver: None, + }, + reader: channel::Reader { rx }, + }), + }) + } + + /// Answer a waiting session once the channel opened or failed to. + fn deliver(&mut self) { + let Some(channel) = &mut self.channel else { + return; + }; + let Some(waiter) = self.waiter.take() else { + return; + }; + let result = match channel { + Ok(open) => Ok(Opened { + channel: open.halves.take().expect("only the first session takes the halves"), + protocol: open.protocol.clone(), + max_record_size: open.max_record_size, + }), + Err(err) => Err(match err { + Error::Unreliable => Error::Unreliable, + Error::MessageSize(size) => Error::MessageSize(*size), + _ => ice::Error::Closed.into(), + }), + }; + let _ = waiter.send(result); + } +} + +/// The session's next record, or `None` when its writer closed. Pends without a channel. +async fn next_write(writes: Option<&mut mpsc::Receiver>) -> Option { + match writes { + Some(writes) => writes.recv().await, + None => std::future::pending().await, + } +} + +/// A str0m connection with this socket's host candidates. +fn rtc(locals: &[SocketAddr]) -> Result { + let mut rtc = Rtc::builder().build(Instant::now()); + for &addr in locals { + let candidate = str0m::Candidate::host(addr, "udp").map_err(|err| Error::Candidate(err.to_string()))?; + rtc.add_local_candidate(candidate); + } + Ok(rtc) +} + +/// Every address worth offering as a host candidate: not loopback, and not +/// link-local, which IPv6 cannot use without a scope and str0m refuses for IPv4. +fn hosts() -> Result, Error> { + let mut hosts = Vec::new(); + for interface in if_addrs::get_if_addrs()? { + let ip = interface.ip(); + if interface.is_loopback() || interface.is_link_local() || ip.is_multicast() || hosts.contains(&ip) { + continue; + } + hosts.push(ip); + } + Ok(hosts) +} + +/// Bind a dual-stack socket on an ephemeral port, pairing it with each host address. +fn bind(hosts: &[IpAddr]) -> Result<(Arc, Vec), Error> { + let socket = crate::bind::udp(crate::bind::Udp::new("[::]:0".parse().unwrap())) + .or_else(|_| crate::bind::udp(crate::bind::Udp::new("0.0.0.0:0".parse().unwrap())))?; + socket.set_nonblocking(true)?; + let socket = UdpSocket::from_std(socket)?; + let local = socket.local_addr()?; + + let locals: Vec = hosts + .iter() + .filter(|ip| local.is_ipv6() || ip.is_ipv4()) + .map(|&ip| SocketAddr::new(ip, local.port())) + .collect(); + if locals.is_empty() { + return Err(Error::NoHost); + } + Ok((Arc::new(socket), locals)) +} + +/// Look up each STUN server. One that does not resolve is skipped, since a missing +/// reflexive candidate only costs the paths that needed it. +async fn resolve_stun(servers: &[Stun]) -> Vec { + let mut addrs = Vec::new(); + for server in servers { + match tokio::net::lookup_host((server.host.as_str(), server.port)).await { + Ok(found) => addrs.extend(found), + Err(err) => tracing::warn!(%err, %server, "failed to resolve STUN server"), + } + } + addrs +} + +/// The SCTP message limit a description advertises, where `0` means unlimited. +fn max_message_size(sdp: &str) -> Result { + let Some(value) = sdp + .lines() + .find_map(|line| line.trim().strip_prefix("a=max-message-size:")) + else { + return Ok(DEFAULT_MAX_MESSAGE_SIZE); + }; + match value.trim().parse::() { + Ok(0) => Ok(u64::MAX), + Ok(size) => Ok(size), + Err(_) => Err(Error::Sdp(format!("invalid max-message-size: {value}"))), + } +} + +/// Split a description's `.local` candidates out, since str0m only parses IP addresses. +fn split_mdns(sdp: &str) -> (String, Vec) { + let mut kept = String::with_capacity(sdp.len()); + let mut mdns = Vec::new(); + for line in sdp.split_inclusive('\n') { + if let Some(candidate) = line.trim_end().strip_prefix("a=") + && candidate.starts_with("candidate:") + && candidate_address(candidate).is_some_and(|addr| addr.ends_with(".local")) + { + mdns.push(candidate.to_string()); + continue; + } + kept.push_str(line); + } + (kept, mdns) +} + +/// The connection address of a `candidate:` line (RFC 8839 section 5.1). +fn candidate_address(line: &str) -> Option<&str> { + line.split_whitespace().nth(4) +} + +/// Parse a `candidate:` line, resolving a `.local` address over mDNS first. +async fn parse_candidate(line: &str) -> Result { + let line = match candidate_address(line) { + Some(host) if host.ends_with(".local") => { + let ip = mdns(host).await?; + let mut fields: Vec = line.split_whitespace().map(str::to_string).collect(); + fields[4] = ip.to_string(); + fields.join(" ") + } + _ => line.to_string(), + }; + str0m::Candidate::from_sdp_string(&line).map_err(|err| Error::Candidate(err.to_string())) +} + +/// The process-wide mDNS daemon, started on the first `.local` candidate. +static MDNS: Mutex> = Mutex::new(None); + +/// Resolve a browser's `.local` host name, preferring an IPv4 address. +async fn mdns(host: &str) -> Result { + let daemon = { + let mut daemon = MDNS.lock().unwrap(); + match daemon.as_ref() { + Some(daemon) => daemon.clone(), + None => daemon + .insert(mdns_sd::ServiceDaemon::new().map_err(|err| Error::Mdns(err.to_string()))?) + .clone(), + } + }; + + let name = format!("{host}."); + let timeout = MDNS_TIMEOUT.as_millis() as u64; + let events = daemon + .resolve_hostname(&name, Some(timeout)) + .map_err(|err| Error::Mdns(err.to_string()))?; + let found = loop { + match events.recv_async().await { + Ok(mdns_sd::HostnameResolutionEvent::AddressesFound(_, addrs)) if !addrs.is_empty() => { + let addrs: Vec = addrs.iter().map(|addr| addr.to_ip_addr()).collect(); + break addrs.iter().find(|ip| ip.is_ipv4()).or(addrs.first()).copied(); + } + Ok(mdns_sd::HostnameResolutionEvent::SearchTimeout(_)) | Err(_) => break None, + Ok(_) => {} + } + }; + let _ = daemon.stop_resolve_hostname(&name); + found.ok_or_else(|| Error::Mdns(format!("{host} did not resolve"))) +} + +#[cfg(test)] +mod tests; diff --git a/rs/moq-tokio/src/webrtc/channel.rs b/rs/moq-tokio/src/webrtc/channel.rs new file mode 100644 index 00000000000..a4e70293f8e --- /dev/null +++ b/rs/moq-tokio/src/webrtc/channel.rs @@ -0,0 +1,62 @@ +//! qmux message I/O over one reliable, ordered data channel. +//! +//! Each SCTP message carries one qmux-01 record without its size field, as a +//! WebSocket message does. The peer's driver task owns the channel; these halves +//! only hand messages to and from it. + +use bytes::Bytes; +use tokio::sync::mpsc; + +/// The data channel, as the qmux transport a [`qmux::Session`] runs over. +pub(crate) struct Channel { + pub writer: Writer, + pub reader: Reader, +} + +impl qmux::transport::Transport for Channel { + type Writer = Writer; + type Reader = Reader; + + fn split(self) -> (Writer, Reader) { + (self.writer, self.reader) + } +} + +/// Hands outbound records to the driver, which waits for SCTP to drain before +/// taking more, so a full send window holds back qmux's writer. +pub(crate) struct Writer { + /// `None` once closed: dropping the sender is what tells the driver to close the channel. + pub tx: Option>, + /// The largest message the peer accepts. + pub max: usize, + /// Keeps the driver running while the session outlives the [`super::Peer`]. + pub _driver: Option>, +} + +impl qmux::transport::Writer for Writer { + async fn send(&mut self, data: Bytes) -> Result<(), qmux::Error> { + if data.len() > self.max { + return Err(qmux::Error::FrameTooLarge); + } + let tx = self.tx.as_ref().ok_or(qmux::Error::Closed)?; + tx.send(data).await.map_err(|_| qmux::Error::Closed) + } + + async fn close(&mut self) -> Result<(), qmux::Error> { + self.tx = None; + Ok(()) + } +} + +/// Receives the records the driver read off the channel. +pub(crate) struct Reader { + /// Unbounded because str0m pushes data as it arrives and cannot be paused; qmux + /// flow control is what bounds the bytes a peer can have in flight. + pub rx: mpsc::UnboundedReceiver>, +} + +impl qmux::transport::Reader for Reader { + async fn recv(&mut self) -> Result { + self.rx.recv().await.unwrap_or(Err(qmux::Error::Closed)) + } +} diff --git a/rs/moq-tokio/src/webrtc/ice.rs b/rs/moq-tokio/src/webrtc/ice.rs new file mode 100644 index 00000000000..0d31e456334 --- /dev/null +++ b/rs/moq-tokio/src/webrtc/ice.rs @@ -0,0 +1,505 @@ +//! The ICE side of a str0m peer connection: one UDP socket, its candidates, and +//! the loop that drives an [`Rtc`] over it. +//! +//! str0m is sans-IO, so [`Agent::next`] alternates between [`Rtc::poll_output`] +//! (transmits, events, the next timeout) and [`Rtc::handle_input`] (datagrams and +//! timeouts), and runs a STUN Binding client on the same socket so the agent can +//! offer server-reflexive candidates. Shared by this crate's data channel +//! transport and `moq-rtc`'s WHIP/WHEP gateway, which only differ in what they do +//! with the events. +//! +//! A dual-stack socket (`[::]`) reports an IPv4 peer as an IPv4-mapped IPv6 +//! address and refuses a send to a real `V4` destination. Neither form leaks past +//! this module: inbound sources are unmapped by [`canonical`], since ICE pairs +//! candidates by address family, and outbound destinations are mapped back. + +use std::collections::VecDeque; +use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use str0m::net::{Protocol, Receive}; +use str0m::{Candidate, IceConnectionState, Input, Output, Rtc}; +use stun_types::attribute::XorMappedAddress; +use stun_types::message::{BINDING, Message, MessageClass, MessageWriteVec, TransactionId}; +use stun_types::prelude::*; +use tokio::net::UdpSocket; +use tokio::sync::mpsc; + +/// One inbound UDP datagram and the address it came from. +pub type Packet = (Vec, SocketAddr); + +/// The datagrams routed to one [`Agent`], and the counters they are tallied in. +pub struct Inbound { + /// Ends once every sender is dropped, which ends the agent. + pub packets: mpsc::Receiver, + /// Counts `rx` where datagrams are routed, so a full queue's drops count too; + /// the agent counts `tx`. + pub wire: Wire, +} + +/// Cumulative UDP payload bytes exchanged with one peer, excluding IP and UDP +/// headers. Clones share counters, which outlive the agent. +#[derive(Clone, Debug, Default)] +pub struct Wire(Arc); + +#[derive(Debug, Default)] +struct Counters { + tx: AtomicU64, + rx: AtomicU64, +} + +/// A snapshot of a [`Wire`]'s counters. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Stats { + /// Bytes the socket accepted, including ICE, DTLS, and retransmissions. + pub tx: u64, + /// Bytes routed to the peer, including those its full queue dropped. + pub rx: u64, +} + +impl Wire { + /// Read both counters without consuming them. + pub fn stats(&self) -> Stats { + Stats { + tx: self.0.tx.load(Ordering::Relaxed), + rx: self.0.rx.load(Ordering::Relaxed), + } + } + + /// Count a datagram routed to this peer. + pub fn received(&self, bytes: usize) { + self.0.rx.fetch_add(bytes as u64, Ordering::Relaxed); + } + + fn sent(&self, bytes: usize) { + self.0.tx.fetch_add(bytes as u64, Ordering::Relaxed); + } +} + +/// Bound on a session's inbound datagram queue, sized like a socket buffer: past +/// it, datagrams are dropped rather than buffered, since ICE and SCTP tolerate loss +/// and a stalled session must not grow memory without limit. +pub const INBOX: usize = 256; + +/// Backstop deadline for reaching a connected ICE state, covering the one case +/// str0m's agent deliberately never times out: a peer that provides no remote +/// candidates and sends nothing. str0m ends a connection whose candidate pairs were +/// tried and exhausted (after ~21s at its defaults), but with no remote candidates +/// it treats the session as still possible forever, since trickle ICE could add +/// one. Set above str0m's own exhaustion so a slow but real peer is not clipped. +const ESTABLISH_TIMEOUT: Duration = Duration::from_secs(30); + +/// RFC 8489 section 6.2.1: the first retransmission timeout of a STUN request. +const STUN_RTO: Duration = Duration::from_millis(500); + +/// RFC 8489 section 6.2.1: how many times a STUN request is sent (`Rc`). +const STUN_SENDS: u32 = 7; + +/// RFC 8489 section 6.2.1: how long to wait after the last send, in RTOs (`Rm`). +const STUN_LAST_WAIT: u32 = 16; + +/// Why an [`Agent`] stopped. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// The peer disconnected, str0m closed the connection, or the inbound queue ended. + #[error("peer connection closed")] + Closed, + + /// ICE never connected before the establishment deadline. + #[error("ICE did not connect before the establishment deadline")] + Timeout, + + /// str0m failed (SDP, DTLS, SCTP, or ICE state). + #[error("rtc error: {0}")] + Rtc(String), +} + +impl Error { + fn rtc(err: impl std::fmt::Display) -> Self { + Self::Rtc(err.to_string()) + } +} + +/// What [`Agent::next`] surfaced. +#[derive(Debug)] +#[non_exhaustive] +// Moved once per poll; boxing would allocate for every data channel message. +#[allow(clippy::large_enum_variant)] +pub enum Event { + /// An event from str0m. ICE disconnection is reported as [`Error::Closed`] instead. + Rtc(str0m::Event), + /// A server-reflexive candidate, already added locally, to signal to the peer. + Candidate(Candidate), + /// Every STUN server answered or gave up: no more local candidates follow. + Gathered, +} + +/// A STUN Binding request in flight to one server. +struct Query { + server: SocketAddr, + /// The host candidate the request leaves from, which a reply's mapping is the reflexive form of. + base: SocketAddr, + id: TransactionId, + request: Vec, + sends: u32, + rto: Duration, + /// When to retransmit, or give up after the last send. + deadline: Instant, +} + +/// Drives one [`Rtc`] over a UDP socket until the connection ends. +pub struct Agent { + rtc: Rtc, + /// Send side, possibly shared by several agents (a demuxed server socket). + socket: Arc, + /// Whether `socket` is AF_INET6, which needs IPv4 destinations mapped. + socket_v6: bool, + /// The host candidates advertised to the peer. Each inbound datagram is tagged + /// with the one whose family matches its source: str0m drops a STUN request + /// whose destination is not a local candidate, and a wildcard bind is not one. + /// Never empty. + locals: Vec, + inbound: Inbound, + stun: Vec, + gathered: bool, + events: VecDeque, + /// Datagrams the socket refused for lack of buffer space, sent first next time. + outbox: VecDeque<(Vec, SocketAddr)>, + started: Instant, + connected: bool, +} + +impl Agent { + /// Drive `rtc`, sending from `socket` and receiving from `inbound`. + /// + /// `locals` are the host candidates advertised to the peer, not the socket's + /// bind address, and must not be empty. + pub fn new(rtc: Rtc, socket: Arc, locals: Vec, inbound: Inbound) -> std::io::Result { + if locals.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "no host candidate to receive on", + )); + } + let socket_v6 = socket.local_addr()?.is_ipv6(); + Ok(Self { + rtc, + socket, + socket_v6, + locals, + inbound, + stun: Vec::new(), + gathered: false, + events: VecDeque::new(), + outbox: VecDeque::new(), + started: Instant::now(), + connected: false, + }) + } + + /// Ask each STUN server for this socket's public mapping, offering what comes + /// back as a server-reflexive candidate. Servers of a family with no host + /// candidate are skipped. + pub fn with_stun(mut self, servers: &[SocketAddr]) -> Self { + let now = Instant::now(); + for &server in servers { + let server = canonical(server); + let Some(base) = self.locals.iter().find(|l| l.is_ipv4() == server.is_ipv4()).copied() else { + tracing::debug!(%server, "no host candidate of this family; skipping STUN server"); + continue; + }; + let request = Message::builder_request(BINDING, MessageWriteVec::new()); + let id = request.transaction_id(); + self.stun.push(Query { + server, + base, + id, + request: request.finish(), + sends: 0, + rto: STUN_RTO, + deadline: now, + }); + } + self + } + + /// The connection being driven. + pub fn rtc(&mut self) -> &mut Rtc { + &mut self.rtc + } + + /// The counters of the bytes exchanged with the peer. + pub fn wire(&self) -> &Wire { + &self.inbound.wire + } + + /// Drive the connection until it surfaces an [`Event`]. + /// + /// Cancel safe: a datagram the socket could not take yet is kept and sent first + /// on the next call. + pub async fn next(&mut self) -> Result { + loop { + if let Some(event) = self.events.pop_front() { + return Ok(event); + } + if !self.gathered && self.stun.is_empty() { + self.gathered = true; + return Ok(Event::Gathered); + } + + while let Some((data, dst)) = self.outbox.front() { + match self.socket.send_to(data, *dst).await { + Ok(len) => self.inbound.wire.sent(len), + Err(err) => tracing::warn!(%err, dst = %dst, "send failed"), + } + self.outbox.pop_front(); + } + + // A dead Rtc (DTLS failure, explicit disconnect) reports a never-firing + // timeout instead of an error, which would park this loop forever. + if !self.rtc.is_alive() { + return Err(Error::Closed); + } + if !self.connected && self.started.elapsed() >= ESTABLISH_TIMEOUT { + return Err(Error::Timeout); + } + + let timeout = match self.rtc.poll_output().map_err(Error::rtc)? { + Output::Timeout(timeout) => timeout, + Output::Transmit(transmit) => { + self.send(&transmit.contents, transmit.destination); + continue; + } + Output::Event(event) => { + if let str0m::Event::IceConnectionStateChange(state) = &event { + tracing::debug!(?state, "ice state"); + if *state == IceConnectionState::Disconnected { + return Err(Error::Closed); + } + self.connected |= state.is_connected(); + } + return Ok(Event::Rtc(event)); + } + }; + + let mut deadline = self.stun.iter().map(|q| q.deadline).fold(timeout, Instant::min); + if !self.connected { + deadline = deadline.min(self.started + ESTABLISH_TIMEOUT); + } + + let now = Instant::now(); + if deadline > now { + tokio::select! { + packet = self.inbound.packets.recv() => { + let Some((data, src)) = packet else { + // Every sender dropped: nothing more will arrive. + return Err(Error::Closed); + }; + self.receive(&data, src)?; + continue; + } + _ = tokio::time::sleep(deadline - now) => {} + } + } + + let now = Instant::now(); + self.rtc.handle_input(Input::Timeout(now)).map_err(Error::rtc)?; + self.retransmit(now); + } + } + + /// Send without waiting, keeping the datagram for later when the socket is full. + fn send(&mut self, data: &[u8], dst: SocketAddr) { + let dst = to_family(dst, self.socket_v6); + match self.socket.try_send_to(data, dst) { + Ok(len) => self.inbound.wire.sent(len), + Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => self.outbox.push_back((data.to_vec(), dst)), + Err(err) => tracing::warn!(%err, %dst, "send failed"), + } + } + + fn receive(&mut self, data: &[u8], src: SocketAddr) -> Result<(), Error> { + if self.stun_reply(data, src) { + return Ok(()); + } + + let local = pick_local(&self.locals, src); + // Anything can arrive on a socket a server-reflexive candidate exposes, so a + // datagram that is not STUN, DTLS, or RTP is noise, not a failure. + let Ok(receive) = Receive::new(Protocol::Udp, src, local, data) else { + tracing::debug!(%src, len = data.len(), "dropped unrecognized datagram"); + return Ok(()); + }; + self.rtc + .handle_input(Input::Receive(Instant::now(), receive)) + .map_err(Error::rtc) + } + + /// Consume a reply to one of our STUN queries, returning false for anything else. + fn stun_reply(&mut self, data: &[u8], src: SocketAddr) -> bool { + if self.stun.is_empty() { + return false; + } + let Ok(reply) = Message::from_bytes(data) else { + return false; + }; + let id = reply.transaction_id(); + let Some(index) = self.stun.iter().position(|q| q.id == id && q.server == src) else { + return false; + }; + let query = self.stun.swap_remove(index); + + if !reply.has_method(BINDING) || !reply.has_class(MessageClass::Success) { + tracing::debug!(server = %query.server, "STUN server refused the binding request"); + return true; + } + let Ok(mapped) = reply.attribute::() else { + tracing::debug!(server = %query.server, "STUN reply carries no mapped address"); + return true; + }; + + let mapped = canonical(mapped.addr(id)); + match Candidate::server_reflexive(mapped, query.base, "udp") { + // None: redundant with a candidate the agent already has, as when there is no NAT. + Ok(candidate) => { + if let Some(candidate) = self.rtc.add_local_candidate(candidate) { + self.events.push_back(Event::Candidate(candidate.clone())); + } + } + Err(err) => tracing::debug!(%err, %mapped, "unusable server-reflexive address"), + } + true + } + + /// Send each STUN request that is due, dropping those out of attempts. + fn retransmit(&mut self, now: Instant) { + let mut due = Vec::new(); + self.stun.retain_mut(|query| { + if query.deadline > now { + return true; + } + if query.sends == STUN_SENDS { + tracing::debug!(server = %query.server, "STUN server did not answer"); + return false; + } + query.sends += 1; + query.deadline = now + + match query.sends { + STUN_SENDS => STUN_RTO * STUN_LAST_WAIT, + _ => query.rto, + }; + query.rto *= 2; + due.push((query.request.clone(), query.server)); + true + }); + for (request, server) in due { + self.send(&request, server); + } + } +} + +/// Unmap an IPv4-mapped address to the `V4` it really is; others pass through. +pub fn canonical(addr: SocketAddr) -> SocketAddr { + SocketAddr::new(addr.ip().to_canonical(), addr.port()) +} + +/// Adapt a destination to what the socket accepts: an AF_INET6 socket only sends +/// to IPv6, so an IPv4 destination goes back to its mapped form. +fn to_family(addr: SocketAddr, socket_v6: bool) -> SocketAddr { + match addr.ip() { + IpAddr::V4(ip) if socket_v6 => SocketAddr::new(IpAddr::V6(ip.to_ipv6_mapped()), addr.port()), + _ => addr, + } +} + +/// The advertised candidate to tag an inbound datagram with: the first of the +/// source's family, falling back to the first, so a dual-stack peer's packets keep +/// a same-family host candidate and str0m's pairing stays consistent. +fn pick_local(locals: &[SocketAddr], src: SocketAddr) -> SocketAddr { + locals + .iter() + .find(|l| l.is_ipv4() == src.is_ipv4()) + .copied() + .unwrap_or(locals[0]) +} + +/// Pump every datagram from a socket the caller owns 1:1 into a queue for +/// [`Agent::new`], until the queue's receiver is dropped. +pub fn reader(socket: Arc) -> Inbound { + let (tx, rx) = mpsc::channel(INBOX); + let wire = Wire::default(); + let counted = wire.clone(); + tokio::spawn(async move { + let mut buf = vec![0u8; 65_535]; + loop { + tokio::select! { + res = socket.recv_from(&mut buf) => match res { + Ok((len, src)) => { + counted.received(len); + // Bounded like a socket buffer: drop on full. + if let Err(mpsc::error::TrySendError::Closed(_)) = tx.try_send((buf[..len].to_vec(), canonical(src))) { + break; + } + } + Err(err) => { + tracing::warn!(%err, "webrtc socket recv failed"); + break; + } + }, + _ = tx.closed() => break, + } + } + }); + Inbound { packets: rx, wire } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pick_local_matches_address_family() { + let v4: SocketAddr = "1.2.3.4:5000".parse().unwrap(); + let v6: SocketAddr = "[2001:db8::1]:5000".parse().unwrap(); + let locals = vec![v4, v6]; + assert_eq!(pick_local(&locals, "9.9.9.9:1".parse().unwrap()), v4); + assert_eq!(pick_local(&locals, "[2001:db8::2]:1".parse().unwrap()), v6); + // No same-family candidate falls back to the first. + assert_eq!(pick_local(&[v4], "[2001:db8::2]:1".parse().unwrap()), v4); + } + + #[tokio::test] + async fn wire_counts_only_accepted_payload() { + let socket = Arc::new(UdpSocket::bind("127.0.0.1:0").await.unwrap()); + let peer = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let local = socket.local_addr().unwrap(); + let inbound = reader(socket.clone()); + let wire = inbound.wire.clone(); + let mut agent = Agent::new(Rtc::builder().build(Instant::now()), socket, vec![local], inbound).unwrap(); + + // Until tokio has seen the socket writable, a send waits in the outbox. + agent.socket.writable().await.unwrap(); + agent.send(&[42; 73], peer.local_addr().unwrap()); + // UDP refuses an oversized datagram, which is never counted as sent. + agent.send(&vec![0; 65_536], peer.local_addr().unwrap()); + assert_eq!(wire.stats(), Stats { tx: 73, rx: 0 }); + + peer.send_to(&[7; 11], local).await.unwrap(); + let (data, _) = agent.inbound.packets.recv().await.unwrap(); + assert_eq!(data.len(), 11); + drop(agent); + assert_eq!(wire.stats(), Stats { tx: 73, rx: 11 }); + } + + #[test] + fn mapped_addresses_round_trip() { + let v4: SocketAddr = "1.2.3.4:5000".parse().unwrap(); + let mapped = to_family(v4, true); + assert_eq!(mapped, "[::ffff:1.2.3.4]:5000".parse().unwrap()); + assert_eq!(canonical(mapped), v4); + assert_eq!(to_family(v4, false), v4); + } +} diff --git a/rs/moq-tokio/src/webrtc/tests.rs b/rs/moq-tokio/src/webrtc/tests.rs new file mode 100644 index 00000000000..5e33a661d23 --- /dev/null +++ b/rs/moq-tokio/src/webrtc/tests.rs @@ -0,0 +1,242 @@ +use std::net::{IpAddr, Ipv4Addr}; +use std::time::Duration; + +use super::*; + +/// Real sockets and real ICE, so these run on the wall clock. +const TIMEOUT: Duration = Duration::from_secs(10); + +const LOOPBACK: IpAddr = IpAddr::V4(Ipv4Addr::LOCALHOST); + +/// A protocol both the default client and server speak. +fn protocol() -> &'static str { + crate::connect::Config::default().versions().alpns()[0] +} + +/// A one-frame broadcast named `test` and the handles that keep it alive. +fn broadcast() -> (moq_net::origin::Producer, impl Sized) { + let origin = crate::origin::spawn(); + let broadcast = origin.create_broadcast("test").unwrap(); + broadcast.announce(Default::default()).unwrap(); + let track = broadcast + .create_track( + "video", + moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI), + ) + .unwrap(); + let mut group = track.append_group().unwrap(); + group.write_frame(moq_net::Timestamp::ZERO, &b"hello"[..]).unwrap(); + group.finish().unwrap(); + (origin, (broadcast, track)) +} + +/// Serve the broadcast from the answering peer, subscribe from the offering one, +/// and read the frame back across the data channel. +async fn transfer(offerer: &Peer, answerer: &Peer) { + let (origin, _keep) = broadcast(); + let listener = crate::listen::Config::default() + .init_streams() + .unwrap() + .listen() + .await + .unwrap(); + let server = async { + let request = listener.accept_webrtc(answerer).await.unwrap(); + assert_eq!(request.transport(), crate::Transport::WebRtc); + request.with_publisher(&origin).ok().await.unwrap() + }; + + let subscribed = crate::origin::spawn(); + let consumer = subscribed.consume(); + let mut announced = consumer.announced(); + let client = crate::client::Config::default() + .init() + .unwrap() + .with_subscriber(subscribed); + let ((_server, _driver), _client) = tokio::time::timeout(TIMEOUT, async { + tokio::join!(server, async { client.connect_webrtc(offerer).await.unwrap() }) + }) + .await + .expect("handshake timed out"); + + let read = async { + let Some(moq_net::announce::Event::Start(route)) = announced.next().await else { + panic!("expected the broadcast to be announced"); + }; + assert_eq!(route.prefix.as_str(), "test"); + let broadcast = consumer.request_broadcast("test", None).await.unwrap(); + let mut track = broadcast.track("video").unwrap().subscribe(None).await.unwrap(); + let mut group = track.recv_group().await.unwrap().unwrap(); + group.read_frame().await.unwrap().unwrap() + }; + let frame = tokio::time::timeout(TIMEOUT, read).await.expect("transfer timed out"); + assert_eq!(&frame.payload[..], b"hello"); +} + +#[tokio::test] +async fn session_over_loopback() { + let config = Config::default(); + let (offerer, offer) = Peer::offer_from(&config, protocol(), vec![LOOPBACK]).await.unwrap(); + let (answerer, answer) = Peer::answer_from(&config, &offer, vec![LOOPBACK]).await.unwrap(); + offerer.accept(&answer).await.unwrap(); + + transfer(&offerer, &answerer).await; +} + +/// Answer STUN Binding requests with the source address, as a public server would. +async fn stun_server() -> SocketAddr { + use stun_types::attribute::XorMappedAddress; + use stun_types::message::{Message, MessageWriteVec}; + use stun_types::prelude::*; + + let socket = UdpSocket::bind((LOOPBACK, 0)).await.unwrap(); + let addr = socket.local_addr().unwrap(); + tokio::spawn(async move { + let mut buf = [0; 1500]; + while let Ok((len, src)) = socket.recv_from(&mut buf).await { + let request = Message::from_bytes(&buf[..len]).unwrap(); + let mut reply = Message::builder_success(&request, MessageWriteVec::new()); + reply + .add_attribute(&XorMappedAddress::new(src, request.transaction_id())) + .unwrap(); + socket.send_to(&reply.finish(), src).await.unwrap(); + } + }); + addr +} + +/// Drop the candidates a description carries, so only trickled ones reach the peer. +fn strip_candidates(sdp: &str) -> String { + sdp.split_inclusive('\n') + .filter(|line| !line.starts_with("a=candidate:")) + .collect() +} + +/// The offerer sits behind a pretend NAT: it advertises 127.0.0.2, which the +/// answerer never learns, and reaches the world as 127.0.0.1. With no candidates +/// in either description and none trickled back, the answerer can only reach the +/// offerer through the server-reflexive candidate STUN found. +#[tokio::test] +async fn server_reflexive_candidate_is_offered_and_selected() { + let stun = stun_server().await; + let config = Config { + stun: vec![format!("stun:{stun}").parse().unwrap()], + }; + let private = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 2)); + let (mut offerer, offer) = Peer::offer_from(&config, protocol(), vec![private]).await.unwrap(); + let offer = Offer::new(strip_candidates(offer.as_str())); + let (answerer, answer) = Peer::answer_from(&Config::default(), &offer, vec![LOOPBACK]) + .await + .unwrap(); + offerer + .accept(&Answer::new(strip_candidates(answer.as_str()))) + .await + .unwrap(); + + let candidate = tokio::time::timeout(TIMEOUT, offerer.candidate()) + .await + .expect("gathering timed out") + .expect("no server-reflexive candidate"); + let parsed = str0m::Candidate::from_sdp_string(candidate.as_str()).unwrap(); + assert_eq!(parsed.kind(), str0m::CandidateKind::ServerReflexive); + assert_eq!(parsed.addr().ip(), LOOPBACK); + answerer.add_candidate(&candidate).await.unwrap(); + assert_eq!( + offerer.candidate().await, + None, + "gathering should end after the only server answers" + ); + + transfer(&offerer, &answerer).await; +} + +#[tokio::test] +async fn roles_are_enforced() { + let config = Config::default(); + let (offerer, offer) = Peer::offer_from(&config, protocol(), vec![LOOPBACK]).await.unwrap(); + let (answerer, answer) = Peer::answer_from(&config, &offer, vec![LOOPBACK]).await.unwrap(); + + assert!(matches!(answerer.accept(&answer).await, Err(Error::Role(_)))); + let client = crate::client::Config::default().init().unwrap(); + assert!(matches!( + client.connect_webrtc(&answerer).await, + Err(crate::Error::WebRtc(err)) if matches!(*err, Error::Role(_)) + )); + let listener = crate::listen::Config::default() + .init_streams() + .unwrap() + .listen() + .await + .unwrap(); + assert!(matches!( + listener.accept_webrtc(&offerer).await, + Err(crate::Error::WebRtc(err)) if matches!(*err, Error::Role(_)) + )); +} + +#[tokio::test] +async fn unsupported_protocol_is_refused() { + let (offerer, _) = Peer::offer_from(&Config::default(), "not-moq", vec![LOOPBACK]) + .await + .unwrap(); + let client = crate::client::Config::default().init().unwrap(); + assert!(matches!( + client.connect_webrtc(&offerer).await, + Err(crate::Error::WebRtc(err)) if matches!(*err, Error::Protocol(_)) + )); +} + +#[test] +fn stun_urls() { + let parse = |url: &str| url.parse::().map(|stun| stun.to_string()).ok(); + assert_eq!( + parse("stun:stun.example.com").as_deref(), + Some("stun:stun.example.com:3478") + ); + assert_eq!(parse("STUN:1.2.3.4:19302").as_deref(), Some("stun:1.2.3.4:19302")); + assert_eq!(parse("stun:[::1]").as_deref(), Some("stun:[::1]:3478")); + assert_eq!(parse("stun:[::1]:5349").as_deref(), Some("stun:[::1]:5349")); + for bad in [ + "turn:turn.example.com", + "stuns:stun.example.com", + "stun:", + "stun:host:port", + "stun:host:3478?transport=udp", + "stun:[::1]x", + "stun:[nope]", + "stun.example.com", + ] { + assert_eq!(parse(bad), None, "{bad}"); + } +} + +#[test] +fn max_message_sizes() { + assert_eq!(max_message_size("v=0\r\n").unwrap(), DEFAULT_MAX_MESSAGE_SIZE); + assert_eq!(max_message_size("a=max-message-size:262144\r\n").unwrap(), 262_144); + assert_eq!(max_message_size("a=max-message-size:0\r\n").unwrap(), u64::MAX); + assert!(max_message_size("a=max-message-size:big\r\n").is_err()); +} + +#[test] +fn mdns_candidates_are_split_out() { + let sdp = "v=0\r\n\ + a=candidate:1 1 udp 2113937151 0a1b2c3d.local 54321 typ host\r\n\ + a=candidate:2 1 udp 2113937151 192.168.1.2 54321 typ host\r\n\ + a=end-of-candidates\r\n"; + let (kept, mdns) = split_mdns(sdp); + assert_eq!( + kept, + "v=0\r\na=candidate:2 1 udp 2113937151 192.168.1.2 54321 typ host\r\na=end-of-candidates\r\n" + ); + assert_eq!(mdns, ["candidate:1 1 udp 2113937151 0a1b2c3d.local 54321 typ host"]); +} + +#[tokio::test] +async fn ip_candidates_parse_without_mdns() { + let candidate = parse_candidate("candidate:2 1 udp 2113937151 192.168.1.2 54321 typ host") + .await + .unwrap(); + assert_eq!(candidate.addr(), "192.168.1.2:54321".parse().unwrap()); + assert!(parse_candidate("candidate:2 1 udp nope").await.is_err()); +} diff --git a/sh/rs/tokio-features.sh b/sh/rs/tokio-features.sh index 8727b4a233a..b873fcb07fd 100755 --- a/sh/rs/tokio-features.sh +++ b/sh/rs/tokio-features.sh @@ -22,7 +22,7 @@ if ((all)); then cargo clippy --locked -p moq-tokio --no-default-features --features noq,aws-lc-rs -- -D warnings cargo clippy --locked -p moq-tokio --no-default-features --features noq,ring -- -D warnings - for feature in aws-lc-rs ring tcp uds websocket; do + for feature in aws-lc-rs ring tcp uds websocket webrtc; do cargo clippy --locked -p moq-tokio --no-default-features --features "$feature" -- -D warnings done fi From 19dcd22e73f411b2e602b6de125eb78e2470ebe4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 00:06:05 -0700 Subject: [PATCH 03/15] quest: finish m1/p2p/webrtc Co-Authored-By: Claude Opus 5.5 --- quest/m1/p2p/README.md | 1 - quest/m1/p2p/cli.md | 1 - quest/m1/p2p/native-quic.md | 10 ++++----- quest/m1/p2p/webrtc.md | 43 ------------------------------------- 4 files changed, 5 insertions(+), 50 deletions(-) delete mode 100644 quest/m1/p2p/webrtc.md diff --git a/quest/m1/p2p/README.md b/quest/m1/p2p/README.md index 8a4c07801cd..a20f09fbde0 100644 --- a/quest/m1/p2p/README.md +++ b/quest/m1/p2p/README.md @@ -141,7 +141,6 @@ Backend choice does not change the policy that watching tabs never re-serve. - [Peer grants](/quest/m1/p2p/peer-grant.md) - the relay issues a node-bound, asymmetrically signed grant a browser can verify; HS256 keys issue none - [Signaling and policy](/quest/m1/p2p/signal.md) - opted-in peers find each other under the prefix, the application picks who to dial, and the roster-size gate decides whether STUN is used -- [Native data channel transport](/quest/m1/p2p/webrtc.md) - `moq-tokio` holds a moq-net session with a browser over str0m with a full ICE agent - [UDP demux](/quest/m1/p2p/udp-demux.md) - one classifier splits QUIC from STUN and WebRTC on the relay's socket and on a native pair's - [Native pairs over ICE and QUIC](/quest/m1/p2p/native-quic.md) - two native peers connect through ICE, then `moq-quic` over the nominated path with fingerprints pinned from the offer - [moq-cli joins](/quest/m1/p2p/cli.md) - `--p2p` publishes a roster entry, dials native peers over ICE and QUIC, and serves browsers as a transit hop diff --git a/quest/m1/p2p/cli.md b/quest/m1/p2p/cli.md index 1c8bc2d7ad8..4d8526f3443 100644 --- a/quest/m1/p2p/cli.md +++ b/quest/m1/p2p/cli.md @@ -41,6 +41,5 @@ Docs: a "P2P" section in `doc/bin/cli.md` beside the LAN cluster one. ## Required -- [Native data channel transport](/quest/m1/p2p/webrtc.md) - [Native pairs over ICE and QUIC](/quest/m1/p2p/native-quic.md) - the native-to-native path - [Signaling and policy](/quest/m1/p2p/signal.md) - the roster and offer schema this mirrors diff --git a/quest/m1/p2p/native-quic.md b/quest/m1/p2p/native-quic.md index 912de3082f5..53b220476fd 100644 --- a/quest/m1/p2p/native-quic.md +++ b/quest/m1/p2p/native-quic.md @@ -7,7 +7,7 @@ selected by their applications connect directly across a NAT: ICE finds a path through host and server-reflexive candidates, then a `moq-quic` connection runs over the nominated pair, with each side's certificate pinned by the fingerprint the other sent in its offer or answer. A pair with a -browser still uses a [data channel](/quest/m1/p2p/webrtc.md). +browser still uses a data channel (`moq_tokio::webrtc`). ## Plan @@ -18,9 +18,10 @@ Decided with the maintainer on 2026-10-13: the relay as rendezvous and fallback. - ICE is str0m's standalone sans-IO agent, the `is` crate (0.11, already in the lockfile through str0m; str0m re-exports it as `str0m::ice`). Gather - candidates through the module [webrtc](/quest/m1/p2p/webrtc.md) lifts out - of `moq-rtc`, with host candidates, `.local` resolution, and the STUN - Binding client; don't copy it. + candidates with `moq_tokio::webrtc`'s host gathering, `.local` resolution, + and the STUN Binding client in `webrtc::ice`; don't copy them. The first + two are private to the data channel transport today, so move them into + `webrtc::ice` rather than export them where they sit. - QUIC is `moq-quic`, fed datagrams from the nominated pair, so this waits on [the switch](/quest/m1/quic/fork/switch.md) instead of building on `moq-noq` and porting it. @@ -63,5 +64,4 @@ covers LAN and NAT pairs between two `moq-cli --p2p` nodes. - [Switch MoQ onto moq-quic](/quest/m1/quic/fork/switch.md) - `moq-quic` replaces `moq-noq` in moq-tokio - [UDP demux](/quest/m1/p2p/udp-demux.md) - the classifier that splits STUN from QUIC -- [Native data channel transport](/quest/m1/p2p/webrtc.md) - the shared ICE gathering module - [Signaling and policy](/quest/m1/p2p/signal.md) - the offer schema and roster this extends diff --git a/quest/m1/p2p/webrtc.md b/quest/m1/p2p/webrtc.md deleted file mode 100644 index 9daac1e3bb8..00000000000 --- a/quest/m1/p2p/webrtc.md +++ /dev/null @@ -1,43 +0,0 @@ -# [L] Native data channel transport - -## Goal - -`moq-tokio` gains a `webrtc` feature: a data channel transport implementing -moq-net's poll `Session` over str0m, qmux over one ordered channel, with a -full ICE agent that gathers host candidates, resolves browsers' `.local` -candidates with mdns-sd, and gathers server-reflexive candidates from the -STUN servers it is given. `Client` and `Server` can hold a moq-net session -with a browser peer across a NAT. - -## Plan - -str0m is already a dependency through `moq-rtc` and mdns-sd through the -`mdns` feature; no new stack. The feature is off by default in `moq-tokio` -and on in `moq-cli`. - -ICE: a full agent with host candidates on every non-loopback interface as raw -addresses, a dual-stack socket per session, and a STUN Binding client for -each configured server so the agent offers server-reflexive candidates. -`moq-rtc` gathers host candidates only today; lift its run loop, socket -reader, and candidate pick into a module both crates share and add the STUN -client there, rather than copying it. Remote `.local` candidates are resolved -with an mdns-sd query before `add_remote_candidate`. - -Signaling is the caller's: the transport exposes the local description and a -stream of local candidates, and accepts the remote description and -candidates, as plain async methods. No callbacks. - -Transport: implement `qmux::transport::{Transport, Writer, Reader}` over one -reliable ordered str0m channel, the way `ws::Upgraded` does, and feed -`qmux::Session` through `transport::Session` like `websocket.rs`. One record -per message, `max_record_size` 16 KiB by default, clamped to the negotiated -message size. The channel is always ordered, with no knob. - -Tests: an in-process str0m pair over loopback runs moq-net's session tests; -a second pair puts a fake STUN server between them and asserts the reflexive -candidate is offered and selected. Browser interop is the harness's job. - -## Related - -- [moq-cli joins](/quest/m1/p2p/cli.md) - the first consumer -- [Native pairs over ICE and QUIC](/quest/m1/p2p/native-quic.md) - reuses the ICE gathering module this lifts out of `moq-rtc` From ace5625315fa5d617e0ed6234a9fd7b6467dfbb4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 12:06:28 -0700 Subject: [PATCH 04/15] refactor(tokio): split webrtc::Peer into typed Offerer and Answerer A runtime Error::Role guarded calls that only one side may make. Each role now has its own type, so connect_webrtc takes an Offerer and accept_webrtc an Answerer, and the wrong pairing does not compile. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/client.rs | 6 +- rs/moq-tokio/src/server.rs | 9 +- rs/moq-tokio/src/webrtc.rs | 230 ++++++++++++++++------------- rs/moq-tokio/src/webrtc/channel.rs | 2 +- rs/moq-tokio/src/webrtc/tests.rs | 36 +---- 5 files changed, 143 insertions(+), 140 deletions(-) diff --git a/rs/moq-tokio/src/client.rs b/rs/moq-tokio/src/client.rs index b184a44383b..5e460f364e4 100644 --- a/rs/moq-tokio/src/client.rs +++ b/rs/moq-tokio/src/client.rs @@ -298,14 +298,14 @@ impl Client { Some(self.with_subscriber(origin).connect(url)) } - /// Run a MoQ client session over the data channel an offering [`crate::webrtc::Peer`] opened. + /// Run a MoQ client session over the data channel an [`crate::webrtc::Offerer`] opened. /// /// Waits for ICE and the channel to come up, bounded by the connect timeout. A /// peer link cannot be redialed, so there is no reconnect: the session ends with /// the peer connection. #[cfg(feature = "webrtc")] - pub async fn connect_webrtc(&self, peer: &crate::webrtc::Peer) -> crate::Result { - let protocol = peer.protocol()?; + pub async fn connect_webrtc(&self, peer: &crate::webrtc::Offerer) -> crate::Result { + let protocol = peer.protocol(); if !self.versions.alpns().contains(&protocol) { return Err(crate::webrtc::Error::Protocol(protocol.to_string()).into()); } diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 413187d7423..8db1d96687f 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -903,18 +903,15 @@ impl Listener { self.server.accept_next().await } - /// Read the MoQ SETUP from the data channel the browser opened on an answering - /// [`crate::webrtc::Peer`], leaving admission to the returned [`Request`]. + /// Read the MoQ SETUP from the data channel the remote peer opened on an + /// [`crate::webrtc::Answerer`], leaving admission to the returned [`Request`]. /// /// Waits for ICE and the channel to come up, bounded by the handshake timeout. /// The channel's protocol must be a MoQ ALPN this server speaks. #[cfg(feature = "webrtc")] - pub async fn accept_webrtc(&self, peer: &crate::webrtc::Peer) -> crate::Result { + pub async fn accept_webrtc(&self, peer: &crate::webrtc::Answerer) -> crate::Result { use crate::webrtc::Error as WebRtcError; - if !peer.answering() { - return Err(WebRtcError::Role("only the answering peer adopts a data channel").into()); - } let server = &self.server; let active = server .activity diff --git a/rs/moq-tokio/src/webrtc.rs b/rs/moq-tokio/src/webrtc.rs index 2089999435c..76e0ef8d5a9 100644 --- a/rs/moq-tokio/src/webrtc.rs +++ b/rs/moq-tokio/src/webrtc.rs @@ -1,14 +1,15 @@ //! MoQ over one reliable, ordered WebRTC data channel, for sessions with browser peers. //! -//! A [`Peer`] is one str0m peer connection with a full ICE agent: a dual-stack UDP -//! socket, a host candidate per non-loopback interface address, a server-reflexive -//! candidate from each configured STUN server, and remote `.local` candidates -//! resolved over mDNS. Signaling is the caller's: send the [`Offer`] or [`Answer`] -//! and the local [`Candidate`]s to the remote peer, and feed its replies back. +//! An [`Offerer`] or [`Answerer`] is one side of a str0m peer connection with a +//! full ICE agent: a dual-stack UDP socket, a host candidate per non-loopback +//! interface address, a server-reflexive candidate from each configured STUN +//! server, and remote `.local` candidates resolved over mDNS. Signaling is the +//! caller's: send the [`Offer`] or [`Answer`] and the local [`Candidate`]s to the +//! remote peer, and feed its replies back. //! -//! The offering peer opens the data channel, labelled `moq` and carrying the -//! agreed MoQ ALPN as its protocol, and runs the MoQ client over it with -//! [`crate::Client::connect_webrtc`]. The answering peer adopts that channel and +//! The [`Offerer`] opens the data channel, labelled `moq` and carrying the agreed +//! MoQ ALPN as its protocol, and runs the MoQ client over it with +//! [`crate::Client::connect_webrtc`]. The [`Answerer`] adopts that channel and //! runs the MoQ server with [`crate::Listener::accept_webrtc`]. This matches //! `@moq/p2p`, where `Transport.connect` creates the channel and //! `Transport.accept` adopts it. @@ -87,11 +88,6 @@ pub enum Error { #[error("mDNS resolution failed: {0}")] Mdns(String), - /// The call needs the other role: only the offering peer opens the data channel - /// and applies an answer, and only the answering peer adopts a channel. - #[error("wrong role: {0}")] - Role(&'static str), - /// The data channel was already taken by a session. #[error("the data channel is already in use")] Taken, @@ -115,7 +111,7 @@ pub enum Error { crate::error::from_message!(qmux::Error => Qmux); -/// How a [`Peer`] gathers its local candidates. +/// How an [`Offerer`] or [`Answerer`] gathers its local candidates. #[derive(Clone, Debug, Default)] #[non_exhaustive] pub struct Config { @@ -222,27 +218,27 @@ text!(Offer); text!(Answer); text!(Candidate); -/// One side of a WebRTC peer connection, for signaling. +/// The offering side of a WebRTC peer connection, which opens the MoQ data channel +/// and runs the MoQ client over it with [`crate::Client::connect_webrtc`]. /// /// A background task drives the connection from creation, so ICE checks and STUN /// replies are answered while signaling is still in progress. It runs until both /// this handle and any session over its data channel are gone. Dropping the handle /// stops signaling but leaves a running session alone. -pub struct Peer { - commands: mpsc::UnboundedSender, - candidates: mpsc::UnboundedReceiver, - /// The ALPN the offerer's channel carries, or `None` for the answering peer. - protocol: Option, +pub struct Offerer { + peer: Peer, + /// The MoQ ALPN the channel carries. + protocol: String, } -impl Peer { - /// Start the offering side, opening a data channel for MoQ `protocol`, the ALPN - /// the peers agreed through signaling. - pub async fn offer(config: &Config, protocol: &str) -> Result<(Self, Offer), Error> { - Self::offer_from(config, protocol, hosts()?).await +impl Offerer { + /// Start a peer connection with a data channel for MoQ `protocol`, the ALPN the + /// peers agreed through signaling, returning the offer to send. + pub async fn new(config: &Config, protocol: &str) -> Result<(Self, Offer), Error> { + Self::with_hosts(config, protocol, hosts()?).await } - async fn offer_from(config: &Config, protocol: &str, hosts: Vec) -> Result<(Self, Offer), Error> { + async fn with_hosts(config: &Config, protocol: &str, hosts: Vec) -> Result<(Self, Offer), Error> { if protocol.is_empty() || protocol.contains(char::is_whitespace) { return Err(Error::Protocol(protocol.to_string())); } @@ -263,22 +259,69 @@ impl Peer { channel: id, pending: Some(pending), }; - let peer = Self::spawn(config, rtc, socket, locals, role, None).await?; - Ok(( - Self { - protocol: Some(protocol.to_string()), - ..peer - }, - Offer(offer.to_sdp_string()), - )) + let peer = Peer::spawn(config, rtc, socket, locals, role, None).await?; + let offerer = Self { + peer, + protocol: protocol.to_string(), + }; + Ok((offerer, Offer(offer.to_sdp_string()))) + } + + /// Apply the remote peer's answer to this offer. + pub async fn accept(&self, answer: &Answer) -> Result<(), Error> { + let (sdp, mdns) = split_mdns(answer.as_str()); + let max_message = max_message_size(answer.as_str())?; + let parsed = SdpAnswer::from_sdp_string(&sdp).map_err(|err| Error::Sdp(err.to_string()))?; + + let (tx, rx) = oneshot::channel(); + self.peer.send(Command::Answer(parsed, max_message, tx))?; + rx.await.map_err(|_| ice::Error::Closed)??; + self.peer.resolve(mdns); + Ok(()) + } + + /// The next local candidate to send the remote peer, or `None` once gathering is done. + /// + /// Host candidates ride in the offer; these are the server-reflexive ones that + /// STUN finds afterwards. + pub async fn candidate(&mut self) -> Option { + self.peer.candidates.recv().await + } + + /// Add a candidate the remote peer sent, resolving a `.local` address over mDNS first. + /// + /// An empty candidate, which a browser sends to mark the end of its candidates, is ignored. + pub async fn add_candidate(&self, candidate: &Candidate) -> Result<(), Error> { + self.peer.add_candidate(candidate).await + } + + /// The MoQ ALPN this peer's data channel carries. + pub(crate) fn protocol(&self) -> &str { + &self.protocol + } + + /// Wait for the data channel to open, and take it. + pub(crate) async fn channel(&self) -> Result { + self.peer.channel().await } +} + +/// The answering side of a WebRTC peer connection, which adopts the remote peer's +/// MoQ data channel and serves it with [`crate::Listener::accept_webrtc`]. +/// +/// Like [`Offerer`], a background task drives the connection until both this +/// handle and any session over its data channel are gone. +pub struct Answerer { + peer: Peer, +} - /// Start the answering side for a browser's offer. - pub async fn answer(config: &Config, offer: &Offer) -> Result<(Self, Answer), Error> { - Self::answer_from(config, offer, hosts()?).await +impl Answerer { + /// Start a peer connection for the remote peer's offer, returning the answer to send. + pub async fn new(config: &Config, offer: &Offer) -> Result<(Self, Answer), Error> { + Self::with_hosts(config, offer, hosts()?).await } - async fn answer_from(config: &Config, offer: &Offer, hosts: Vec) -> Result<(Self, Answer), Error> { + async fn with_hosts(config: &Config, offer: &Offer, hosts: Vec) -> Result<(Self, Answer), Error> { let (sdp, mdns) = split_mdns(offer.as_str()); let max_message = max_message_size(offer.as_str())?; let parsed = SdpOffer::from_sdp_string(&sdp).map_err(|err| Error::Sdp(err.to_string()))?; @@ -290,11 +333,40 @@ impl Peer { .accept_offer(parsed) .map_err(|err| Error::Sdp(err.to_string()))?; - let peer = Self::spawn(config, rtc, socket, locals, Role::Answerer, Some(max_message)).await?; + let peer = Peer::spawn(config, rtc, socket, locals, Role::Answerer, Some(max_message)).await?; peer.resolve(mdns); - Ok((peer, Answer(answer.to_sdp_string()))) + Ok((Self { peer }, Answer(answer.to_sdp_string()))) + } + + /// The next local candidate to send the remote peer, or `None` once gathering is done. + /// + /// Host candidates ride in the answer; these are the server-reflexive ones that + /// STUN finds afterwards. + pub async fn candidate(&mut self) -> Option { + self.peer.candidates.recv().await } + /// Add a candidate the remote peer sent, resolving a `.local` address over mDNS first. + /// + /// An empty candidate, which a browser sends to mark the end of its candidates, is ignored. + pub async fn add_candidate(&self, candidate: &Candidate) -> Result<(), Error> { + self.peer.add_candidate(candidate).await + } + + /// Wait for the remote peer's data channel to open, and take it. + pub(crate) async fn channel(&self) -> Result { + self.peer.channel().await + } +} + +/// The signaling handle both roles share: commands to the driver task and the +/// candidates it gathers. +struct Peer { + commands: mpsc::UnboundedSender, + candidates: mpsc::UnboundedReceiver, +} + +impl Peer { async fn spawn( config: &Config, rtc: Rtc, @@ -322,41 +394,10 @@ impl Peer { use tracing::Instrument; tokio::spawn(driver.run().instrument(tracing::Span::current())); - Ok(Self { - commands, - candidates, - protocol: None, - }) - } - - /// Apply the browser's answer to this peer's offer. - pub async fn accept(&self, answer: &Answer) -> Result<(), Error> { - if self.protocol.is_none() { - return Err(Error::Role("only the offering peer applies an answer")); - } - let (sdp, mdns) = split_mdns(answer.as_str()); - let max_message = max_message_size(answer.as_str())?; - let parsed = SdpAnswer::from_sdp_string(&sdp).map_err(|err| Error::Sdp(err.to_string()))?; - - let (tx, rx) = oneshot::channel(); - self.send(Command::Answer(parsed, max_message, tx))?; - rx.await.map_err(|_| ice::Error::Closed)??; - self.resolve(mdns); - Ok(()) + Ok(Self { commands, candidates }) } - /// The next local candidate to send the remote peer, or `None` once gathering is done. - /// - /// Host candidates ride in the description; these are the server-reflexive ones - /// that STUN finds afterwards. - pub async fn candidate(&mut self) -> Option { - self.candidates.recv().await - } - - /// Add a candidate the remote peer sent, resolving a `.local` address over mDNS first. - /// - /// An empty candidate, which a browser sends to mark the end of its candidates, is ignored. - pub async fn add_candidate(&self, candidate: &Candidate) -> Result<(), Error> { + async fn add_candidate(&self, candidate: &Candidate) -> Result<(), Error> { let line = candidate.as_str().trim(); let line = line.strip_prefix("a=").unwrap_or(line); if line.is_empty() { @@ -366,20 +407,7 @@ impl Peer { self.send(Command::Candidate(candidate)) } - /// The MoQ ALPN this peer's data channel carries, for the offering side. - pub(crate) fn protocol(&self) -> Result<&str, Error> { - self.protocol - .as_deref() - .ok_or(Error::Role("only the offering peer opens the data channel")) - } - - /// Whether this is the answering side, which adopts the remote peer's channel. - pub(crate) fn answering(&self) -> bool { - self.protocol.is_none() - } - - /// Wait for the data channel to open, and take it. - pub(crate) async fn channel(&self) -> Result { + async fn channel(&self) -> Result { let (tx, rx) = oneshot::channel(); self.send(Command::Channel(tx))?; let mut opened = rx.await.map_err(|_| ice::Error::Closed)??; @@ -461,7 +489,7 @@ struct Open { struct Driver { agent: ice::Agent, commands: mpsc::UnboundedReceiver, - /// Dropped once gathering finishes, ending [`Peer::candidate`]. + /// Dropped once gathering finishes, ending [`Offerer::candidate`] and [`Answerer::candidate`]. candidates: Option>, role: Role, /// The remote peer's SCTP message limit, once its description is applied. @@ -553,18 +581,20 @@ impl Driver { fn command(&mut self, command: Command) { match command { Command::Answer(answer, max_message, reply) => { - let result = match &mut self.role { - Role::Offerer { pending, .. } => match pending.take() { - Some(pending) => self - .agent - .rtc() - .sdp_api() - .accept_answer(pending, answer) - .map(|()| self.max_message = Some(max_message)) - .map_err(|err| Error::Sdp(err.to_string())), - None => Err(Error::Sdp("the answer was already applied".into())), - }, - Role::Answerer => Err(Error::Role("only the offering peer applies an answer")), + // Only an `Offerer` sends an answer, so the answering role never has one pending. + let pending = match &mut self.role { + Role::Offerer { pending, .. } => pending.take(), + Role::Answerer => None, + }; + let result = match pending { + Some(pending) => self + .agent + .rtc() + .sdp_api() + .accept_answer(pending, answer) + .map(|()| self.max_message = Some(max_message)) + .map_err(|err| Error::Sdp(err.to_string())), + None => Err(Error::Sdp("the answer was already applied".into())), }; let _ = reply.send(result); } diff --git a/rs/moq-tokio/src/webrtc/channel.rs b/rs/moq-tokio/src/webrtc/channel.rs index a4e70293f8e..49fefce4bdf 100644 --- a/rs/moq-tokio/src/webrtc/channel.rs +++ b/rs/moq-tokio/src/webrtc/channel.rs @@ -29,7 +29,7 @@ pub(crate) struct Writer { pub tx: Option>, /// The largest message the peer accepts. pub max: usize, - /// Keeps the driver running while the session outlives the [`super::Peer`]. + /// Keeps the driver running while the session outlives the signaling handle. pub _driver: Option>, } diff --git a/rs/moq-tokio/src/webrtc/tests.rs b/rs/moq-tokio/src/webrtc/tests.rs index 5e33a661d23..26171643469 100644 --- a/rs/moq-tokio/src/webrtc/tests.rs +++ b/rs/moq-tokio/src/webrtc/tests.rs @@ -32,7 +32,7 @@ fn broadcast() -> (moq_net::origin::Producer, impl Sized) { /// Serve the broadcast from the answering peer, subscribe from the offering one, /// and read the frame back across the data channel. -async fn transfer(offerer: &Peer, answerer: &Peer) { +async fn transfer(offerer: &Offerer, answerer: &Answerer) { let (origin, _keep) = broadcast(); let listener = crate::listen::Config::default() .init_streams() @@ -76,8 +76,8 @@ async fn transfer(offerer: &Peer, answerer: &Peer) { #[tokio::test] async fn session_over_loopback() { let config = Config::default(); - let (offerer, offer) = Peer::offer_from(&config, protocol(), vec![LOOPBACK]).await.unwrap(); - let (answerer, answer) = Peer::answer_from(&config, &offer, vec![LOOPBACK]).await.unwrap(); + let (offerer, offer) = Offerer::with_hosts(&config, protocol(), vec![LOOPBACK]).await.unwrap(); + let (answerer, answer) = Answerer::with_hosts(&config, &offer, vec![LOOPBACK]).await.unwrap(); offerer.accept(&answer).await.unwrap(); transfer(&offerer, &answerer).await; @@ -123,9 +123,9 @@ async fn server_reflexive_candidate_is_offered_and_selected() { stun: vec![format!("stun:{stun}").parse().unwrap()], }; let private = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 2)); - let (mut offerer, offer) = Peer::offer_from(&config, protocol(), vec![private]).await.unwrap(); + let (mut offerer, offer) = Offerer::with_hosts(&config, protocol(), vec![private]).await.unwrap(); let offer = Offer::new(strip_candidates(offer.as_str())); - let (answerer, answer) = Peer::answer_from(&Config::default(), &offer, vec![LOOPBACK]) + let (answerer, answer) = Answerer::with_hosts(&Config::default(), &offer, vec![LOOPBACK]) .await .unwrap(); offerer @@ -150,33 +150,9 @@ async fn server_reflexive_candidate_is_offered_and_selected() { transfer(&offerer, &answerer).await; } -#[tokio::test] -async fn roles_are_enforced() { - let config = Config::default(); - let (offerer, offer) = Peer::offer_from(&config, protocol(), vec![LOOPBACK]).await.unwrap(); - let (answerer, answer) = Peer::answer_from(&config, &offer, vec![LOOPBACK]).await.unwrap(); - - assert!(matches!(answerer.accept(&answer).await, Err(Error::Role(_)))); - let client = crate::client::Config::default().init().unwrap(); - assert!(matches!( - client.connect_webrtc(&answerer).await, - Err(crate::Error::WebRtc(err)) if matches!(*err, Error::Role(_)) - )); - let listener = crate::listen::Config::default() - .init_streams() - .unwrap() - .listen() - .await - .unwrap(); - assert!(matches!( - listener.accept_webrtc(&offerer).await, - Err(crate::Error::WebRtc(err)) if matches!(*err, Error::Role(_)) - )); -} - #[tokio::test] async fn unsupported_protocol_is_refused() { - let (offerer, _) = Peer::offer_from(&Config::default(), "not-moq", vec![LOOPBACK]) + let (offerer, _) = Offerer::with_hosts(&Config::default(), "not-moq", vec![LOOPBACK]) .await .unwrap(); let client = crate::client::Config::default().init().unwrap(); From bda48c976231ca1c3ded959966ae72c5d7a3251b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 12:35:23 -0700 Subject: [PATCH 05/15] fix(tokio): drain str0m after every mutation and bound data channel reads str0m requires a full poll_output drain after each mutation of an Rtc. The ICE agent returned on the first event, so a caller could mutate again mid-drain (an egress write burst in moq-rtc, or back-to-back candidates). The agent now drains on every change, hands out only a shared Rtc, and routes mutations through Agent::mutate. The data channel read queue was unbounded, so a peer could grow memory while qmux's reader waited on the application. It now holds four records, and a full queue stops the driver reading the socket, as TCP would. The answerer now ignores channels not labelled moq instead of taking the first one to open. Co-Authored-By: Claude Opus 5.5 --- rs/moq-rtc/src/session.rs | 2 +- rs/moq-tokio/src/webrtc.rs | 96 ++++++++++++++--------- rs/moq-tokio/src/webrtc/channel.rs | 5 +- rs/moq-tokio/src/webrtc/ice.rs | 120 +++++++++++++++++++---------- rs/moq-tokio/src/webrtc/tests.rs | 31 +++++--- 5 files changed, 164 insertions(+), 90 deletions(-) diff --git a/rs/moq-rtc/src/session.rs b/rs/moq-rtc/src/session.rs index 49059316d8b..06bef5f7701 100644 --- a/rs/moq-rtc/src/session.rs +++ b/rs/moq-rtc/src/session.rs @@ -174,7 +174,7 @@ impl Session { } } => { let wallclock = self.egress_clock.wallclock(req.time, Instant::now()); - crate::egress::dispatch(self.agent.rtc(), req, wallclock); + self.agent.mutate(|rtc| crate::egress::dispatch(rtc, req, wallclock)); } event = self.agent.next() => match event { diff --git a/rs/moq-tokio/src/webrtc.rs b/rs/moq-tokio/src/webrtc.rs index 76e0ef8d5a9..4cf00ec26e8 100644 --- a/rs/moq-tokio/src/webrtc.rs +++ b/rs/moq-tokio/src/webrtc.rs @@ -49,6 +49,11 @@ const DEFAULT_MAX_MESSAGE_SIZE: u64 = 64 * 1024; /// Bytes the channel may hold unsent before qmux's writer waits, as `@moq/p2p` allows. const SEND_WINDOW: usize = RECORD_SIZE as usize * 4; +/// Records read off the channel that qmux has yet to take, matching the send +/// window. A full queue stops the driver reading the socket, so a slow reader +/// pushes back on the peer through SCTP's congestion control, as TCP would. +const READ_QUEUE: usize = SEND_WINDOW / RECORD_SIZE as usize; + /// How long to wait for a `.local` name to resolve: mdns-sd's first query plus its /// retransmissions at one and two seconds (RFC 6762 section 5.2). const MDNS_TIMEOUT: Duration = Duration::from_secs(5); @@ -477,7 +482,7 @@ enum Role { struct Open { id: ChannelId, protocol: String, - reads: mpsc::UnboundedSender>, + reads: mpsc::Sender>, /// `None` once the session's writer closed. writes: Option>, max_record_size: u64, @@ -505,6 +510,11 @@ impl Driver { async fn run(mut self) { let err = loop { let writable = self.writable(); + // Only a full read queue is kept, to wait for the session to take a record. + let full = match &self.channel { + Some(Ok(open)) if open.reads.capacity() == 0 && !open.reads.is_closed() => Some(open.reads.clone()), + _ => None, + }; let writes = match &mut self.channel { Some(Ok(Open { writes: Some(writes), .. @@ -513,7 +523,8 @@ impl Driver { }; tokio::select! { - event = self.agent.next() => match event { + // Each event carries at most one record, so the read queue never overflows. + event = self.agent.next(), if full.is_none() => match event { Ok(event) => if let Err(err) = self.event(event) { break err; }, @@ -529,6 +540,8 @@ impl Driver { break err; } } + // Closed or not, the queue has room once this returns. + _ = async { full.as_ref().unwrap().reserve().await.ok(); }, if full.is_some() => {} } }; @@ -550,10 +563,10 @@ impl Driver { return false; }; let id = *id; - self.agent - .rtc() - .channel(id) - .is_some_and(|mut channel| channel.buffered_amount() < SEND_WINDOW) + self.agent.mutate(|rtc| { + rtc.channel(id) + .is_some_and(|mut channel| channel.buffered_amount() < SEND_WINDOW) + }) } /// Hand a record from the session to SCTP, or close the channel when the session's writer closed. @@ -564,17 +577,19 @@ impl Driver { let id = open.id; let Some(data) = data else { open.writes = None; - self.agent.rtc().direct_api().close_data_channel(id); + self.agent.mutate(|rtc| rtc.direct_api().close_data_channel(id)); return Ok(()); }; - let Some(mut channel) = self.agent.rtc().channel(id) else { - return Err(ice::Error::Closed.into()); - }; + let written = self.agent.mutate(|rtc| { + let mut channel = rtc.channel(id).ok_or(ice::Error::Closed)?; + channel + .write(true, &data) + .map_err(|err| ice::Error::Rtc(err.to_string())) + })?; // The send window keeps the buffer well below str0m's, so a refusal is a bug, not backpressure. - match channel.write(true, &data) { - Ok(true) => Ok(()), - Ok(false) => Err(ice::Error::Rtc("SCTP refused a record within the send window".into()).into()), - Err(err) => Err(ice::Error::Rtc(err.to_string()).into()), + match written { + true => Ok(()), + false => Err(ice::Error::Rtc("SCTP refused a record within the send window".into()).into()), } } @@ -589,16 +604,14 @@ impl Driver { let result = match pending { Some(pending) => self .agent - .rtc() - .sdp_api() - .accept_answer(pending, answer) + .mutate(|rtc| rtc.sdp_api().accept_answer(pending, answer)) .map(|()| self.max_message = Some(max_message)) .map_err(|err| Error::Sdp(err.to_string())), None => Err(Error::Sdp("the answer was already applied".into())), }; let _ = reply.send(result); } - Command::Candidate(candidate) => self.agent.rtc().add_remote_candidate(candidate), + Command::Candidate(candidate) => self.agent.mutate(|rtc| rtc.add_remote_candidate(candidate)), Command::Channel(reply) => { if self.taken { let _ = reply.send(Err(Error::Taken)); @@ -642,7 +655,10 @@ impl Driver { } else { Ok(Bytes::from(data.data)) }; - let _ = open.reads.send(record); + // Room was checked before this event was taken; a closed reader drops it. + if let Err(mpsc::error::TrySendError::Full(_)) = open.reads.try_send(record) { + unreachable!("the driver takes no event while the read queue is full"); + } } str0m::Event::ChannelClose(id) => { if let Some(Ok(open)) = &self.channel @@ -659,21 +675,24 @@ impl Driver { /// Take a channel that opened as the session's, if it is the one this role expects. fn open(&mut self, id: ChannelId) { - if self.channel.is_some() { - tracing::debug!(?id, "closing an extra data channel"); - self.agent.rtc().direct_api().close_data_channel(id); - return; - } - if let Role::Offerer { channel, .. } = &self.role - && *channel != id - { - self.agent.rtc().direct_api().close_data_channel(id); + let expected = match &self.role { + Role::Offerer { channel, .. } => *channel == id, + // The remote peer may open other channels first; only the `moq` one is the session's. + Role::Answerer => self.agent.mutate(|rtc| { + rtc.channel(id) + .and_then(|channel| channel.config().map(|config| config.label == LABEL)) + .unwrap_or(false) + }), + }; + if !expected || self.channel.is_some() { + tracing::debug!(?id, "closing an unexpected data channel"); + self.agent.mutate(|rtc| rtc.direct_api().close_data_channel(id)); return; } let opened = self.adopt(id); if opened.is_err() { - self.agent.rtc().direct_api().close_data_channel(id); + self.agent.mutate(|rtc| rtc.direct_api().close_data_channel(id)); } self.channel = Some(opened); self.deliver(); @@ -686,15 +705,18 @@ impl Driver { return Err(Error::MessageSize(max_message)); } - let mut channel = self.agent.rtc().channel(id).ok_or(ice::Error::Closed)?; - let config = channel.config().ok_or(Error::Unreliable)?; - if !config.ordered || config.reliability != Reliability::Reliable { - return Err(Error::Unreliable); - } - let protocol = config.protocol.clone(); - channel.set_buffered_amount_low_threshold(SEND_WINDOW / 2); + let protocol = self.agent.mutate(|rtc| { + let mut channel = rtc.channel(id).ok_or(ice::Error::Closed)?; + let config = channel.config().ok_or(Error::Unreliable)?; + if !config.ordered || config.reliability != Reliability::Reliable { + return Err(Error::Unreliable); + } + let protocol = config.protocol.clone(); + channel.set_buffered_amount_low_threshold(SEND_WINDOW / 2); + Ok(protocol) + })?; - let (reads, rx) = mpsc::unbounded_channel(); + let (reads, rx) = mpsc::channel(READ_QUEUE); let (tx, writes) = mpsc::channel(1); Ok(Open { id, diff --git a/rs/moq-tokio/src/webrtc/channel.rs b/rs/moq-tokio/src/webrtc/channel.rs index 49fefce4bdf..81e9df96852 100644 --- a/rs/moq-tokio/src/webrtc/channel.rs +++ b/rs/moq-tokio/src/webrtc/channel.rs @@ -50,9 +50,8 @@ impl qmux::transport::Writer for Writer { /// Receives the records the driver read off the channel. pub(crate) struct Reader { - /// Unbounded because str0m pushes data as it arrives and cannot be paused; qmux - /// flow control is what bounds the bytes a peer can have in flight. - pub rx: mpsc::UnboundedReceiver>, + /// Bounded: while it is full, the driver stops reading the socket. + pub rx: mpsc::Receiver>, } impl qmux::transport::Reader for Reader { diff --git a/rs/moq-tokio/src/webrtc/ice.rs b/rs/moq-tokio/src/webrtc/ice.rs index 0d31e456334..1c456484b1f 100644 --- a/rs/moq-tokio/src/webrtc/ice.rs +++ b/rs/moq-tokio/src/webrtc/ice.rs @@ -1,13 +1,18 @@ //! The ICE side of a str0m peer connection: one UDP socket, its candidates, and //! the loop that drives an [`Rtc`] over it. //! -//! str0m is sans-IO, so [`Agent::next`] alternates between [`Rtc::poll_output`] -//! (transmits, events, the next timeout) and [`Rtc::handle_input`] (datagrams and -//! timeouts), and runs a STUN Binding client on the same socket so the agent can +//! str0m is sans-IO, so [`Agent::next`] alternates between [`Rtc::handle_input`] +//! (datagrams and timeouts) and [`Rtc::poll_output`] (transmits, events, the next +//! timeout), and runs a STUN Binding client on the same socket so the agent can //! offer server-reflexive candidates. Shared by this crate's data channel //! transport and `moq-rtc`'s WHIP/WHEP gateway, which only differ in what they do //! with the events. //! +//! str0m requires every mutation of an [`Rtc`] to be followed by draining +//! `poll_output` to a timeout before the next one. The agent owns that cycle: it +//! hands out only a shared [`Agent::rtc`], and [`Agent::mutate`] drains after the +//! caller's change, queueing the events for [`Agent::next`]. +//! //! A dual-stack socket (`[::]`) reports an IPv4 peer as an IPv4-mapped IPv6 //! address and refuses a send to a real `V4` destination. Neither form leaks past //! this module: inbound sources are unmapped by [`canonical`], since ICE pairs @@ -166,6 +171,10 @@ pub struct Agent { stun: Vec, gathered: bool, events: VecDeque, + /// Why the connection ended, found while draining and reported by the next [`Agent::next`]. + failed: Option, + /// When str0m next wants a timeout, as of the last drain. + timeout: Instant, /// Datagrams the socket refused for lack of buffer space, sent first next time. outbox: VecDeque<(Vec, SocketAddr)>, started: Instant, @@ -185,7 +194,8 @@ impl Agent { )); } let socket_v6 = socket.local_addr()?.is_ipv6(); - Ok(Self { + let now = Instant::now(); + let mut agent = Self { rtc, socket, socket_v6, @@ -194,10 +204,15 @@ impl Agent { stun: Vec::new(), gathered: false, events: VecDeque::new(), + failed: None, + timeout: now, outbox: VecDeque::new(), - started: Instant::now(), + started: now, connected: false, - }) + }; + // Signaling already changed the connection. + agent.drain(); + Ok(agent) } /// Ask each STUN server for this socket's public mapping, offering what comes @@ -226,9 +241,17 @@ impl Agent { self } - /// The connection being driven. - pub fn rtc(&mut self) -> &mut Rtc { - &mut self.rtc + /// The connection being driven, read only. + pub fn rtc(&self) -> &Rtc { + &self.rtc + } + + /// Change the connection, then drain what the change produced: transmits are + /// sent, and events wait for [`Agent::next`]. + pub fn mutate(&mut self, change: impl FnOnce(&mut Rtc) -> T) -> T { + let result = change(&mut self.rtc); + self.drain(); + result } /// The counters of the bytes exchanged with the peer. @@ -238,13 +261,16 @@ impl Agent { /// Drive the connection until it surfaces an [`Event`]. /// - /// Cancel safe: a datagram the socket could not take yet is kept and sent first - /// on the next call. + /// Cancel safe: the connection is always drained between calls, and a datagram + /// the socket could not take yet is kept and sent first on the next call. pub async fn next(&mut self) -> Result { loop { if let Some(event) = self.events.pop_front() { return Ok(event); } + if let Some(err) = self.failed.take() { + return Err(err); + } if !self.gathered && self.stun.is_empty() { self.gathered = true; return Ok(Event::Gathered); @@ -267,25 +293,7 @@ impl Agent { return Err(Error::Timeout); } - let timeout = match self.rtc.poll_output().map_err(Error::rtc)? { - Output::Timeout(timeout) => timeout, - Output::Transmit(transmit) => { - self.send(&transmit.contents, transmit.destination); - continue; - } - Output::Event(event) => { - if let str0m::Event::IceConnectionStateChange(state) = &event { - tracing::debug!(?state, "ice state"); - if *state == IceConnectionState::Disconnected { - return Err(Error::Closed); - } - self.connected |= state.is_connected(); - } - return Ok(Event::Rtc(event)); - } - }; - - let mut deadline = self.stun.iter().map(|q| q.deadline).fold(timeout, Instant::min); + let mut deadline = self.stun.iter().map(|q| q.deadline).fold(self.timeout, Instant::min); if !self.connected { deadline = deadline.min(self.started + ESTABLISH_TIMEOUT); } @@ -298,7 +306,7 @@ impl Agent { // Every sender dropped: nothing more will arrive. return Err(Error::Closed); }; - self.receive(&data, src)?; + self.receive(&data, src); continue; } _ = tokio::time::sleep(deadline - now) => {} @@ -306,11 +314,44 @@ impl Agent { } let now = Instant::now(); - self.rtc.handle_input(Input::Timeout(now)).map_err(Error::rtc)?; + self.mutate(|rtc| rtc.handle_input(Input::Timeout(now))) + .unwrap_or_else(|err| self.fail(Error::rtc(err))); self.retransmit(now); } } + /// Poll str0m until it asks for a timeout: transmits are sent, events queued. + fn drain(&mut self) { + while self.failed.is_none() { + let output = match self.rtc.poll_output() { + Ok(output) => output, + Err(err) => return self.fail(Error::rtc(err)), + }; + match output { + Output::Timeout(timeout) => { + self.timeout = timeout; + return; + } + Output::Transmit(transmit) => self.send(&transmit.contents, transmit.destination), + Output::Event(event) => { + if let str0m::Event::IceConnectionStateChange(state) = &event { + tracing::debug!(?state, "ice state"); + if *state == IceConnectionState::Disconnected { + return self.fail(Error::Closed); + } + self.connected |= state.is_connected(); + } + self.events.push_back(Event::Rtc(event)); + } + } + } + } + + /// Keep the first reason the connection ended. + fn fail(&mut self, err: Error) { + self.failed.get_or_insert(err); + } + /// Send without waiting, keeping the datagram for later when the socket is full. fn send(&mut self, data: &[u8], dst: SocketAddr) { let dst = to_family(dst, self.socket_v6); @@ -321,9 +362,9 @@ impl Agent { } } - fn receive(&mut self, data: &[u8], src: SocketAddr) -> Result<(), Error> { + fn receive(&mut self, data: &[u8], src: SocketAddr) { if self.stun_reply(data, src) { - return Ok(()); + return; } let local = pick_local(&self.locals, src); @@ -331,11 +372,10 @@ impl Agent { // datagram that is not STUN, DTLS, or RTP is noise, not a failure. let Ok(receive) = Receive::new(Protocol::Udp, src, local, data) else { tracing::debug!(%src, len = data.len(), "dropped unrecognized datagram"); - return Ok(()); + return; }; - self.rtc - .handle_input(Input::Receive(Instant::now(), receive)) - .map_err(Error::rtc) + self.mutate(|rtc| rtc.handle_input(Input::Receive(Instant::now(), receive))) + .unwrap_or_else(|err| self.fail(Error::rtc(err))); } /// Consume a reply to one of our STUN queries, returning false for anything else. @@ -365,8 +405,8 @@ impl Agent { match Candidate::server_reflexive(mapped, query.base, "udp") { // None: redundant with a candidate the agent already has, as when there is no NAT. Ok(candidate) => { - if let Some(candidate) = self.rtc.add_local_candidate(candidate) { - self.events.push_back(Event::Candidate(candidate.clone())); + if let Some(candidate) = self.mutate(|rtc| rtc.add_local_candidate(candidate).cloned()) { + self.events.push_back(Event::Candidate(candidate)); } } Err(err) => tracing::debug!(%err, %mapped, "unusable server-reflexive address"), diff --git a/rs/moq-tokio/src/webrtc/tests.rs b/rs/moq-tokio/src/webrtc/tests.rs index 26171643469..5bce0473b48 100644 --- a/rs/moq-tokio/src/webrtc/tests.rs +++ b/rs/moq-tokio/src/webrtc/tests.rs @@ -14,7 +14,7 @@ fn protocol() -> &'static str { } /// A one-frame broadcast named `test` and the handles that keep it alive. -fn broadcast() -> (moq_net::origin::Producer, impl Sized) { +fn broadcast(payload: &[u8]) -> (moq_net::origin::Producer, impl Sized) { let origin = crate::origin::spawn(); let broadcast = origin.create_broadcast("test").unwrap(); broadcast.announce(Default::default()).unwrap(); @@ -25,15 +25,15 @@ fn broadcast() -> (moq_net::origin::Producer, impl Sized) { ) .unwrap(); let mut group = track.append_group().unwrap(); - group.write_frame(moq_net::Timestamp::ZERO, &b"hello"[..]).unwrap(); + group.write_frame(moq_net::Timestamp::ZERO, payload.to_vec()).unwrap(); group.finish().unwrap(); (origin, (broadcast, track)) } -/// Serve the broadcast from the answering peer, subscribe from the offering one, -/// and read the frame back across the data channel. -async fn transfer(offerer: &Offerer, answerer: &Answerer) { - let (origin, _keep) = broadcast(); +/// Serve a one-frame broadcast from the answering peer, subscribe from the +/// offering one, and read the frame back across the data channel. +async fn transfer(offerer: &Offerer, answerer: &Answerer, payload: &[u8]) { + let (origin, _keep) = broadcast(payload); let listener = crate::listen::Config::default() .init_streams() .unwrap() @@ -70,7 +70,7 @@ async fn transfer(offerer: &Offerer, answerer: &Answerer) { group.read_frame().await.unwrap().unwrap() }; let frame = tokio::time::timeout(TIMEOUT, read).await.expect("transfer timed out"); - assert_eq!(&frame.payload[..], b"hello"); + assert!(frame.payload[..] == *payload, "the frame arrived corrupted"); } #[tokio::test] @@ -80,7 +80,20 @@ async fn session_over_loopback() { let (answerer, answer) = Answerer::with_hosts(&config, &offer, vec![LOOPBACK]).await.unwrap(); offerer.accept(&answer).await.unwrap(); - transfer(&offerer, &answerer).await; + transfer(&offerer, &answerer, b"hello").await; +} + +/// A frame many records long fills the SCTP send window and the read queue, so +/// both sides have to wait for the other and resume. +#[tokio::test] +async fn large_frame_over_loopback() { + let config = Config::default(); + let (offerer, offer) = Offerer::with_hosts(&config, protocol(), vec![LOOPBACK]).await.unwrap(); + let (answerer, answer) = Answerer::with_hosts(&config, &offer, vec![LOOPBACK]).await.unwrap(); + offerer.accept(&answer).await.unwrap(); + + let payload: Vec = (0..1024 * 1024).map(|i| i as u8).collect(); + transfer(&offerer, &answerer, &payload).await; } /// Answer STUN Binding requests with the source address, as a public server would. @@ -147,7 +160,7 @@ async fn server_reflexive_candidate_is_offered_and_selected() { "gathering should end after the only server answers" ); - transfer(&offerer, &answerer).await; + transfer(&offerer, &answerer, b"hello").await; } #[tokio::test] From 9e6fe3dd8db874ed90d041a63cdc6d720fb7e9c7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 16:45:49 -0700 Subject: [PATCH 06/15] refactor(ice): move the shared ICE agent into a moq-ice crate The agent, its STUN client, and the wire counters move out of a public `moq_tokio::webrtc::ice` into a new `moq-ice` crate, which `moq-tokio` uses privately and `moq-rtc` uses directly, so str0m types stay out of `moq-tokio`'s API. `Agent::mutate(closure)` becomes `Agent::rtc_mut()`, a guard that derefs to `&mut Rtc` and drains when dropped. A slow data channel reader no longer freezes the agent, which stalled our own sends and ICE consent. The driver keeps reading and closes the channel once unread records pass qmux's `max_data` plus slack. `accept_webrtc` on a closing listener returns `Error::Stopped`. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ts.yml | 1 + Cargo.lock | 13 ++ Cargo.toml | 3 + doc/lib/rs/index.md | 1 + quest/m2/p2p/native-quic.md | 6 +- rs/moq-ice/CHANGELOG.md | 12 ++ rs/moq-ice/Cargo.toml | 23 +++ rs/moq-ice/README.md | 12 ++ .../webrtc/ice.rs => moq-ice/src/agent.rs} | 163 ++++------------ rs/moq-ice/src/lib.rs | 27 +++ rs/moq-ice/src/wire.rs | 94 +++++++++ rs/moq-rtc/Cargo.toml | 7 +- rs/moq-rtc/src/client/whep.rs | 2 +- rs/moq-rtc/src/client/whip.rs | 2 +- rs/moq-rtc/src/server/mux.rs | 22 +-- rs/moq-rtc/src/session.rs | 31 ++- rs/moq-rtc/src/wire.rs | 8 +- rs/moq-tokio/Cargo.toml | 7 +- rs/moq-tokio/src/client.rs | 2 +- rs/moq-tokio/src/error.rs | 4 +- rs/moq-tokio/src/server.rs | 10 +- rs/moq-tokio/src/webrtc.rs | 183 ++++++++++-------- rs/moq-tokio/src/webrtc/channel.rs | 74 ++++++- rs/moq-tokio/src/webrtc/tests.rs | 69 ++++++- 24 files changed, 516 insertions(+), 260 deletions(-) create mode 100644 rs/moq-ice/CHANGELOG.md create mode 100644 rs/moq-ice/Cargo.toml create mode 100644 rs/moq-ice/README.md rename rs/{moq-tokio/src/webrtc/ice.rs => moq-ice/src/agent.rs} (75%) create mode 100644 rs/moq-ice/src/lib.rs create mode 100644 rs/moq-ice/src/wire.rs diff --git a/.github/workflows/ts.yml b/.github/workflows/ts.yml index e76956f3c31..2c241b05c55 100644 --- a/.github/workflows/ts.yml +++ b/.github/workflows/ts.yml @@ -22,6 +22,7 @@ on: # Workspace dependencies of moq-net, moq-tokio and moq-mux on the TS path. - "rs/kio/**" - "rs/moq-flate/**" + - "rs/moq-ice/**" - "rs/moq-msf/**" - "rs/moq-pattern/**" - "rs/moq-sock/**" diff --git a/Cargo.lock b/Cargo.lock index 7197f1bc3f0..206816a0da1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4562,6 +4562,17 @@ dependencies = [ "url", ] +[[package]] +name = "moq-ice" +version = "0.1.0" +dependencies = [ + "str0m", + "stun-types", + "thiserror 2.0.21", + "tokio", + "tracing", +] + [[package]] name = "moq-json" version = "0.5.8" @@ -4809,6 +4820,7 @@ dependencies = [ "axum", "bytes", "hang", + "moq-ice", "moq-mux", "moq-net", "moq-tokio", @@ -4948,6 +4960,7 @@ dependencies = [ "libc", "mdns-sd", "moq-clock", + "moq-ice", "moq-net", "moq-quic", "moq-sock", diff --git a/Cargo.toml b/Cargo.toml index 4a5f7f6d094..4c16c2caeb2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ "rs/moq-flate", "rs/moq-gst", "rs/moq-hls", + "rs/moq-ice", "rs/moq-json", "rs/moq-loc", "rs/moq-msf", @@ -62,6 +63,7 @@ default-members = [ # "rs/moq-gst", # requires GStreamer "rs/moq-flate", "rs/moq-hls", + "rs/moq-ice", "rs/moq-json", "rs/moq-loc", "rs/moq-msf", @@ -155,6 +157,7 @@ moq-clock = { version = "0.11.0", path = "rs/moq-clock" } moq-clock-macros = { version = "0.1.0", path = "rs/moq-clock/macros" } moq-flate = { version = "0.2.0", path = "rs/moq-flate" } moq-hls = { version = "0.5.11", path = "rs/moq-hls", default-features = false } +moq-ice = { version = "0.1.0", path = "rs/moq-ice" } moq-json = { version = "0.5.8", path = "rs/moq-json" } moq-loc = { version = "0.2.19", path = "rs/moq-loc" } moq-msf = { version = "0.5.1", path = "rs/moq-msf" } diff --git a/doc/lib/rs/index.md b/doc/lib/rs/index.md index 171427dc9b3..8690d4c9edb 100644 --- a/doc/lib/rs/index.md +++ b/doc/lib/rs/index.md @@ -16,6 +16,7 @@ The reference implementation. Every crate is on | [moq-pattern](https://docs.rs/moq-pattern) | Path patterns such as `room/**`. Re-exported by moq-net and moq-auth. | | [moq-tokio](https://docs.rs/moq-tokio) | Connects and listens over QUIC, WebSocket, or iroh, from config or CLI flags. | | [moq-sock](https://docs.rs/moq-sock) | Socket binding and CPU pinning for thread-per-core listeners. | +| [moq-ice](https://docs.rs/moq-ice) | The ICE agent behind native WebRTC: drives a str0m connection over UDP, with a STUN client. | | [moq-uring](https://docs.rs/moq-uring) | Experimental Linux io\_uring server for moq-lite. | | [hang](/lib/rs/hang) | The media layer: catalog, containers, ordered frame delivery. | | [moq-mux](/lib/rs/moq-mux) | Import and export fMP4/CMAF, MPEG-TS, Matroska, FLV, and Annex-B. | diff --git a/quest/m2/p2p/native-quic.md b/quest/m2/p2p/native-quic.md index fd4767e8aec..4d05a99735e 100644 --- a/quest/m2/p2p/native-quic.md +++ b/quest/m2/p2p/native-quic.md @@ -19,9 +19,9 @@ Decided with the maintainer on 2026-10-13: - ICE is str0m's standalone sans-IO agent, the `is` crate (0.11, already in the lockfile through str0m; str0m re-exports it as `str0m::ice`). Gather candidates with `moq_tokio::webrtc`'s host gathering, `.local` resolution, - and the STUN Binding client in `webrtc::ice`; don't copy them. The first - two are private to the data channel transport today, so move them into - `webrtc::ice` rather than export them where they sit. + and the STUN Binding client in `moq-ice`; don't copy them. The first two + are private to the data channel transport today, so move them into + `moq-ice` rather than export them from `moq-tokio`. - QUIC is `moq-quic`, fed datagrams from the nominated pair. - Authentication: each side generates a self-signed certificate and sends its SHA-256 fingerprint in its offer or answer, as DTLS fingerprints ride diff --git a/rs/moq-ice/CHANGELOG.md b/rs/moq-ice/CHANGELOG.md new file mode 100644 index 00000000000..2a718d0b3af --- /dev/null +++ b/rs/moq-ice/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +- First release: the ICE agent shared by `moq-tokio`'s WebRTC data channel transport and `moq-rtc`'s WHIP/WHEP gateway, with a STUN Binding client and per-peer wire counters. diff --git a/rs/moq-ice/Cargo.toml b/rs/moq-ice/Cargo.toml new file mode 100644 index 00000000000..e34bbdc354c --- /dev/null +++ b/rs/moq-ice/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "moq-ice" +description = "Media over QUIC - drives a str0m WebRTC connection's ICE over a tokio UDP socket" +authors = ["Luke Curley"] +repository = "https://github.com/moq-dev/moq" +license = "MIT OR Apache-2.0" + +version = "0.1.0" +edition = "2024" +rust-version.workspace = true + +keywords = ["webrtc", "ice", "stun", "str0m", "moq"] +categories = ["network-programming", "multimedia"] + +[dependencies] +str0m = { workspace = true } +stun-types = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["macros", "net", "rt", "sync", "time"] } +tracing = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/rs/moq-ice/README.md b/rs/moq-ice/README.md new file mode 100644 index 00000000000..23d9f8b6a60 --- /dev/null +++ b/rs/moq-ice/README.md @@ -0,0 +1,12 @@ +# moq-ice + +Drives a [str0m](https://github.com/algesten/str0m) WebRTC connection over a +tokio UDP socket: the run loop that feeds it datagrams and timeouts, a STUN +Binding client for server-reflexive candidates, and per-peer wire counters. + +str0m is sans-IO and requires every mutation to be followed by draining its +output. `Agent::rtc_mut` hands out a guard that drains when dropped, so callers +can't forget. + +This is infrastructure, not an entry point: `moq-tokio`'s `webrtc` feature +builds MoQ data channels on it, and `moq-rtc` builds its WHIP/WHEP gateway. diff --git a/rs/moq-tokio/src/webrtc/ice.rs b/rs/moq-ice/src/agent.rs similarity index 75% rename from rs/moq-tokio/src/webrtc/ice.rs rename to rs/moq-ice/src/agent.rs index 1c456484b1f..0d0c4ffd98f 100644 --- a/rs/moq-tokio/src/webrtc/ice.rs +++ b/rs/moq-ice/src/agent.rs @@ -1,27 +1,7 @@ -//! The ICE side of a str0m peer connection: one UDP socket, its candidates, and -//! the loop that drives an [`Rtc`] over it. -//! -//! str0m is sans-IO, so [`Agent::next`] alternates between [`Rtc::handle_input`] -//! (datagrams and timeouts) and [`Rtc::poll_output`] (transmits, events, the next -//! timeout), and runs a STUN Binding client on the same socket so the agent can -//! offer server-reflexive candidates. Shared by this crate's data channel -//! transport and `moq-rtc`'s WHIP/WHEP gateway, which only differ in what they do -//! with the events. -//! -//! str0m requires every mutation of an [`Rtc`] to be followed by draining -//! `poll_output` to a timeout before the next one. The agent owns that cycle: it -//! hands out only a shared [`Agent::rtc`], and [`Agent::mutate`] drains after the -//! caller's change, queueing the events for [`Agent::next`]. -//! -//! A dual-stack socket (`[::]`) reports an IPv4 peer as an IPv4-mapped IPv6 -//! address and refuses a send to a real `V4` destination. Neither form leaks past -//! this module: inbound sources are unmapped by [`canonical`], since ICE pairs -//! candidates by address family, and outbound destinations are mapped back. - use std::collections::VecDeque; use std::net::{IpAddr, SocketAddr}; +use std::ops::{Deref, DerefMut}; use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, Instant}; use str0m::net::{Protocol, Receive}; @@ -30,63 +10,8 @@ use stun_types::attribute::XorMappedAddress; use stun_types::message::{BINDING, Message, MessageClass, MessageWriteVec, TransactionId}; use stun_types::prelude::*; use tokio::net::UdpSocket; -use tokio::sync::mpsc; - -/// One inbound UDP datagram and the address it came from. -pub type Packet = (Vec, SocketAddr); - -/// The datagrams routed to one [`Agent`], and the counters they are tallied in. -pub struct Inbound { - /// Ends once every sender is dropped, which ends the agent. - pub packets: mpsc::Receiver, - /// Counts `rx` where datagrams are routed, so a full queue's drops count too; - /// the agent counts `tx`. - pub wire: Wire, -} - -/// Cumulative UDP payload bytes exchanged with one peer, excluding IP and UDP -/// headers. Clones share counters, which outlive the agent. -#[derive(Clone, Debug, Default)] -pub struct Wire(Arc); - -#[derive(Debug, Default)] -struct Counters { - tx: AtomicU64, - rx: AtomicU64, -} - -/// A snapshot of a [`Wire`]'s counters. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct Stats { - /// Bytes the socket accepted, including ICE, DTLS, and retransmissions. - pub tx: u64, - /// Bytes routed to the peer, including those its full queue dropped. - pub rx: u64, -} - -impl Wire { - /// Read both counters without consuming them. - pub fn stats(&self) -> Stats { - Stats { - tx: self.0.tx.load(Ordering::Relaxed), - rx: self.0.rx.load(Ordering::Relaxed), - } - } - - /// Count a datagram routed to this peer. - pub fn received(&self, bytes: usize) { - self.0.rx.fetch_add(bytes as u64, Ordering::Relaxed); - } - fn sent(&self, bytes: usize) { - self.0.tx.fetch_add(bytes as u64, Ordering::Relaxed); - } -} - -/// Bound on a session's inbound datagram queue, sized like a socket buffer: past -/// it, datagrams are dropped rather than buffered, since ICE and SCTP tolerate loss -/// and a stalled session must not grow memory without limit. -pub const INBOX: usize = 256; +use crate::{Inbound, Wire}; /// Backstop deadline for reaching a connected ICE state, covering the one case /// str0m's agent deliberately never times out: a peer that provides no remote @@ -246,12 +171,10 @@ impl Agent { &self.rtc } - /// Change the connection, then drain what the change produced: transmits are - /// sent, and events wait for [`Agent::next`]. - pub fn mutate(&mut self, change: impl FnOnce(&mut Rtc) -> T) -> T { - let result = change(&mut self.rtc); - self.drain(); - result + /// Change the connection through a guard that drains what the change produced + /// when dropped: transmits are sent, and events wait for [`Agent::next`]. + pub fn rtc_mut(&mut self) -> RtcMut<'_> { + RtcMut(self) } /// The counters of the bytes exchanged with the peer. @@ -314,8 +237,10 @@ impl Agent { } let now = Instant::now(); - self.mutate(|rtc| rtc.handle_input(Input::Timeout(now))) - .unwrap_or_else(|err| self.fail(Error::rtc(err))); + let result = self.rtc_mut().handle_input(Input::Timeout(now)); + if let Err(err) = result { + self.fail(Error::rtc(err)); + } self.retransmit(now); } } @@ -374,8 +299,10 @@ impl Agent { tracing::debug!(%src, len = data.len(), "dropped unrecognized datagram"); return; }; - self.mutate(|rtc| rtc.handle_input(Input::Receive(Instant::now(), receive))) - .unwrap_or_else(|err| self.fail(Error::rtc(err))); + let result = self.rtc_mut().handle_input(Input::Receive(Instant::now(), receive)); + if let Err(err) = result { + self.fail(Error::rtc(err)); + } } /// Consume a reply to one of our STUN queries, returning false for anything else. @@ -405,7 +332,8 @@ impl Agent { match Candidate::server_reflexive(mapped, query.base, "udp") { // None: redundant with a candidate the agent already has, as when there is no NAT. Ok(candidate) => { - if let Some(candidate) = self.mutate(|rtc| rtc.add_local_candidate(candidate).cloned()) { + let added = self.rtc_mut().add_local_candidate(candidate).cloned(); + if let Some(candidate) = added { self.events.push_back(Event::Candidate(candidate)); } } @@ -441,6 +369,29 @@ impl Agent { } } +/// Mutable access to an [`Agent`]'s connection, which drains it when dropped. +pub struct RtcMut<'a>(&'a mut Agent); + +impl Deref for RtcMut<'_> { + type Target = Rtc; + + fn deref(&self) -> &Rtc { + &self.0.rtc + } +} + +impl DerefMut for RtcMut<'_> { + fn deref_mut(&mut self) -> &mut Rtc { + &mut self.0.rtc + } +} + +impl Drop for RtcMut<'_> { + fn drop(&mut self) { + self.0.drain(); + } +} + /// Unmap an IPv4-mapped address to the `V4` it really is; others pass through. pub fn canonical(addr: SocketAddr) -> SocketAddr { SocketAddr::new(addr.ip().to_canonical(), addr.port()) @@ -466,36 +417,6 @@ fn pick_local(locals: &[SocketAddr], src: SocketAddr) -> SocketAddr { .unwrap_or(locals[0]) } -/// Pump every datagram from a socket the caller owns 1:1 into a queue for -/// [`Agent::new`], until the queue's receiver is dropped. -pub fn reader(socket: Arc) -> Inbound { - let (tx, rx) = mpsc::channel(INBOX); - let wire = Wire::default(); - let counted = wire.clone(); - tokio::spawn(async move { - let mut buf = vec![0u8; 65_535]; - loop { - tokio::select! { - res = socket.recv_from(&mut buf) => match res { - Ok((len, src)) => { - counted.received(len); - // Bounded like a socket buffer: drop on full. - if let Err(mpsc::error::TrySendError::Closed(_)) = tx.try_send((buf[..len].to_vec(), canonical(src))) { - break; - } - } - Err(err) => { - tracing::warn!(%err, "webrtc socket recv failed"); - break; - } - }, - _ = tx.closed() => break, - } - } - }); - Inbound { packets: rx, wire } -} - #[cfg(test)] mod tests { use super::*; @@ -516,7 +437,7 @@ mod tests { let socket = Arc::new(UdpSocket::bind("127.0.0.1:0").await.unwrap()); let peer = UdpSocket::bind("127.0.0.1:0").await.unwrap(); let local = socket.local_addr().unwrap(); - let inbound = reader(socket.clone()); + let inbound = crate::reader(socket.clone()); let wire = inbound.wire.clone(); let mut agent = Agent::new(Rtc::builder().build(Instant::now()), socket, vec![local], inbound).unwrap(); @@ -525,13 +446,13 @@ mod tests { agent.send(&[42; 73], peer.local_addr().unwrap()); // UDP refuses an oversized datagram, which is never counted as sent. agent.send(&vec![0; 65_536], peer.local_addr().unwrap()); - assert_eq!(wire.stats(), Stats { tx: 73, rx: 0 }); + assert_eq!(wire.stats(), crate::Stats { tx: 73, rx: 0 }); peer.send_to(&[7; 11], local).await.unwrap(); let (data, _) = agent.inbound.packets.recv().await.unwrap(); assert_eq!(data.len(), 11); drop(agent); - assert_eq!(wire.stats(), Stats { tx: 73, rx: 11 }); + assert_eq!(wire.stats(), crate::Stats { tx: 73, rx: 11 }); } #[test] diff --git a/rs/moq-ice/src/lib.rs b/rs/moq-ice/src/lib.rs new file mode 100644 index 00000000000..da2e68a1520 --- /dev/null +++ b/rs/moq-ice/src/lib.rs @@ -0,0 +1,27 @@ +//! The ICE side of a str0m peer connection: one UDP socket, its candidates, and +//! the loop that drives an [`str0m::Rtc`] over it. +//! +//! str0m is sans-IO, so [`Agent::next`] alternates between feeding it datagrams +//! and timeouts and draining what it produces (transmits, events, the next +//! timeout), and runs a STUN Binding client on the same socket so the agent can +//! offer server-reflexive candidates. Shared by `moq-tokio`'s data channel +//! transport and `moq-rtc`'s WHIP/WHEP gateway, which only differ in what they do +//! with the events. +//! +//! str0m requires every mutation of an `Rtc` to be followed by draining its output +//! to a timeout before the next one. The agent owns that cycle: [`Agent::rtc`] is +//! read only, and the [`RtcMut`] guard from [`Agent::rtc_mut`] drains when dropped, +//! queueing the events for [`Agent::next`]. +//! +//! A dual-stack socket (`[::]`) reports an IPv4 peer as an IPv4-mapped IPv6 +//! address and refuses a send to a real `V4` destination. Neither form leaks past +//! this crate: inbound sources are unmapped by [`canonical`], since ICE pairs +//! candidates by address family, and outbound destinations are mapped back. + +#![warn(missing_docs)] + +mod agent; +mod wire; + +pub use agent::{Agent, Error, Event, RtcMut, canonical}; +pub use wire::{INBOX, Inbound, Packet, Stats, Wire, reader}; diff --git a/rs/moq-ice/src/wire.rs b/rs/moq-ice/src/wire.rs new file mode 100644 index 00000000000..5eac3db8738 --- /dev/null +++ b/rs/moq-ice/src/wire.rs @@ -0,0 +1,94 @@ +use std::net::SocketAddr; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use tokio::net::UdpSocket; +use tokio::sync::mpsc; + +use crate::canonical; + +/// One inbound UDP datagram and the address it came from. +pub type Packet = (Vec, SocketAddr); + +/// Bound on a session's inbound datagram queue, sized like a socket buffer: past +/// it, datagrams are dropped rather than buffered, since ICE and SCTP tolerate loss +/// and a stalled session must not grow memory without limit. +pub const INBOX: usize = 256; + +/// The datagrams routed to one [`crate::Agent`], and the counters they are tallied in. +pub struct Inbound { + /// Ends once every sender is dropped, which ends the agent. + pub packets: mpsc::Receiver, + /// Counts `rx` where datagrams are routed, so a full queue's drops count too; + /// the agent counts `tx`. + pub wire: Wire, +} + +/// Cumulative UDP payload bytes exchanged with one peer, excluding IP and UDP +/// headers. Clones share counters, which outlive the agent. +#[derive(Clone, Debug, Default)] +pub struct Wire(Arc); + +#[derive(Debug, Default)] +struct Counters { + tx: AtomicU64, + rx: AtomicU64, +} + +/// A snapshot of a [`Wire`]'s counters. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Stats { + /// Bytes the socket accepted, including ICE, DTLS, and retransmissions. + pub tx: u64, + /// Bytes routed to the peer, including those its full queue dropped. + pub rx: u64, +} + +impl Wire { + /// Read both counters without consuming them. + pub fn stats(&self) -> Stats { + Stats { + tx: self.0.tx.load(Ordering::Relaxed), + rx: self.0.rx.load(Ordering::Relaxed), + } + } + + /// Count a datagram routed to this peer. + pub fn received(&self, bytes: usize) { + self.0.rx.fetch_add(bytes as u64, Ordering::Relaxed); + } + + pub(crate) fn sent(&self, bytes: usize) { + self.0.tx.fetch_add(bytes as u64, Ordering::Relaxed); + } +} + +/// Pump every datagram from a socket the caller owns 1:1 into a queue for +/// [`crate::Agent::new`], until the queue's receiver is dropped. +pub fn reader(socket: Arc) -> Inbound { + let (tx, rx) = mpsc::channel(INBOX); + let wire = Wire::default(); + let counted = wire.clone(); + tokio::spawn(async move { + let mut buf = vec![0u8; 65_535]; + loop { + tokio::select! { + res = socket.recv_from(&mut buf) => match res { + Ok((len, src)) => { + counted.received(len); + // Bounded like a socket buffer: drop on full. + if let Err(mpsc::error::TrySendError::Closed(_)) = tx.try_send((buf[..len].to_vec(), canonical(src))) { + break; + } + } + Err(err) => { + tracing::warn!(%err, "webrtc socket recv failed"); + break; + } + }, + _ = tx.closed() => break, + } + } + }); + Inbound { packets: rx, wire } +} diff --git a/rs/moq-rtc/Cargo.toml b/rs/moq-rtc/Cargo.toml index b296ed18008..f8c637637ad 100644 --- a/rs/moq-rtc/Cargo.toml +++ b/rs/moq-rtc/Cargo.toml @@ -28,9 +28,10 @@ bytes = { workspace = true } hang = { workspace = true } moq-mux = { workspace = true } moq-net = { workspace = true } -# `RedactedUrl` keeps a dialed URL's query token and userinfo out of the logs, and -# `webrtc::ice` drives each str0m connection over its UDP socket. -moq-tokio = { workspace = true, features = ["webrtc"] } +# Drives each str0m connection over its UDP socket. +moq-ice = { workspace = true } +# `RedactedUrl` keeps a dialed URL's query token and userinfo out of the logs. +moq-tokio = { workspace = true } reqwest = { workspace = true, features = ["rustls"] } str0m = { workspace = true } thiserror = { workspace = true } diff --git a/rs/moq-rtc/src/client/whep.rs b/rs/moq-rtc/src/client/whep.rs index 07c8aa63b47..2851de3917d 100644 --- a/rs/moq-rtc/src/client/whep.rs +++ b/rs/moq-rtc/src/client/whep.rs @@ -66,7 +66,7 @@ pub(crate) async fn dial( // 1:1 socket (no demux on the client): pump its datagrams into the session. // The session tags each datagram with the advertised candidate matching its // family (str0m matches the destination against a host candidate, not the bind). - let inbound = moq_tokio::webrtc::ice::reader(socket.clone()); + let inbound = moq_ice::reader(socket.clone()); let mut session = session::Session::ingest(rtc, socket, candidates, inbound, sink)?; for mid in mids { session.open_media(mid)?; diff --git a/rs/moq-rtc/src/client/whip.rs b/rs/moq-rtc/src/client/whip.rs index 9df6023d32c..563f3118b50 100644 --- a/rs/moq-rtc/src/client/whip.rs +++ b/rs/moq-rtc/src/client/whip.rs @@ -78,7 +78,7 @@ pub(crate) async fn dial( // 1:1 socket (no demux on the client): pump its datagrams into the session. // The session tags each datagram with the advertised candidate matching its // family (str0m matches the destination against a host candidate, not the bind). - let inbound = moq_tokio::webrtc::ice::reader(socket.clone()); + let inbound = moq_ice::reader(socket.clone()); let mut session = session::Session::egress(rtc, socket, candidates, inbound, source)?; for mid in mids { session.open_media(mid)?; diff --git a/rs/moq-rtc/src/server/mux.rs b/rs/moq-rtc/src/server/mux.rs index 3eb0e8baf41..df5a852ec86 100644 --- a/rs/moq-rtc/src/server/mux.rs +++ b/rs/moq-rtc/src/server/mux.rs @@ -23,7 +23,7 @@ use std::collections::HashMap; use std::net::SocketAddr; use std::sync::{Arc, Mutex}; -use moq_tokio::webrtc::ice::{self, Inbound, Packet}; +use moq_ice::{Inbound, Packet}; use str0m::ice::{IceCreds, StunMessage}; use tokio::net::UdpSocket; use tokio::sync::mpsc; @@ -45,7 +45,7 @@ struct Registry { struct Entry { ufrag: String, sender: mpsc::Sender, - wire: ice::Wire, + wire: moq_ice::Wire, } /// The shared media socket plus its demux routing table. @@ -125,8 +125,8 @@ impl Mux { /// [`Registration`] guard the session task must hold for its lifetime. pub(crate) fn register(&self) -> (IceCreds, Inbound, Registration) { let creds = IceCreds::new(); - let (tx, rx) = mpsc::channel(ice::INBOX); - let wire = ice::Wire::default(); + let (tx, rx) = mpsc::channel(moq_ice::INBOX); + let wire = moq_ice::Wire::default(); let entry = Arc::new(Entry { ufrag: creds.ufrag.clone(), sender: tx, @@ -171,8 +171,8 @@ async fn demux(socket: Arc, registry: Arc>) { } }; // A dual-stack bind reports IPv4 peers as `::ffff:a.b.c.d`; unmap before - // this address becomes a registry key or reaches ICE (see `ice::canonical`). - let src = ice::canonical(src); + // this address becomes a registry key or reaches ICE (see `moq_ice::canonical`). + let src = moq_ice::canonical(src); let data = &buf[..len]; // Cached peers need one routing lock; only a new address parses STUN. @@ -255,7 +255,7 @@ mod tests { let address_b = "127.0.0.1:10002".parse().unwrap(); let mut registry = mux.registry.lock().unwrap(); assert!(registry.route(100, address_a, Some("unknown")).is_none()); - for _ in 0..ice::INBOX { + for _ in 0..moq_ice::INBOX { let entry = registry.route(11, address_a, Some(&a.ufrag)).unwrap(); entry.sender.try_send((vec![0; 11], address_a)).unwrap(); } @@ -268,12 +268,12 @@ mod tests { drop(registry); assert_eq!( source_a.stats(), - ice::Stats { + moq_ice::Stats { tx: 0, - rx: 11 * ice::INBOX as u64 + 23 + rx: 11 * moq_ice::INBOX as u64 + 23 } ); - assert_eq!(source_b.stats(), ice::Stats { tx: 0, rx: 37 }); + assert_eq!(source_b.stats(), moq_ice::Stats { tx: 0, rx: 37 }); // Unregister even while an inbox is still held: cached addresses cannot // keep attributing traffic after the peer ends. @@ -300,7 +300,7 @@ mod tests { let mut addresses = Vec::new(); for peer in 0..peers { let (sender, receiver) = mpsc::channel(1); - let wire = ice::Wire::default(); + let wire = moq_ice::Wire::default(); let address = SocketAddr::from(([127, 0, 0, 1], peer as u16 + 1)); let entry = Arc::new(Entry { ufrag: peer.to_string(), diff --git a/rs/moq-rtc/src/session.rs b/rs/moq-rtc/src/session.rs index 06bef5f7701..88cdfbaa098 100644 --- a/rs/moq-rtc/src/session.rs +++ b/rs/moq-rtc/src/session.rs @@ -1,6 +1,6 @@ //! str0m session driver shared by every HTTP role / media direction. //! -//! [`moq_tokio::webrtc::ice::Agent`] drives the sans-IO [`str0m::Rtc`] over UDP; +//! [`moq_tokio::webrtc::moq_ice::Agent`] drives the sans-IO [`str0m::Rtc`] over UDP; //! [`Session::run`] feeds it egress writes and handles the media events it surfaces. //! //! The session itself doesn't care whether the [`Rtc`] was populated by @@ -13,7 +13,6 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::sync::Arc; use std::time::{Instant, SystemTime, UNIX_EPOCH}; -use moq_tokio::webrtc::ice; use str0m::{Candidate, Event, Rtc}; use tokio::net::UdpSocket; use tokio::sync::mpsc; @@ -81,7 +80,7 @@ impl MediaFlow { /// `inbound` rather than being read off the socket directly, so several /// sessions can share one socket behind the `crate::server::mux`. pub struct Session { - agent: ice::Agent, + agent: moq_ice::Agent, role: MediaRole, opened_media: HashSet, /// Egress write requests. `Some` only for [`MediaRole::Egress`] @@ -98,17 +97,17 @@ pub struct Session { impl Session { /// Convenience for the ingest case (WHIP server, WHEP client). `locals` are the - /// advertised ICE candidates, not the socket bind (see [`ice::Agent::new`]). - /// `inbound` comes from the server mux, or [`ice::reader`] for a 1:1 client socket. + /// advertised ICE candidates, not the socket bind (see [`moq_ice::Agent::new`]). + /// `inbound` comes from the server mux, or [`moq_ice::reader`] for a 1:1 client socket. pub fn ingest( rtc: Rtc, socket: Arc, locals: Vec, - inbound: ice::Inbound, + inbound: moq_ice::Inbound, sink: Box, ) -> Result { Ok(Self { - agent: ice::Agent::new(rtc, socket, locals, inbound)?, + agent: moq_ice::Agent::new(rtc, socket, locals, inbound)?, role: MediaRole::Ingest(sink), opened_media: HashSet::new(), writes_rx: None, @@ -118,18 +117,18 @@ impl Session { } /// Convenience for the egress case (WHEP server, WHIP client). `locals` are the - /// advertised ICE candidates, not the socket bind (see [`ice::Agent::new`]). - /// `inbound` comes from the server mux, or [`ice::reader`] for a 1:1 client socket. + /// advertised ICE candidates, not the socket bind (see [`moq_ice::Agent::new`]). + /// `inbound` comes from the server mux, or [`moq_ice::reader`] for a 1:1 client socket. pub fn egress( rtc: Rtc, socket: Arc, locals: Vec, - inbound: ice::Inbound, + inbound: moq_ice::Inbound, mut source: EgressSource, ) -> Result { let writes_rx = source.take_writes(); Ok(Self { - agent: ice::Agent::new(rtc, socket, locals, inbound)?, + agent: moq_ice::Agent::new(rtc, socket, locals, inbound)?, role: MediaRole::Egress(Box::new(source)), opened_media: HashSet::new(), writes_rx: Some(writes_rx), @@ -174,15 +173,15 @@ impl Session { } } => { let wallclock = self.egress_clock.wallclock(req.time, Instant::now()); - self.agent.mutate(|rtc| crate::egress::dispatch(rtc, req, wallclock)); + crate::egress::dispatch(&mut self.agent.rtc_mut(), req, wallclock); } event = self.agent.next() => match event { - Ok(ice::Event::Rtc(event)) => self.handle_event(event)?, + Ok(moq_ice::Event::Rtc(event)) => self.handle_event(event)?, // No STUN servers are configured, so there is nothing to trickle. Ok(_) => {} - Err(ice::Error::Closed) => return Err(Error::SessionClosed), - Err(ice::Error::Timeout) => return Err(Error::IceTimeout), + Err(moq_ice::Error::Closed) => return Err(Error::SessionClosed), + Err(moq_ice::Error::Timeout) => return Err(Error::IceTimeout), Err(err) => return Err(Error::rtc(err)), }, } @@ -516,7 +515,7 @@ pub fn rtc_with_codecs(codecs: &[str0m::format::Codec]) -> Rtc { } /// Bind an ephemeral UDP socket for a single client session and return it -/// (to share with its [`ice::reader`]) plus the ICE candidates +/// (to share with its [`moq_ice::reader`]) plus the ICE candidates /// to advertise. /// /// The client paths are 1:1 (one socket per dialed session, no demux); the diff --git a/rs/moq-rtc/src/wire.rs b/rs/moq-rtc/src/wire.rs index 2cf816028db..2580970d852 100644 --- a/rs/moq-rtc/src/wire.rs +++ b/rs/moq-rtc/src/wire.rs @@ -1,16 +1,14 @@ //! Cumulative UDP payload bytes for one WebRTC peer. -use moq_tokio::webrtc::ice; - /// A peer's cumulative datagram bytes, excluding IP and UDP headers. -pub use ice::Stats; +pub use moq_ice::Stats; /// A read-only source whose counters survive the WebRTC session ending. #[derive(Clone, Debug)] -pub struct Source(ice::Wire); +pub struct Source(moq_ice::Wire); impl Source { - pub(crate) fn new(wire: ice::Wire) -> Self { + pub(crate) fn new(wire: moq_ice::Wire) -> Self { Self(wire) } diff --git a/rs/moq-tokio/Cargo.toml b/rs/moq-tokio/Cargo.toml index d6be9de1f04..6eb358877a9 100644 --- a/rs/moq-tokio/Cargo.toml +++ b/rs/moq-tokio/Cargo.toml @@ -61,7 +61,7 @@ tcp = ["_transport", "dep:qmux", "dep:dns-lookup"] # inspection on accept. Pulls in `tcp` (shares qmux's stream transport). uds = ["tcp", "qmux/uds"] # Native WebRTC data channels over str0m, with a full ICE agent (the `webrtc` module). -webrtc = ["_transport", "dep:qmux", "dep:str0m", "dep:stun-types", "dep:if-addrs", "dep:mdns-sd"] +webrtc = ["_transport", "dep:qmux", "dep:moq-ice", "dep:str0m", "dep:if-addrs", "dep:mdns-sd"] ring = ["_certs", "rustls/ring", "rcgen/ring", "moq-quic?/rustls-ring"] [dependencies] @@ -79,6 +79,8 @@ kio = { workspace = true, optional = true } mdns-sd = { workspace = true, optional = true } moq-clock = { workspace = true, optional = true } +# The ICE agent behind the `webrtc` module. Private, like str0m: neither appears in this crate's API. +moq-ice = { workspace = true, optional = true } moq-net = { workspace = true } # The QUIC state machine the `quic` module drives; the crypto-provider features pick its provider. moq-quic = { workspace = true, features = ["bloom"], optional = true } @@ -105,7 +107,6 @@ serde_with = { version = "3", features = ["hex"] } sha2 = { workspace = true, optional = true } socket2 = { workspace = true, optional = true } str0m = { workspace = true, optional = true } -stun-types = { workspace = true, optional = true } subtle = { workspace = true, optional = true } thiserror = "2" tikv-jemalloc-ctl = { version = "0.6", optional = true } @@ -143,6 +144,8 @@ chrono = "0.4" # be driven with synthetic daemon events. flume = "0.12" rcgen = { version = "0.14", default-features = false, features = ["pem", "aws_lc_rs"] } +# A fake STUN server for the WebRTC server-reflexive candidate test. +stun-types = { workspace = true } tempfile = "3" # test-util adds tokio::time::pause/auto-advance for the time-dependent tests. tokio = { workspace = true, features = ["test-util"] } diff --git a/rs/moq-tokio/src/client.rs b/rs/moq-tokio/src/client.rs index 83a76ba355b..7016cad2300 100644 --- a/rs/moq-tokio/src/client.rs +++ b/rs/moq-tokio/src/client.rs @@ -312,7 +312,7 @@ impl Client { let attempt = async { let opened = peer.channel().await?; - let config = opened.config(); + let config = opened.config; let session = qmux::Session::connect(opened.channel, config) .await .map_err(crate::webrtc::Error::from)?; diff --git a/rs/moq-tokio/src/error.rs b/rs/moq-tokio/src/error.rs index 5c87cec4cf6..abde1fdf9d6 100644 --- a/rs/moq-tokio/src/error.rs +++ b/rs/moq-tokio/src/error.rs @@ -191,11 +191,11 @@ pub enum Error { #[error("{0}")] Reconnect(String), - /// The connection was stopped locally, by closing it or dropping the last handle. + /// The connection or listener was stopped locally, by closing it or dropping the last handle. /// /// Not a failure: it is what a caller asked for. Distinct from [`Self::Reconnect`] /// so a status watcher can tell an expected teardown from a connection that gave up. - #[error("connection stopped")] + #[error("stopped locally")] Stopped, /// A task serving this connection panicked. The panic itself is logged. diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 1a06089aba3..993b17b60d7 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -882,16 +882,14 @@ impl Listener { /// [`crate::webrtc::Answerer`], leaving admission to the returned [`Request`]. /// /// Waits for ICE and the channel to come up, bounded by the handshake timeout. - /// The channel's protocol must be a MoQ ALPN this server speaks. + /// The channel's protocol must be a MoQ ALPN this server speaks. Returns + /// [`Error::Stopped`] once the listener is closing. #[cfg(feature = "webrtc")] pub async fn accept_webrtc(&self, peer: &crate::webrtc::Answerer) -> crate::Result { use crate::webrtc::Error as WebRtcError; let server = &self.server; - let active = server - .activity - .enter() - .ok_or(Error::NoBackend("the listener is closing"))?; + let active = server.activity.enter().ok_or(Error::Stopped)?; let deadline = server .timeout .and_then(|timeout| tokio::time::Instant::now().checked_add(timeout)); @@ -901,7 +899,7 @@ impl Listener { if !server.versions.alpns().contains(&opened.protocol.as_str()) { return Err(WebRtcError::Protocol(opened.protocol)); } - let config = opened.config(); + let config = opened.config; let protocol = opened.protocol; let session = qmux::Session::accept(opened.channel, config).await?; Ok((session, protocol)) diff --git a/rs/moq-tokio/src/webrtc.rs b/rs/moq-tokio/src/webrtc.rs index 4cf00ec26e8..1a53c8f7ba7 100644 --- a/rs/moq-tokio/src/webrtc.rs +++ b/rs/moq-tokio/src/webrtc.rs @@ -31,7 +31,6 @@ use tokio::net::UdpSocket; use tokio::sync::{mpsc, oneshot}; mod channel; -pub mod ice; /// The data channel label, as `@moq/p2p` creates it. const LABEL: &str = "moq"; @@ -49,10 +48,11 @@ const DEFAULT_MAX_MESSAGE_SIZE: u64 = 64 * 1024; /// Bytes the channel may hold unsent before qmux's writer waits, as `@moq/p2p` allows. const SEND_WINDOW: usize = RECORD_SIZE as usize * 4; -/// Records read off the channel that qmux has yet to take, matching the send -/// window. A full queue stops the driver reading the socket, so a slow reader -/// pushes back on the peer through SCTP's congestion control, as TCP would. -const READ_QUEUE: usize = SEND_WINDOW / RECORD_SIZE as usize; +/// Unread bytes the channel tolerates past qmux's advertised `max_data`, for the +/// frame headers and control frames that flow control does not count. A peer that +/// honors flow control with reasonably full records stays under it; one that +/// ignores it has the channel closed instead of growing memory without limit. +const READ_SLACK: u64 = 16 * RECORD_SIZE; /// How long to wait for a `.local` name to resolve: mdns-sd's first query plus its /// retransmissions at one and two seconds (RFC 6762 section 5.2). @@ -69,9 +69,17 @@ pub enum Error { #[error(transparent)] Io(#[from] std::io::Error), - /// The peer connection ended. - #[error(transparent)] - Ice(#[from] ice::Error), + /// The peer disconnected, or the connection was closed. + #[error("peer connection closed")] + Closed, + + /// ICE never connected before the establishment deadline. + #[error("ICE did not connect before the establishment deadline")] + Timeout, + + /// The WebRTC stack failed (DTLS, SCTP, or ICE state). + #[error("rtc error: {0}")] + Rtc(String), /// A session description failed to parse or apply. #[error("invalid session description: {0}")] @@ -116,6 +124,18 @@ pub enum Error { crate::error::from_message!(qmux::Error => Qmux); +impl Error { + /// Adopt the agent's error. Not a `From` impl, which would put moq-ice in this crate's API. + fn ice(err: moq_ice::Error) -> Self { + match err { + moq_ice::Error::Closed => Self::Closed, + moq_ice::Error::Timeout => Self::Timeout, + moq_ice::Error::Rtc(err) => Self::Rtc(err), + err => Self::Rtc(err.to_string()), + } + } +} + /// How an [`Offerer`] or [`Answerer`] gathers its local candidates. #[derive(Clone, Debug, Default)] #[non_exhaustive] @@ -280,7 +300,7 @@ impl Offerer { let (tx, rx) = oneshot::channel(); self.peer.send(Command::Answer(parsed, max_message, tx))?; - rx.await.map_err(|_| ice::Error::Closed)??; + rx.await.map_err(|_| Error::Closed)??; self.peer.resolve(mdns); Ok(()) } @@ -381,8 +401,8 @@ impl Peer { max_message: Option, ) -> Result { let stun = resolve_stun(&config.stun).await; - let inbound = ice::reader(socket.clone()); - let agent = ice::Agent::new(rtc, socket, locals, inbound)?.with_stun(&stun); + let inbound = moq_ice::reader(socket.clone()); + let agent = moq_ice::Agent::new(rtc, socket, locals, inbound)?.with_stun(&stun); let (commands, rx) = mpsc::unbounded_channel(); let (candidates_tx, candidates) = mpsc::unbounded_channel(); @@ -415,14 +435,14 @@ impl Peer { async fn channel(&self) -> Result { let (tx, rx) = oneshot::channel(); self.send(Command::Channel(tx))?; - let mut opened = rx.await.map_err(|_| ice::Error::Closed)??; + let mut opened = rx.await.map_err(|_| Error::Closed)??; // The session keeps the driver alive once this handle is gone. opened.channel.writer._driver = Some(self.commands.clone()); Ok(opened) } fn send(&self, command: Command) -> Result<(), Error> { - self.commands.send(command).map_err(|_| ice::Error::Closed.into()) + self.commands.send(command).map_err(|_| Error::Closed) } /// Add the `.local` candidates a description carried once mDNS resolves them. @@ -446,21 +466,20 @@ pub(crate) struct Opened { pub channel: channel::Channel, /// The MoQ ALPN the channel carries. pub protocol: String, - /// The record size to advertise: 16 KiB, capped by the peer's SCTP message limit. - pub max_record_size: u64, + /// The qmux session settings for this channel. + pub config: qmux::Config, } -impl Opened { - /// The qmux session settings for this channel. - pub fn config(&self) -> qmux::Config { - let mut config = crate::qmux_config(); - config.version = qmux::Version::QMux01; - config.protocol = qmux::Protocol::Negotiated(self.protocol.clone()); - config.max_record_size = self.max_record_size; - // The binding has no datagrams. - config.max_datagram_frame_size = 0; - config - } +/// The qmux session settings for a channel carrying `protocol`, advertising +/// `max_record_size`: 16 KiB, capped by the peer's SCTP message limit. +fn session_config(protocol: &str, max_record_size: u64) -> qmux::Config { + let mut config = crate::qmux_config(); + config.version = qmux::Version::QMux01; + config.protocol = qmux::Protocol::Negotiated(protocol.to_string()); + config.max_record_size = max_record_size; + // The binding has no datagrams. + config.max_datagram_frame_size = 0; + config } pub(crate) enum Command { @@ -482,17 +501,17 @@ enum Role { struct Open { id: ChannelId, protocol: String, - reads: mpsc::Sender>, + reads: channel::Inbox, /// `None` once the session's writer closed. writes: Option>, - max_record_size: u64, + config: qmux::Config, /// The session's halves, until a session takes them. halves: Option, } /// Owns the connection and runs it on its own task. struct Driver { - agent: ice::Agent, + agent: moq_ice::Agent, commands: mpsc::UnboundedReceiver, /// Dropped once gathering finishes, ending [`Offerer::candidate`] and [`Answerer::candidate`]. candidates: Option>, @@ -510,11 +529,6 @@ impl Driver { async fn run(mut self) { let err = loop { let writable = self.writable(); - // Only a full read queue is kept, to wait for the session to take a record. - let full = match &self.channel { - Some(Ok(open)) if open.reads.capacity() == 0 && !open.reads.is_closed() => Some(open.reads.clone()), - _ => None, - }; let writes = match &mut self.channel { Some(Ok(Open { writes: Some(writes), .. @@ -523,12 +537,11 @@ impl Driver { }; tokio::select! { - // Each event carries at most one record, so the read queue never overflows. - event = self.agent.next(), if full.is_none() => match event { + event = self.agent.next() => match event { Ok(event) => if let Err(err) = self.event(event) { break err; }, - Err(err) => break err.into(), + Err(err) => break Error::ice(err), }, command = self.commands.recv() => match command { Some(command) => self.command(command), @@ -540,13 +553,11 @@ impl Driver { break err; } } - // Closed or not, the queue has room once this returns. - _ = async { full.as_ref().unwrap().reserve().await.ok(); }, if full.is_some() => {} } }; match &err { - Error::Ice(ice::Error::Closed) => tracing::debug!("peer connection closed"), + Error::Closed => tracing::debug!("peer connection closed"), err => tracing::debug!(%err, "peer connection failed"), } if let Some(waiter) = self.waiter.take() { @@ -563,10 +574,10 @@ impl Driver { return false; }; let id = *id; - self.agent.mutate(|rtc| { - rtc.channel(id) - .is_some_and(|mut channel| channel.buffered_amount() < SEND_WINDOW) - }) + self.agent + .rtc_mut() + .channel(id) + .is_some_and(|mut channel| channel.buffered_amount() < SEND_WINDOW) } /// Hand a record from the session to SCTP, or close the channel when the session's writer closed. @@ -577,19 +588,18 @@ impl Driver { let id = open.id; let Some(data) = data else { open.writes = None; - self.agent.mutate(|rtc| rtc.direct_api().close_data_channel(id)); + self.agent.rtc_mut().direct_api().close_data_channel(id); return Ok(()); }; - let written = self.agent.mutate(|rtc| { - let mut channel = rtc.channel(id).ok_or(ice::Error::Closed)?; - channel - .write(true, &data) - .map_err(|err| ice::Error::Rtc(err.to_string())) - })?; + let written = { + let mut rtc = self.agent.rtc_mut(); + let mut channel = rtc.channel(id).ok_or(Error::Closed)?; + channel.write(true, &data).map_err(|err| Error::Rtc(err.to_string()))? + }; // The send window keeps the buffer well below str0m's, so a refusal is a bug, not backpressure. match written { true => Ok(()), - false => Err(ice::Error::Rtc("SCTP refused a record within the send window".into()).into()), + false => Err(Error::Rtc("SCTP refused a record within the send window".into())), } } @@ -602,16 +612,17 @@ impl Driver { Role::Answerer => None, }; let result = match pending { - Some(pending) => self - .agent - .mutate(|rtc| rtc.sdp_api().accept_answer(pending, answer)) - .map(|()| self.max_message = Some(max_message)) - .map_err(|err| Error::Sdp(err.to_string())), + Some(pending) => { + let accepted = self.agent.rtc_mut().sdp_api().accept_answer(pending, answer); + accepted + .map(|()| self.max_message = Some(max_message)) + .map_err(|err| Error::Sdp(err.to_string())) + } None => Err(Error::Sdp("the answer was already applied".into())), }; let _ = reply.send(result); } - Command::Candidate(candidate) => self.agent.mutate(|rtc| rtc.add_remote_candidate(candidate)), + Command::Candidate(candidate) => self.agent.rtc_mut().add_remote_candidate(candidate), Command::Channel(reply) => { if self.taken { let _ = reply.send(Err(Error::Taken)); @@ -624,19 +635,20 @@ impl Driver { } } - fn event(&mut self, event: ice::Event) -> Result<(), Error> { + fn event(&mut self, event: moq_ice::Event) -> Result<(), Error> { let event = match event { - ice::Event::Candidate(candidate) => { + moq_ice::Event::Candidate(candidate) => { if let Some(candidates) = &self.candidates { let _ = candidates.send(Candidate(candidate.to_sdp_string())); } return Ok(()); } - ice::Event::Gathered => { + moq_ice::Event::Gathered => { self.candidates = None; return Ok(()); } - ice::Event::Rtc(event) => event, + moq_ice::Event::Rtc(event) => event, + _ => return Ok(()), }; match event { @@ -650,14 +662,16 @@ impl Driver { } let record = if !data.binary { Err(qmux::Error::ProtocolViolation) - } else if data.data.len() as u64 > open.max_record_size { + } else if data.data.len() as u64 > open.config.max_record_size { Err(qmux::Error::FrameTooLarge) } else { Ok(Bytes::from(data.data)) }; - // Room was checked before this event was taken; a closed reader drops it. - if let Err(mpsc::error::TrySendError::Full(_)) = open.reads.try_send(record) { - unreachable!("the driver takes no event while the read queue is full"); + if !open.reads.push(record) { + tracing::debug!("closing the data channel: the peer ignored qmux flow control"); + let id = open.id; + self.agent.rtc_mut().direct_api().close_data_channel(id); + self.channel = Some(Err(Error::Closed)); } } str0m::Event::ChannelClose(id) => { @@ -665,7 +679,7 @@ impl Driver { && open.id == id { // The session's reader sees the end of its records. - self.channel = Some(Err(ice::Error::Closed.into())); + self.channel = Some(Err(Error::Closed)); } } _ => {} @@ -678,21 +692,22 @@ impl Driver { let expected = match &self.role { Role::Offerer { channel, .. } => *channel == id, // The remote peer may open other channels first; only the `moq` one is the session's. - Role::Answerer => self.agent.mutate(|rtc| { - rtc.channel(id) - .and_then(|channel| channel.config().map(|config| config.label == LABEL)) - .unwrap_or(false) - }), + Role::Answerer => self + .agent + .rtc_mut() + .channel(id) + .and_then(|channel| channel.config().map(|config| config.label == LABEL)) + .unwrap_or(false), }; if !expected || self.channel.is_some() { tracing::debug!(?id, "closing an unexpected data channel"); - self.agent.mutate(|rtc| rtc.direct_api().close_data_channel(id)); + self.agent.rtc_mut().direct_api().close_data_channel(id); return; } let opened = self.adopt(id); if opened.is_err() { - self.agent.mutate(|rtc| rtc.direct_api().close_data_channel(id)); + self.agent.rtc_mut().direct_api().close_data_channel(id); } self.channel = Some(opened); self.deliver(); @@ -705,32 +720,34 @@ impl Driver { return Err(Error::MessageSize(max_message)); } - let protocol = self.agent.mutate(|rtc| { - let mut channel = rtc.channel(id).ok_or(ice::Error::Closed)?; + let protocol = { + let mut rtc = self.agent.rtc_mut(); + let mut channel = rtc.channel(id).ok_or(Error::Closed)?; let config = channel.config().ok_or(Error::Unreliable)?; if !config.ordered || config.reliability != Reliability::Reliable { return Err(Error::Unreliable); } let protocol = config.protocol.clone(); channel.set_buffered_amount_low_threshold(SEND_WINDOW / 2); - Ok(protocol) - })?; + protocol + }; - let (reads, rx) = mpsc::channel(READ_QUEUE); + let config = session_config(&protocol, max_record_size); + let (reads, reader) = channel::queue(config.max_data + READ_SLACK); let (tx, writes) = mpsc::channel(1); Ok(Open { id, protocol, reads, writes: Some(writes), - max_record_size, + config, halves: Some(channel::Channel { writer: channel::Writer { tx: Some(tx), max: usize::try_from(max_message).unwrap_or(usize::MAX), _driver: None, }, - reader: channel::Reader { rx }, + reader, }), }) } @@ -747,12 +764,12 @@ impl Driver { Ok(open) => Ok(Opened { channel: open.halves.take().expect("only the first session takes the halves"), protocol: open.protocol.clone(), - max_record_size: open.max_record_size, + config: open.config.clone(), }), Err(err) => Err(match err { Error::Unreliable => Error::Unreliable, Error::MessageSize(size) => Error::MessageSize(*size), - _ => ice::Error::Closed.into(), + _ => Error::Closed, }), }; let _ = waiter.send(result); diff --git a/rs/moq-tokio/src/webrtc/channel.rs b/rs/moq-tokio/src/webrtc/channel.rs index 81e9df96852..86583f8e358 100644 --- a/rs/moq-tokio/src/webrtc/channel.rs +++ b/rs/moq-tokio/src/webrtc/channel.rs @@ -4,6 +4,9 @@ //! WebSocket message does. The peer's driver task owns the channel; these halves //! only hand messages to and from it. +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + use bytes::Bytes; use tokio::sync::mpsc; @@ -48,14 +51,79 @@ impl qmux::transport::Writer for Writer { } } +/// A queue of inbound records bounded by bytes the session has yet to read. +/// +/// The driver never stops reading the socket to wait for the session: that would +/// also stall ICE consent checks and the SACKs that free our own send buffer. +/// qmux's flow control bounds what an honest peer can have in flight, so the +/// queue only overflows when the peer ignores it, and then the channel closes. +pub(crate) fn queue(budget: u64) -> (Inbox, Reader) { + let (tx, rx) = mpsc::unbounded_channel(); + let unread = Arc::new(AtomicU64::new(0)); + let inbox = Inbox { + tx, + unread: unread.clone(), + budget, + }; + (inbox, Reader { rx, unread }) +} + +/// The driver's end of the inbound queue. +pub(crate) struct Inbox { + tx: mpsc::UnboundedSender>, + /// Bytes queued but not yet read, shared with the [`Reader`]. + unread: Arc, + budget: u64, +} + +impl Inbox { + /// Queue a record for the session, or end its reads with a flow control error + /// and return false once that would leave more than the budget unread. + pub fn push(&self, record: Result) -> bool { + let len = record.as_ref().map_or(0, |data| data.len() as u64); + if self.unread.fetch_add(len, Ordering::Relaxed) + len > self.budget { + let _ = self.tx.send(Err(qmux::Error::FlowControlError)); + return false; + } + // A closed reader drops the record. + let _ = self.tx.send(record); + true + } +} + /// Receives the records the driver read off the channel. pub(crate) struct Reader { - /// Bounded: while it is full, the driver stops reading the socket. - pub rx: mpsc::Receiver>, + rx: mpsc::UnboundedReceiver>, + unread: Arc, } impl qmux::transport::Reader for Reader { async fn recv(&mut self) -> Result { - self.rx.recv().await.unwrap_or(Err(qmux::Error::Closed)) + let record = self.rx.recv().await.unwrap_or(Err(qmux::Error::Closed))?; + self.unread.fetch_sub(record.len() as u64, Ordering::Relaxed); + Ok(record) + } +} + +#[cfg(test)] +mod tests { + use qmux::transport::Reader as _; + + use super::*; + + #[tokio::test] + async fn queue_closes_past_its_budget() { + let (inbox, mut reader) = queue(10); + assert!(inbox.push(Ok(Bytes::from_static(&[0; 6])))); + assert!(inbox.push(Ok(Bytes::from_static(&[1; 4])))); + // Reading frees budget for the next record. + assert_eq!(reader.recv().await.unwrap().len(), 6); + assert!(inbox.push(Ok(Bytes::from_static(&[2; 6])))); + assert!(!inbox.push(Ok(Bytes::from_static(&[3; 1])))); + + // Records queued before the overflow still arrive, then the error. + assert_eq!(reader.recv().await.unwrap()[0], 1); + assert_eq!(reader.recv().await.unwrap()[0], 2); + assert!(matches!(reader.recv().await, Err(qmux::Error::FlowControlError))); } } diff --git a/rs/moq-tokio/src/webrtc/tests.rs b/rs/moq-tokio/src/webrtc/tests.rs index 5bce0473b48..51241bf1bae 100644 --- a/rs/moq-tokio/src/webrtc/tests.rs +++ b/rs/moq-tokio/src/webrtc/tests.rs @@ -83,8 +83,8 @@ async fn session_over_loopback() { transfer(&offerer, &answerer, b"hello").await; } -/// A frame many records long fills the SCTP send window and the read queue, so -/// both sides have to wait for the other and resume. +/// A frame many records long fills the SCTP send window, so the writer has to +/// wait for the peer's acknowledgements and resume. #[tokio::test] async fn large_frame_over_loopback() { let config = Config::default(); @@ -96,6 +96,71 @@ async fn large_frame_over_loopback() { transfer(&offerer, &answerer, &payload).await; } +/// Open the data channel between two loopback peers, without a MoQ session over it. +async fn channels() -> (Offerer, Answerer, Opened, Opened) { + let config = Config::default(); + let (offerer, offer) = Offerer::with_hosts(&config, protocol(), vec![LOOPBACK]).await.unwrap(); + let (answerer, answer) = Answerer::with_hosts(&config, &offer, vec![LOOPBACK]).await.unwrap(); + offerer.accept(&answer).await.unwrap(); + let (sent, received) = tokio::time::timeout(TIMEOUT, async { tokio::join!(offerer.channel(), answerer.channel()) }) + .await + .expect("the channel did not open"); + (offerer, answerer, sent.unwrap(), received.unwrap()) +} + +/// A full record, numbered so its order can be checked. +fn record(index: usize) -> Bytes { + Bytes::from(vec![index as u8; RECORD_SIZE as usize]) +} + +/// A session that is not reading must not stop the peer connection: the agent +/// keeps acknowledging, so the remote writer gets well past the send window. +#[tokio::test] +async fn unread_records_do_not_stall_the_writer() { + use qmux::transport::{Reader as _, Writer as _}; + + let (_offerer, _answerer, sent, received) = channels().await; + let (mut writer, mut reader) = (sent.channel.writer, received.channel.reader); + let count = 8 * SEND_WINDOW / RECORD_SIZE as usize; + tokio::time::timeout(TIMEOUT, async { + for index in 0..count { + writer.send(record(index)).await.unwrap(); + } + }) + .await + .expect("the writer stalled while the reader was idle"); + + for index in 0..count { + assert_eq!(reader.recv().await.unwrap(), record(index)); + } +} + +/// A peer that sends past qmux's receive window while the session is not reading +/// has the channel closed, after the records within the budget. +#[tokio::test] +async fn ignoring_flow_control_closes_the_channel() { + use qmux::transport::{Reader as _, Writer as _}; + + let (_offerer, _answerer, sent, received) = channels().await; + let budget = received.config.max_data + READ_SLACK; + let within = (budget / RECORD_SIZE) as usize; + let (mut writer, mut reader) = (sent.channel.writer, received.channel.reader); + tokio::time::timeout(TIMEOUT, async { + for index in 0..within + 8 { + if writer.send(record(index)).await.is_err() { + break; + } + } + }) + .await + .expect("the writer stalled"); + + for index in 0..within { + assert_eq!(reader.recv().await.unwrap(), record(index)); + } + assert!(matches!(reader.recv().await, Err(qmux::Error::FlowControlError))); +} + /// Answer STUN Binding requests with the source address, as a public server would. async fn stun_server() -> SocketAddr { use stun_types::attribute::XorMappedAddress; From a244d5774548adaa27a2553724f1256597ab8d04 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 16:50:04 -0700 Subject: [PATCH 07/15] fix(ice): bound the send backlog and hold an offer's ICE deadline A socket that stays full while changes keep coming no longer grows the outbox without limit: past the inbound queue's size, datagrams drop as a full socket buffer would. An offerer's agent holds its establishment deadline until the answer applies, so signaling time is not charged against ICE. Co-Authored-By: Claude Opus 5.5 --- rs/moq-ice/src/agent.rs | 77 ++++++++++++++++++++++++++++++++------ rs/moq-tokio/src/webrtc.rs | 6 ++- 2 files changed, 71 insertions(+), 12 deletions(-) diff --git a/rs/moq-ice/src/agent.rs b/rs/moq-ice/src/agent.rs index 0d0c4ffd98f..d3fdf54bfdb 100644 --- a/rs/moq-ice/src/agent.rs +++ b/rs/moq-ice/src/agent.rs @@ -102,7 +102,8 @@ pub struct Agent { timeout: Instant, /// Datagrams the socket refused for lack of buffer space, sent first next time. outbox: VecDeque<(Vec, SocketAddr)>, - started: Instant, + /// When the [`ESTABLISH_TIMEOUT`] started, or `None` while an offer awaits its answer. + started: Option, connected: bool, } @@ -132,7 +133,7 @@ impl Agent { failed: None, timeout: now, outbox: VecDeque::new(), - started: now, + started: Some(now), connected: false, }; // Signaling already changed the connection. @@ -171,9 +172,22 @@ impl Agent { &self.rtc } + /// Hold off the deadline for ICE to connect until the first [`Agent::rtc_mut`], + /// for an offer whose answer has yet to arrive: signaling time is the caller's. + pub fn awaiting_answer(mut self) -> Self { + self.started = None; + self + } + /// Change the connection through a guard that drains what the change produced /// when dropped: transmits are sent, and events wait for [`Agent::next`]. pub fn rtc_mut(&mut self) -> RtcMut<'_> { + self.started.get_or_insert_with(Instant::now); + self.change() + } + + /// [`Agent::rtc_mut`] for the agent's own changes, which never start the deadline. + fn change(&mut self) -> RtcMut<'_> { RtcMut(self) } @@ -212,14 +226,16 @@ impl Agent { if !self.rtc.is_alive() { return Err(Error::Closed); } - if !self.connected && self.started.elapsed() >= ESTABLISH_TIMEOUT { + let establish = self + .started + .filter(|_| !self.connected) + .map(|at| at + ESTABLISH_TIMEOUT); + if establish.is_some_and(|at| at <= Instant::now()) { return Err(Error::Timeout); } - let mut deadline = self.stun.iter().map(|q| q.deadline).fold(self.timeout, Instant::min); - if !self.connected { - deadline = deadline.min(self.started + ESTABLISH_TIMEOUT); - } + let deadline = self.stun.iter().map(|q| q.deadline).chain(establish); + let deadline = deadline.fold(self.timeout, Instant::min); let now = Instant::now(); if deadline > now { @@ -237,7 +253,7 @@ impl Agent { } let now = Instant::now(); - let result = self.rtc_mut().handle_input(Input::Timeout(now)); + let result = self.change().handle_input(Input::Timeout(now)); if let Err(err) = result { self.fail(Error::rtc(err)); } @@ -282,7 +298,14 @@ impl Agent { let dst = to_family(dst, self.socket_v6); match self.socket.try_send_to(data, dst) { Ok(len) => self.inbound.wire.sent(len), - Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => self.outbox.push_back((data.to_vec(), dst)), + // Changes can keep coming while the socket stays full, so the backlog is + // bounded like the inbound queue; ICE, SCTP, and RTP recover the loss. + Err(err) if err.kind() == std::io::ErrorKind::WouldBlock && self.outbox.len() < crate::INBOX => { + self.outbox.push_back((data.to_vec(), dst)) + } + Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => { + tracing::debug!(%dst, len = data.len(), "dropped a datagram: the socket is full") + } Err(err) => tracing::warn!(%err, %dst, "send failed"), } } @@ -299,7 +322,7 @@ impl Agent { tracing::debug!(%src, len = data.len(), "dropped unrecognized datagram"); return; }; - let result = self.rtc_mut().handle_input(Input::Receive(Instant::now(), receive)); + let result = self.change().handle_input(Input::Receive(Instant::now(), receive)); if let Err(err) = result { self.fail(Error::rtc(err)); } @@ -332,7 +355,7 @@ impl Agent { match Candidate::server_reflexive(mapped, query.base, "udp") { // None: redundant with a candidate the agent already has, as when there is no NAT. Ok(candidate) => { - let added = self.rtc_mut().add_local_candidate(candidate).cloned(); + let added = self.change().add_local_candidate(candidate).cloned(); if let Some(candidate) = added { self.events.push_back(Event::Candidate(candidate)); } @@ -455,6 +478,38 @@ mod tests { assert_eq!(wire.stats(), crate::Stats { tx: 73, rx: 11 }); } + /// An agent on a fresh socket, which tokio reports unwritable until the runtime + /// polls it, so every send waits in the outbox until the test yields. + fn unpolled() -> Agent { + let socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + socket.set_nonblocking(true).unwrap(); + let socket = Arc::new(UdpSocket::from_std(socket).unwrap()); + let local = socket.local_addr().unwrap(); + let inbound = crate::reader(socket.clone()); + Agent::new(Rtc::builder().build(Instant::now()), socket, vec![local], inbound).unwrap() + } + + #[tokio::test] + async fn outbox_is_bounded() { + let mut agent = unpolled(); + let dst = agent.locals[0]; + for _ in 0..crate::INBOX + 10 { + agent.send(&[1; 100], dst); + } + assert_eq!(agent.outbox.len(), crate::INBOX); + } + + #[tokio::test] + async fn awaiting_answer_holds_the_deadline() { + let mut agent = unpolled().awaiting_answer(); + assert_eq!(agent.started, None); + // The agent's own changes, such as timeouts and STUN, leave it held. + drop(agent.change()); + assert_eq!(agent.started, None); + drop(agent.rtc_mut()); + assert!(agent.started.is_some()); + } + #[test] fn mapped_addresses_round_trip() { let v4: SocketAddr = "1.2.3.4:5000".parse().unwrap(); diff --git a/rs/moq-tokio/src/webrtc.rs b/rs/moq-tokio/src/webrtc.rs index 1a53c8f7ba7..a06d9b3ce73 100644 --- a/rs/moq-tokio/src/webrtc.rs +++ b/rs/moq-tokio/src/webrtc.rs @@ -402,7 +402,11 @@ impl Peer { ) -> Result { let stun = resolve_stun(&config.stun).await; let inbound = moq_ice::reader(socket.clone()); - let agent = moq_ice::Agent::new(rtc, socket, locals, inbound)?.with_stun(&stun); + let mut agent = moq_ice::Agent::new(rtc, socket, locals, inbound)?.with_stun(&stun); + if let Role::Offerer { .. } = role { + // The answer applies through `rtc_mut`, which starts the deadline. + agent = agent.awaiting_answer(); + } let (commands, rx) = mpsc::unbounded_channel(); let (candidates_tx, candidates) = mpsc::unbounded_channel(); From 6eb1649a803a66740e3db36c7ee0fcc6b44bf7d0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 17:11:33 -0700 Subject: [PATCH 08/15] refactor(ice): bundle an agent's socket, candidates, and inbound queue `moq_ice::Agent::new` took four arguments. The UDP socket, its advertised host candidates, and the datagrams routed to the agent now travel together as `moq_ice::Socket`, so `Agent::new(rtc, socket)` and moq-rtc's `Session::ingest` / `Session::egress` take one value instead of three. The client `bind_udp` helpers in moq-tokio and moq-rtc return it directly. Co-Authored-By: Claude Opus 5.5 --- rs/moq-ice/src/agent.rs | 29 +++++++++++++++------- rs/moq-ice/src/lib.rs | 2 +- rs/moq-ice/src/wire.rs | 14 ++++++++++- rs/moq-rtc/src/client/whep.rs | 10 +++----- rs/moq-rtc/src/client/whip.rs | 10 +++----- rs/moq-rtc/src/server/whep.rs | 7 +++++- rs/moq-rtc/src/server/whip.rs | 7 +++++- rs/moq-rtc/src/session.rs | 46 +++++++++++++---------------------- rs/moq-tokio/src/webrtc.rs | 30 +++++++++++++---------- 9 files changed, 86 insertions(+), 69 deletions(-) diff --git a/rs/moq-ice/src/agent.rs b/rs/moq-ice/src/agent.rs index d3fdf54bfdb..03d73ae5f55 100644 --- a/rs/moq-ice/src/agent.rs +++ b/rs/moq-ice/src/agent.rs @@ -11,7 +11,7 @@ use stun_types::message::{BINDING, Message, MessageClass, MessageWriteVec, Trans use stun_types::prelude::*; use tokio::net::UdpSocket; -use crate::{Inbound, Wire}; +use crate::{Inbound, Socket, Wire}; /// Backstop deadline for reaching a connected ICE state, covering the one case /// str0m's agent deliberately never times out: a peer that provides no remote @@ -108,11 +108,13 @@ pub struct Agent { } impl Agent { - /// Drive `rtc`, sending from `socket` and receiving from `inbound`. - /// - /// `locals` are the host candidates advertised to the peer, not the socket's - /// bind address, and must not be empty. - pub fn new(rtc: Rtc, socket: Arc, locals: Vec, inbound: Inbound) -> std::io::Result { + /// Drive `rtc` over `socket`, which must have a host candidate. + pub fn new(rtc: Rtc, socket: Socket) -> std::io::Result { + let Socket { + udp: socket, + locals, + inbound, + } = socket; if locals.is_empty() { return Err(std::io::Error::new( std::io::ErrorKind::InvalidInput, @@ -462,7 +464,12 @@ mod tests { let local = socket.local_addr().unwrap(); let inbound = crate::reader(socket.clone()); let wire = inbound.wire.clone(); - let mut agent = Agent::new(Rtc::builder().build(Instant::now()), socket, vec![local], inbound).unwrap(); + let socket = Socket { + udp: socket, + locals: vec![local], + inbound, + }; + let mut agent = Agent::new(Rtc::builder().build(Instant::now()), socket).unwrap(); // Until tokio has seen the socket writable, a send waits in the outbox. agent.socket.writable().await.unwrap(); @@ -485,8 +492,12 @@ mod tests { socket.set_nonblocking(true).unwrap(); let socket = Arc::new(UdpSocket::from_std(socket).unwrap()); let local = socket.local_addr().unwrap(); - let inbound = crate::reader(socket.clone()); - Agent::new(Rtc::builder().build(Instant::now()), socket, vec![local], inbound).unwrap() + let socket = Socket { + inbound: crate::reader(socket.clone()), + udp: socket, + locals: vec![local], + }; + Agent::new(Rtc::builder().build(Instant::now()), socket).unwrap() } #[tokio::test] diff --git a/rs/moq-ice/src/lib.rs b/rs/moq-ice/src/lib.rs index da2e68a1520..af782457c8e 100644 --- a/rs/moq-ice/src/lib.rs +++ b/rs/moq-ice/src/lib.rs @@ -24,4 +24,4 @@ mod agent; mod wire; pub use agent::{Agent, Error, Event, RtcMut, canonical}; -pub use wire::{INBOX, Inbound, Packet, Stats, Wire, reader}; +pub use wire::{INBOX, Inbound, Packet, Socket, Stats, Wire, reader}; diff --git a/rs/moq-ice/src/wire.rs b/rs/moq-ice/src/wire.rs index 5eac3db8738..53c022371fd 100644 --- a/rs/moq-ice/src/wire.rs +++ b/rs/moq-ice/src/wire.rs @@ -15,6 +15,18 @@ pub type Packet = (Vec, SocketAddr); /// and a stalled session must not grow memory without limit. pub const INBOX: usize = 256; +/// The UDP socket an [`crate::Agent`] runs on: where it sends, what it receives, +/// and the host candidates it is reachable at. +pub struct Socket { + /// Sends datagrams, possibly shared by several agents (a demuxed server socket). + pub udp: Arc, + /// The host candidates advertised to the peer, not the socket's bind address. + /// Must not be empty. + pub locals: Vec, + /// The datagrams routed to this agent. + pub inbound: Inbound, +} + /// The datagrams routed to one [`crate::Agent`], and the counters they are tallied in. pub struct Inbound { /// Ends once every sender is dropped, which ends the agent. @@ -64,7 +76,7 @@ impl Wire { } /// Pump every datagram from a socket the caller owns 1:1 into a queue for -/// [`crate::Agent::new`], until the queue's receiver is dropped. +/// [`Socket::inbound`], until the queue's receiver is dropped. pub fn reader(socket: Arc) -> Inbound { let (tx, rx) = mpsc::channel(INBOX); let wire = Wire::default(); diff --git a/rs/moq-rtc/src/client/whep.rs b/rs/moq-rtc/src/client/whep.rs index 2851de3917d..1b148235f92 100644 --- a/rs/moq-rtc/src/client/whep.rs +++ b/rs/moq-rtc/src/client/whep.rs @@ -27,9 +27,9 @@ pub(crate) async fn dial( .with_bandwidth(client.config().bandwidth.clone()); let sink = Box::new(IngestSink::new(broadcast, config)?); - let (socket, candidates) = session::bind_udp(&client.config().ice_candidates).await?; + let socket = session::bind_udp(&client.config().ice_candidates).await?; let mut rtc = Rtc::new(Instant::now()); - for addr in &candidates { + for addr in &socket.locals { let cand = Candidate::host(*addr, "udp").map_err(Error::rtc)?; rtc.add_local_candidate(cand); } @@ -63,11 +63,7 @@ pub(crate) async fn dial( // The URL routinely carries the auth token in its query, or userinfo. tracing::info!(url = %moq_tokio::RedactedUrl::new(&url), "whep client connected"); - // 1:1 socket (no demux on the client): pump its datagrams into the session. - // The session tags each datagram with the advertised candidate matching its - // family (str0m matches the destination against a host candidate, not the bind). - let inbound = moq_ice::reader(socket.clone()); - let mut session = session::Session::ingest(rtc, socket, candidates, inbound, sink)?; + let mut session = session::Session::ingest(rtc, socket, sink)?; for mid in mids { session.open_media(mid)?; } diff --git a/rs/moq-rtc/src/client/whip.rs b/rs/moq-rtc/src/client/whip.rs index 563f3118b50..3727953664b 100644 --- a/rs/moq-rtc/src/client/whip.rs +++ b/rs/moq-rtc/src/client/whip.rs @@ -30,11 +30,11 @@ pub(crate) async fn dial( return Err(Error::NoRenditions); } - let (socket, candidates) = session::bind_udp(&client.config().ice_candidates).await?; + let socket = session::bind_udp(&client.config().ice_candidates).await?; // Restrict to codecs the catalog can actually source so the remote // answer doesn't pick a codec we have no rendition for. let mut rtc = session::rtc_with_codecs(&codecs); - for addr in &candidates { + for addr in &socket.locals { let cand = Candidate::host(*addr, "udp").map_err(Error::rtc)?; rtc.add_local_candidate(cand); } @@ -75,11 +75,7 @@ pub(crate) async fn dial( // The URL routinely carries the auth token in its query, or userinfo. tracing::info!(url = %moq_tokio::RedactedUrl::new(&url), "whip client connected"); - // 1:1 socket (no demux on the client): pump its datagrams into the session. - // The session tags each datagram with the advertised candidate matching its - // family (str0m matches the destination against a host candidate, not the bind). - let inbound = moq_ice::reader(socket.clone()); - let mut session = session::Session::egress(rtc, socket, candidates, inbound, source)?; + let mut session = session::Session::egress(rtc, socket, source)?; for mid in mids { session.open_media(mid)?; } diff --git a/rs/moq-rtc/src/server/whep.rs b/rs/moq-rtc/src/server/whep.rs index ca69a2c872f..c8915c19a3c 100644 --- a/rs/moq-rtc/src/server/whep.rs +++ b/rs/moq-rtc/src/server/whep.rs @@ -159,7 +159,12 @@ pub async fn accept( let answer = rtc.sdp_api().accept_offer(offer).map_err(Error::rtc)?; let resource_id = sdp::new_resource_id(); - let session = session::Session::egress(rtc, mux.socket(), mux.candidates().to_vec(), inbound, source)?; + let socket = moq_ice::Socket { + udp: mux.socket(), + locals: mux.candidates().to_vec(), + inbound, + }; + let session = session::Session::egress(rtc, socket, source)?; // Register before returning so a DELETE that races startup still finds the // session; Response::run unregisters itself when it ends. diff --git a/rs/moq-rtc/src/server/whip.rs b/rs/moq-rtc/src/server/whip.rs index 5f0241af3a5..425b0952f20 100644 --- a/rs/moq-rtc/src/server/whip.rs +++ b/rs/moq-rtc/src/server/whip.rs @@ -153,7 +153,12 @@ pub async fn accept( handle.announce(moq_net::origin::Route::default().with_epoch(epoch.clone()))?; let resource_id = sdp::new_resource_id(); - let session = session::Session::ingest(rtc, mux.socket(), mux.candidates().to_vec(), inbound, sink)?; + let socket = moq_ice::Socket { + udp: mux.socket(), + locals: mux.candidates().to_vec(), + inbound, + }; + let session = session::Session::ingest(rtc, socket, sink)?; // Register before returning so a DELETE that races the first packet still // finds the session; Response::run unregisters itself when it ends. diff --git a/rs/moq-rtc/src/session.rs b/rs/moq-rtc/src/session.rs index 88cdfbaa098..2021139e742 100644 --- a/rs/moq-rtc/src/session.rs +++ b/rs/moq-rtc/src/session.rs @@ -1,6 +1,6 @@ //! str0m session driver shared by every HTTP role / media direction. //! -//! [`moq_tokio::webrtc::moq_ice::Agent`] drives the sans-IO [`str0m::Rtc`] over UDP; +//! [`moq_ice::Agent`] drives the sans-IO [`str0m::Rtc`] over UDP; //! [`Session::run`] feeds it egress writes and handles the media events it surfaces. //! //! The session itself doesn't care whether the [`Rtc`] was populated by @@ -96,18 +96,11 @@ pub struct Session { } impl Session { - /// Convenience for the ingest case (WHIP server, WHEP client). `locals` are the - /// advertised ICE candidates, not the socket bind (see [`moq_ice::Agent::new`]). + /// Convenience for the ingest case (WHIP server, WHEP client). The socket's /// `inbound` comes from the server mux, or [`moq_ice::reader`] for a 1:1 client socket. - pub fn ingest( - rtc: Rtc, - socket: Arc, - locals: Vec, - inbound: moq_ice::Inbound, - sink: Box, - ) -> Result { + pub fn ingest(rtc: Rtc, socket: moq_ice::Socket, sink: Box) -> Result { Ok(Self { - agent: moq_ice::Agent::new(rtc, socket, locals, inbound)?, + agent: moq_ice::Agent::new(rtc, socket)?, role: MediaRole::Ingest(sink), opened_media: HashSet::new(), writes_rx: None, @@ -116,19 +109,12 @@ impl Session { }) } - /// Convenience for the egress case (WHEP server, WHIP client). `locals` are the - /// advertised ICE candidates, not the socket bind (see [`moq_ice::Agent::new`]). + /// Convenience for the egress case (WHEP server, WHIP client). The socket's /// `inbound` comes from the server mux, or [`moq_ice::reader`] for a 1:1 client socket. - pub fn egress( - rtc: Rtc, - socket: Arc, - locals: Vec, - inbound: moq_ice::Inbound, - mut source: EgressSource, - ) -> Result { + pub fn egress(rtc: Rtc, socket: moq_ice::Socket, mut source: EgressSource) -> Result { let writes_rx = source.take_writes(); Ok(Self { - agent: moq_ice::Agent::new(rtc, socket, locals, inbound)?, + agent: moq_ice::Agent::new(rtc, socket)?, role: MediaRole::Egress(Box::new(source)), opened_media: HashSet::new(), writes_rx: Some(writes_rx), @@ -514,19 +500,21 @@ pub fn rtc_with_codecs(codecs: &[str0m::format::Codec]) -> Rtc { rtc_config_with_codecs(codecs).build(std::time::Instant::now()) } -/// Bind an ephemeral UDP socket for a single client session and return it -/// (to share with its [`moq_ice::reader`]) plus the ICE candidates -/// to advertise. +/// Bind an ephemeral UDP socket for a single client session, read by its own +/// [`moq_ice::reader`], with the ICE candidates to advertise. /// /// The client paths are 1:1 (one socket per dialed session, no demux); the /// server paths share one socket via `crate::server::mux` instead. `advertise` /// IPs are used verbatim (reusing the bound port); empty falls back to the /// bound address, substituting loopback when that address is unspecified. -pub async fn bind_udp(advertise: &[SocketAddr]) -> Result<(Arc, Vec)> { - let socket = UdpSocket::bind(("0.0.0.0", 0)).await?; - let local = socket.local_addr()?; - let candidates = advertised_candidates(advertise, local)?; - Ok((Arc::new(socket), candidates)) +pub async fn bind_udp(advertise: &[SocketAddr]) -> Result { + let udp = Arc::new(UdpSocket::bind(("0.0.0.0", 0)).await?); + let locals = advertised_candidates(advertise, udp.local_addr()?)?; + Ok(moq_ice::Socket { + inbound: moq_ice::reader(udp.clone()), + udp, + locals, + }) } /// Pair configured ICE candidates with the bound UDP port and validate them. diff --git a/rs/moq-tokio/src/webrtc.rs b/rs/moq-tokio/src/webrtc.rs index a06d9b3ce73..88cef01d79e 100644 --- a/rs/moq-tokio/src/webrtc.rs +++ b/rs/moq-tokio/src/webrtc.rs @@ -267,8 +267,8 @@ impl Offerer { if protocol.is_empty() || protocol.contains(char::is_whitespace) { return Err(Error::Protocol(protocol.to_string())); } - let (socket, locals) = bind(&hosts)?; - let mut rtc = rtc(&locals)?; + let socket = bind(&hosts)?; + let mut rtc = rtc(&socket.locals)?; let mut api = rtc.sdp_api(); let id = api.add_channel_with_config(ChannelConfig { @@ -284,7 +284,7 @@ impl Offerer { channel: id, pending: Some(pending), }; - let peer = Peer::spawn(config, rtc, socket, locals, role, None).await?; + let peer = Peer::spawn(config, rtc, socket, role, None).await?; let offerer = Self { peer, protocol: protocol.to_string(), @@ -351,14 +351,14 @@ impl Answerer { let max_message = max_message_size(offer.as_str())?; let parsed = SdpOffer::from_sdp_string(&sdp).map_err(|err| Error::Sdp(err.to_string()))?; - let (socket, locals) = bind(&hosts)?; - let mut rtc = rtc(&locals)?; + let socket = bind(&hosts)?; + let mut rtc = rtc(&socket.locals)?; let answer = rtc .sdp_api() .accept_offer(parsed) .map_err(|err| Error::Sdp(err.to_string()))?; - let peer = Peer::spawn(config, rtc, socket, locals, Role::Answerer, Some(max_message)).await?; + let peer = Peer::spawn(config, rtc, socket, Role::Answerer, Some(max_message)).await?; peer.resolve(mdns); Ok((Self { peer }, Answer(answer.to_sdp_string()))) } @@ -395,14 +395,12 @@ impl Peer { async fn spawn( config: &Config, rtc: Rtc, - socket: Arc, - locals: Vec, + socket: moq_ice::Socket, role: Role, max_message: Option, ) -> Result { let stun = resolve_stun(&config.stun).await; - let inbound = moq_ice::reader(socket.clone()); - let mut agent = moq_ice::Agent::new(rtc, socket, locals, inbound)?.with_stun(&stun); + let mut agent = moq_ice::Agent::new(rtc, socket)?.with_stun(&stun); if let Role::Offerer { .. } = role { // The answer applies through `rtc_mut`, which starts the deadline. agent = agent.awaiting_answer(); @@ -812,8 +810,9 @@ fn hosts() -> Result, Error> { Ok(hosts) } -/// Bind a dual-stack socket on an ephemeral port, pairing it with each host address. -fn bind(hosts: &[IpAddr]) -> Result<(Arc, Vec), Error> { +/// Bind a dual-stack socket on an ephemeral port, pairing it with each host +/// address, and start reading it. +fn bind(hosts: &[IpAddr]) -> Result { let socket = crate::bind::udp(crate::bind::Udp::new("[::]:0".parse().unwrap())) .or_else(|_| crate::bind::udp(crate::bind::Udp::new("0.0.0.0:0".parse().unwrap())))?; socket.set_nonblocking(true)?; @@ -828,7 +827,12 @@ fn bind(hosts: &[IpAddr]) -> Result<(Arc, Vec), Error> { if locals.is_empty() { return Err(Error::NoHost); } - Ok((Arc::new(socket), locals)) + let udp = Arc::new(socket); + Ok(moq_ice::Socket { + inbound: moq_ice::reader(udp.clone()), + udp, + locals, + }) } /// Look up each STUN server. One that does not resolve is skipped, since a missing From 06276d3dd9ba2d5acc8c0c815dc68704772b72bc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 17:31:28 -0700 Subject: [PATCH 09/15] chore(rtc): sort moq-rtc dependencies Co-Authored-By: Claude Opus 5.5 --- rs/moq-rtc/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/moq-rtc/Cargo.toml b/rs/moq-rtc/Cargo.toml index f8c637637ad..cd79d6ef3cf 100644 --- a/rs/moq-rtc/Cargo.toml +++ b/rs/moq-rtc/Cargo.toml @@ -26,10 +26,10 @@ server = ["dep:axum"] axum = { workspace = true, optional = true } bytes = { workspace = true } hang = { workspace = true } -moq-mux = { workspace = true } -moq-net = { workspace = true } # Drives each str0m connection over its UDP socket. moq-ice = { workspace = true } +moq-mux = { workspace = true } +moq-net = { workspace = true } # `RedactedUrl` keeps a dialed URL's query token and userinfo out of the logs. moq-tokio = { workspace = true } reqwest = { workspace = true, features = ["rustls"] } From c0111e2fd9f672cb91257bd22c5435ac8f7b78c5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 17:41:33 -0700 Subject: [PATCH 10/15] fix(webrtc): end reads on zero-cost records and gate IPv4 on dual-stack An empty or error record cost nothing against the inbound byte budget, so a peer could queue them without bound. Empty records are malformed qmux, and both now end the session's reads and close the channel. A [::] socket that stayed v6-only no longer advertises IPv4 host candidates. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/webrtc.rs | 6 +++-- rs/moq-tokio/src/webrtc/channel.rs | 35 +++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/rs/moq-tokio/src/webrtc.rs b/rs/moq-tokio/src/webrtc.rs index 88cef01d79e..b083b48efb9 100644 --- a/rs/moq-tokio/src/webrtc.rs +++ b/rs/moq-tokio/src/webrtc.rs @@ -670,7 +670,7 @@ impl Driver { Ok(Bytes::from(data.data)) }; if !open.reads.push(record) { - tracing::debug!("closing the data channel: the peer ignored qmux flow control"); + tracing::debug!("closing the data channel: the peer broke qmux framing or flow control"); let id = open.id; self.agent.rtc_mut().direct_api().close_data_channel(id); self.channel = Some(Err(Error::Closed)); @@ -816,12 +816,14 @@ fn bind(hosts: &[IpAddr]) -> Result { let socket = crate::bind::udp(crate::bind::Udp::new("[::]:0".parse().unwrap())) .or_else(|_| crate::bind::udp(crate::bind::Udp::new("0.0.0.0:0".parse().unwrap())))?; socket.set_nonblocking(true)?; + // A `[::]` socket that stayed v6-only can't send to IPv4 candidates. + let ipv4 = socket.local_addr()?.is_ipv4() || crate::bind::udp_is_dual_stack(&socket); let socket = UdpSocket::from_std(socket)?; let local = socket.local_addr()?; let locals: Vec = hosts .iter() - .filter(|ip| local.is_ipv6() || ip.is_ipv4()) + .filter(|ip| if ip.is_ipv4() { ipv4 } else { local.is_ipv6() }) .map(|&ip| SocketAddr::new(ip, local.port())) .collect(); if locals.is_empty() { diff --git a/rs/moq-tokio/src/webrtc/channel.rs b/rs/moq-tokio/src/webrtc/channel.rs index 86583f8e358..e5c1c7ca1a6 100644 --- a/rs/moq-tokio/src/webrtc/channel.rs +++ b/rs/moq-tokio/src/webrtc/channel.rs @@ -77,16 +77,29 @@ pub(crate) struct Inbox { } impl Inbox { - /// Queue a record for the session, or end its reads with a flow control error - /// and return false once that would leave more than the budget unread. + /// Queue a record for the session, returning false once its reads have ended: + /// on an error, or with a flow control error when the record would leave more + /// than the budget unread. pub fn push(&self, record: Result) -> bool { - let len = record.as_ref().map_or(0, |data| data.len() as u64); + // A closed reader drops the record. + let data = match record { + Ok(data) => data, + Err(err) => { + let _ = self.tx.send(Err(err)); + return false; + } + }; + // qmux records are never empty, so the byte budget also bounds their count. + if data.is_empty() { + let _ = self.tx.send(Err(qmux::Error::ProtocolViolation)); + return false; + } + let len = data.len() as u64; if self.unread.fetch_add(len, Ordering::Relaxed) + len > self.budget { let _ = self.tx.send(Err(qmux::Error::FlowControlError)); return false; } - // A closed reader drops the record. - let _ = self.tx.send(record); + let _ = self.tx.send(Ok(data)); true } } @@ -126,4 +139,16 @@ mod tests { assert_eq!(reader.recv().await.unwrap()[0], 2); assert!(matches!(reader.recv().await, Err(qmux::Error::FlowControlError))); } + + #[tokio::test] + async fn queue_ends_on_empty_or_error_records() { + // Neither costs any bytes, so the budget alone would never stop a flood of them. + let (inbox, mut reader) = queue(10); + assert!(!inbox.push(Ok(Bytes::new()))); + assert!(matches!(reader.recv().await, Err(qmux::Error::ProtocolViolation))); + + let (inbox, mut reader) = queue(10); + assert!(!inbox.push(Err(qmux::Error::FrameTooLarge))); + assert!(matches!(reader.recv().await, Err(qmux::Error::FrameTooLarge))); + } } From 4b96283039df97c7173e395432c5b75e875f72fd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 17:54:30 -0700 Subject: [PATCH 11/15] fix(webrtc): reject STUN hosts with characters a DNS name can't hold Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/webrtc.rs | 8 +++++++- rs/moq-tokio/src/webrtc/tests.rs | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/rs/moq-tokio/src/webrtc.rs b/rs/moq-tokio/src/webrtc.rs index b083b48efb9..50242ffe52d 100644 --- a/rs/moq-tokio/src/webrtc.rs +++ b/rs/moq-tokio/src/webrtc.rs @@ -180,7 +180,13 @@ impl FromStr for Stun { Some((host, port)) => (host, Some(port)), None => (rest, None), }; - if host.is_empty() || host.contains(['/', '?', '@', '[', ']']) { + // A DNS name or IPv4 address, so a fragment or stray character fails here + // instead of as a lookup that only warns. + if host.is_empty() + || !host + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '.' | '_')) + { return Err(invalid()); } (host.to_string(), port) diff --git a/rs/moq-tokio/src/webrtc/tests.rs b/rs/moq-tokio/src/webrtc/tests.rs index 51241bf1bae..9e5c1f211f5 100644 --- a/rs/moq-tokio/src/webrtc/tests.rs +++ b/rs/moq-tokio/src/webrtc/tests.rs @@ -258,6 +258,9 @@ fn stun_urls() { "stun:host:3478?transport=udp", "stun:[::1]x", "stun:[nope]", + "stun:server.example#fragment", + "stun:server example", + "stun:server\texample", "stun.example.com", ] { assert_eq!(parse(bad), None, "{bad}"); From 5a9984772fdb510a03cbe7379b8890f5dc8cff79 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 19:05:04 -0700 Subject: [PATCH 12/15] fix(ice): hold the ICE deadline until gathering ends The 30 s establishment backstop started at `Agent::new` for an answerer, so a caller that sends its answer only after the last candidate lost the driver to an unreachable STUN server's ~39.5 s retransmit schedule. The deadline now counts from the later of gathering's end and, for an offer, the answer applying. Gathering always ends, so nothing can hang on it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-ice/src/agent.rs | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/rs/moq-ice/src/agent.rs b/rs/moq-ice/src/agent.rs index 03d73ae5f55..20f310f2e53 100644 --- a/rs/moq-ice/src/agent.rs +++ b/rs/moq-ice/src/agent.rs @@ -19,6 +19,11 @@ use crate::{Inbound, Socket, Wire}; /// tried and exhausted (after ~21s at its defaults), but with no remote candidates /// it treats the session as still possible forever, since trickle ICE could add /// one. Set above str0m's own exhaustion so a slow but real peer is not clipped. +/// +/// It counts from when the peer could hold every local candidate: once gathering +/// ends, so a caller that sends its description only after the last candidate is +/// not clipped by an unreachable STUN server's retransmissions, and for an +/// [`Agent::awaiting_answer`], once the answer applies. const ESTABLISH_TIMEOUT: Duration = Duration::from_secs(30); /// RFC 8489 section 6.2.1: the first retransmission timeout of a STUN request. @@ -94,7 +99,8 @@ pub struct Agent { locals: Vec, inbound: Inbound, stun: Vec, - gathered: bool, + /// When every STUN query answered or gave up, once [`Event::Gathered`] went out. + gathered: Option, events: VecDeque, /// Why the connection ended, found while draining and reported by the next [`Agent::next`]. failed: Option, @@ -130,7 +136,7 @@ impl Agent { locals, inbound, stun: Vec::new(), - gathered: false, + gathered: None, events: VecDeque::new(), failed: None, timeout: now, @@ -210,8 +216,8 @@ impl Agent { if let Some(err) = self.failed.take() { return Err(err); } - if !self.gathered && self.stun.is_empty() { - self.gathered = true; + if self.gathered.is_none() && self.stun.is_empty() { + self.gathered = Some(Instant::now()); return Ok(Event::Gathered); } @@ -228,10 +234,7 @@ impl Agent { if !self.rtc.is_alive() { return Err(Error::Closed); } - let establish = self - .started - .filter(|_| !self.connected) - .map(|at| at + ESTABLISH_TIMEOUT); + let establish = self.establish(); if establish.is_some_and(|at| at <= Instant::now()) { return Err(Error::Timeout); } @@ -263,6 +266,14 @@ impl Agent { } } + /// When [`ESTABLISH_TIMEOUT`] expires, or `None` while it is held or ICE connected. + fn establish(&self) -> Option { + match (self.started, self.gathered) { + (Some(started), Some(gathered)) if !self.connected => Some(started.max(gathered) + ESTABLISH_TIMEOUT), + _ => None, + } + } + /// Poll str0m until it asks for a timeout: transmits are sent, events queued. fn drain(&mut self) { while self.failed.is_none() { @@ -521,6 +532,18 @@ mod tests { assert!(agent.started.is_some()); } + #[tokio::test] + async fn gathering_holds_the_deadline() { + let mut agent = unpolled().with_stun(&["127.0.0.1:3478".parse().unwrap()]); + // An unreachable STUN server retransmits for longer than the deadline, so the + // deadline waits for gathering to end. + assert_eq!(agent.establish(), None); + agent.stun.clear(); + assert!(matches!(agent.next().await, Ok(Event::Gathered))); + let gathered = agent.gathered.unwrap(); + assert_eq!(agent.establish(), Some(gathered + ESTABLISH_TIMEOUT)); + } + #[test] fn mapped_addresses_round_trip() { let v4: SocketAddr = "1.2.3.4:5000".parse().unwrap(); From 9c59474c258074558f8ab4139520611d6630f283 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 19:16:13 -0700 Subject: [PATCH 13/15] fix(tokio): give a webrtc-only build the qmux send-group impls Main's send groups gate qmux's `Grouped` impls on `websocket` or `tcp`, so `--features webrtc` alone no longer compiled. They now follow every feature that pulls in qmux. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/transport.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/moq-tokio/src/transport.rs b/rs/moq-tokio/src/transport.rs index 79cee4f9a16..f25b6525baf 100644 --- a/rs/moq-tokio/src/transport.rs +++ b/rs/moq-tokio/src/transport.rs @@ -897,10 +897,10 @@ impl GroupedSend for web_transport_iroh::SendStream { } } -#[cfg(any(feature = "websocket", feature = "tcp"))] +#[cfg(any(feature = "websocket", feature = "tcp", feature = "webrtc"))] impl Grouped for qmux::Session {} -#[cfg(any(feature = "websocket", feature = "tcp"))] +#[cfg(any(feature = "websocket", feature = "tcp", feature = "webrtc"))] impl GroupedSend for qmux::SendStream { type Group = moq_net::transport::poll::Ungrouped; From 39e9ab35bd2f5c4e8abffce2ada0c5616d0a6e08 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 20:00:00 -0700 Subject: [PATCH 14/15] fix(relay): map WebRTC requests into relay authentication `request_for` panicked on `Transport::WebRtc`, so an embedder feeding `Listener::accept_webrtc` requests through relay admission crashed. It now maps to `moq_auth::Transport::WebRtc`, whose doc widens to cover data channel sessions. Co-Authored-By: Claude Opus 5.5 --- rs/moq-auth/src/request.rs | 2 +- rs/moq-relay/src/auth.rs | 44 ++++++++++++++++++++++++++++---------- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/rs/moq-auth/src/request.rs b/rs/moq-auth/src/request.rs index f16c97594fa..77f06509ab3 100644 --- a/rs/moq-auth/src/request.rs +++ b/rs/moq-auth/src/request.rs @@ -146,7 +146,7 @@ pub enum Transport { Rtmp, /// An SRT gateway session. Srt, - /// A WebRTC gateway session, including WHIP and WHEP. + /// A WebRTC session: a WHIP or WHEP gateway, or MoQ over a data channel. WebRtc, /// A WebTransport connection over HTTP/3. WebTransport, diff --git a/rs/moq-relay/src/auth.rs b/rs/moq-relay/src/auth.rs index e5287ac558b..505d8132cc3 100644 --- a/rs/moq-relay/src/auth.rs +++ b/rs/moq-relay/src/auth.rs @@ -914,17 +914,7 @@ impl Admission { /// The `moq_auth::Request` for an accepted transport request: every fact the /// transport knows, nothing parsed on the server's behalf. pub fn request_for(auth: &Auth, request: &moq_tokio::server::Request) -> Request { - let transport = match request.transport() { - moq_tokio::Transport::Quic => moq_auth::Transport::Quic, - moq_tokio::Transport::WebTransport => moq_auth::Transport::WebTransport, - moq_tokio::Transport::Iroh => moq_auth::Transport::Iroh, - moq_tokio::Transport::WebSocket => moq_auth::Transport::WebSocket, - moq_tokio::Transport::Tcp => moq_auth::Transport::Tcp, - moq_tokio::Transport::Unix => moq_auth::Transport::Unix, - // A transport this build does not know is still a session on the wire; the - // server sees the same facts either way. - other => unreachable!("unknown transport {other}"), - }; + let transport = transport(request.transport()); // A URL-less transport reports its root as empty; the contract says what was dialed. let path = match request.path() { "" => "/".to_string(), @@ -948,6 +938,22 @@ pub fn request_for(auth: &Auth, request: &moq_tokio::server::Request) -> Request out } +/// The auth transport for a session's transport. +fn transport(transport: moq_tokio::Transport) -> moq_auth::Transport { + match transport { + moq_tokio::Transport::Quic => moq_auth::Transport::Quic, + moq_tokio::Transport::WebTransport => moq_auth::Transport::WebTransport, + moq_tokio::Transport::Iroh => moq_auth::Transport::Iroh, + moq_tokio::Transport::WebSocket => moq_auth::Transport::WebSocket, + moq_tokio::Transport::Tcp => moq_auth::Transport::Tcp, + moq_tokio::Transport::Unix => moq_auth::Transport::Unix, + moq_tokio::Transport::WebRtc => moq_auth::Transport::WebRtc, + // A transport this build does not know is still a session on the wire; the + // server sees the same facts either way. + other => unreachable!("unknown transport {other}"), + } +} + /// The auth role for a session role, `None` for one `moq-auth` cannot name yet. pub(crate) fn role(role: moq_net::Role) -> Option { match role { @@ -985,6 +991,22 @@ mod tests { } } + #[test] + fn every_transport_has_an_auth_name() { + use moq_tokio::Transport as T; + for transport in [ + T::Quic, + T::WebTransport, + T::Iroh, + T::WebSocket, + T::Tcp, + T::Unix, + T::WebRtc, + ] { + assert_eq!(super::transport(transport).as_str(), transport.as_str()); + } + } + #[test] fn exactly_one_source() { assert!(config(None, &[]).validate(false).is_err()); From 6de08b34d769ec55c882c63ba19bce72608e82b4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 21:11:40 -0700 Subject: [PATCH 15/15] fix(webrtc): hold the answerer to the signaled protocol `Answerer::new` had no way to learn the ALPN the peers agreed, so `accept_webrtc` took any channel protocol the server speaks. It now takes the protocol, as `Offerer::new` and `@moq/p2p`'s accept do, and refuses a channel that carries a different one. Also pin that building a peer queues no str0m output, so the mutations made before the agent's first drain stay equivalent to draining between them. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/server.rs | 9 ++-- rs/moq-tokio/src/webrtc.rs | 43 +++++++++++++------ rs/moq-tokio/src/webrtc/tests.rs | 73 ++++++++++++++++++++++++++++++-- 3 files changed, 105 insertions(+), 20 deletions(-) diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 5dfa4bf0395..8378a1c3176 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -882,21 +882,24 @@ impl Listener { /// [`crate::webrtc::Answerer`], leaving admission to the returned [`Request`]. /// /// Waits for ICE and the channel to come up, bounded by the handshake timeout. - /// The channel's protocol must be a MoQ ALPN this server speaks. Returns - /// [`Error::Stopped`] once the listener is closing. + /// The channel must carry the answerer's protocol, a MoQ ALPN this server + /// speaks. Returns [`Error::Stopped`] once the listener is closing. #[cfg(feature = "webrtc")] pub async fn accept_webrtc(&self, peer: &crate::webrtc::Answerer) -> crate::Result { use crate::webrtc::Error as WebRtcError; let server = &self.server; let active = server.activity.enter().ok_or(Error::Stopped)?; + if !server.versions.alpns().contains(&peer.protocol()) { + return Err(WebRtcError::Protocol(peer.protocol().to_string()).into()); + } let deadline = server .timeout .and_then(|timeout| tokio::time::Instant::now().checked_add(timeout)); let handshake = async { let opened = peer.channel().await?; - if !server.versions.alpns().contains(&opened.protocol.as_str()) { + if opened.protocol != peer.protocol() { return Err(WebRtcError::Protocol(opened.protocol)); } let config = opened.config; diff --git a/rs/moq-tokio/src/webrtc.rs b/rs/moq-tokio/src/webrtc.rs index 50242ffe52d..5664d861797 100644 --- a/rs/moq-tokio/src/webrtc.rs +++ b/rs/moq-tokio/src/webrtc.rs @@ -105,7 +105,7 @@ pub enum Error { #[error("the data channel is already in use")] Taken, - /// The channel's protocol is not a MoQ ALPN this endpoint speaks. + /// The protocol is not a MoQ ALPN this endpoint speaks, or the channel's is not the one signaling agreed. #[error("unsupported MoQ protocol: {0:?}")] Protocol(String), @@ -270,9 +270,7 @@ impl Offerer { } async fn with_hosts(config: &Config, protocol: &str, hosts: Vec) -> Result<(Self, Offer), Error> { - if protocol.is_empty() || protocol.contains(char::is_whitespace) { - return Err(Error::Protocol(protocol.to_string())); - } + let protocol = negotiated(protocol)?; let socket = bind(&hosts)?; let mut rtc = rtc(&socket.locals)?; @@ -291,10 +289,7 @@ impl Offerer { pending: Some(pending), }; let peer = Peer::spawn(config, rtc, socket, role, None).await?; - let offerer = Self { - peer, - protocol: protocol.to_string(), - }; + let offerer = Self { peer, protocol }; Ok((offerer, Offer(offer.to_sdp_string()))) } @@ -344,15 +339,24 @@ impl Offerer { /// handle and any session over its data channel are gone. pub struct Answerer { peer: Peer, + /// The MoQ ALPN the remote peer's channel must carry. + protocol: String, } impl Answerer { - /// Start a peer connection for the remote peer's offer, returning the answer to send. - pub async fn new(config: &Config, offer: &Offer) -> Result<(Self, Answer), Error> { - Self::with_hosts(config, offer, hosts()?).await + /// Start a peer connection for the remote peer's offer and MoQ `protocol`, the + /// ALPN the peers agreed through signaling, returning the answer to send. + pub async fn new(config: &Config, offer: &Offer, protocol: &str) -> Result<(Self, Answer), Error> { + Self::with_hosts(config, offer, protocol, hosts()?).await } - async fn with_hosts(config: &Config, offer: &Offer, hosts: Vec) -> Result<(Self, Answer), Error> { + async fn with_hosts( + config: &Config, + offer: &Offer, + protocol: &str, + hosts: Vec, + ) -> Result<(Self, Answer), Error> { + let protocol = negotiated(protocol)?; let (sdp, mdns) = split_mdns(offer.as_str()); let max_message = max_message_size(offer.as_str())?; let parsed = SdpOffer::from_sdp_string(&sdp).map_err(|err| Error::Sdp(err.to_string()))?; @@ -366,7 +370,7 @@ impl Answerer { let peer = Peer::spawn(config, rtc, socket, Role::Answerer, Some(max_message)).await?; peer.resolve(mdns); - Ok((Self { peer }, Answer(answer.to_sdp_string()))) + Ok((Self { peer, protocol }, Answer(answer.to_sdp_string()))) } /// The next local candidate to send the remote peer, or `None` once gathering is done. @@ -384,12 +388,25 @@ impl Answerer { self.peer.add_candidate(candidate).await } + /// The MoQ ALPN the remote peer's data channel must carry. + pub(crate) fn protocol(&self) -> &str { + &self.protocol + } + /// Wait for the remote peer's data channel to open, and take it. pub(crate) async fn channel(&self) -> Result { self.peer.channel().await } } +/// Check a MoQ ALPN agreed through signaling, as `@moq/p2p` does. +fn negotiated(protocol: &str) -> Result { + match protocol.is_empty() || protocol.contains(char::is_whitespace) { + true => Err(Error::Protocol(protocol.to_string())), + false => Ok(protocol.to_string()), + } +} + /// The signaling handle both roles share: commands to the driver task and the /// candidates it gathers. struct Peer { diff --git a/rs/moq-tokio/src/webrtc/tests.rs b/rs/moq-tokio/src/webrtc/tests.rs index 9e5c1f211f5..b3afab026bb 100644 --- a/rs/moq-tokio/src/webrtc/tests.rs +++ b/rs/moq-tokio/src/webrtc/tests.rs @@ -77,7 +77,9 @@ async fn transfer(offerer: &Offerer, answerer: &Answerer, payload: &[u8]) { async fn session_over_loopback() { let config = Config::default(); let (offerer, offer) = Offerer::with_hosts(&config, protocol(), vec![LOOPBACK]).await.unwrap(); - let (answerer, answer) = Answerer::with_hosts(&config, &offer, vec![LOOPBACK]).await.unwrap(); + let (answerer, answer) = Answerer::with_hosts(&config, &offer, protocol(), vec![LOOPBACK]) + .await + .unwrap(); offerer.accept(&answer).await.unwrap(); transfer(&offerer, &answerer, b"hello").await; @@ -89,7 +91,9 @@ async fn session_over_loopback() { async fn large_frame_over_loopback() { let config = Config::default(); let (offerer, offer) = Offerer::with_hosts(&config, protocol(), vec![LOOPBACK]).await.unwrap(); - let (answerer, answer) = Answerer::with_hosts(&config, &offer, vec![LOOPBACK]).await.unwrap(); + let (answerer, answer) = Answerer::with_hosts(&config, &offer, protocol(), vec![LOOPBACK]) + .await + .unwrap(); offerer.accept(&answer).await.unwrap(); let payload: Vec = (0..1024 * 1024).map(|i| i as u8).collect(); @@ -100,7 +104,9 @@ async fn large_frame_over_loopback() { async fn channels() -> (Offerer, Answerer, Opened, Opened) { let config = Config::default(); let (offerer, offer) = Offerer::with_hosts(&config, protocol(), vec![LOOPBACK]).await.unwrap(); - let (answerer, answer) = Answerer::with_hosts(&config, &offer, vec![LOOPBACK]).await.unwrap(); + let (answerer, answer) = Answerer::with_hosts(&config, &offer, protocol(), vec![LOOPBACK]) + .await + .unwrap(); offerer.accept(&answer).await.unwrap(); let (sent, received) = tokio::time::timeout(TIMEOUT, async { tokio::join!(offerer.channel(), answerer.channel()) }) .await @@ -203,7 +209,7 @@ async fn server_reflexive_candidate_is_offered_and_selected() { let private = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 2)); let (mut offerer, offer) = Offerer::with_hosts(&config, protocol(), vec![private]).await.unwrap(); let offer = Offer::new(strip_candidates(offer.as_str())); - let (answerer, answer) = Answerer::with_hosts(&Config::default(), &offer, vec![LOOPBACK]) + let (answerer, answer) = Answerer::with_hosts(&Config::default(), &offer, protocol(), vec![LOOPBACK]) .await .unwrap(); offerer @@ -240,6 +246,35 @@ async fn unsupported_protocol_is_refused() { )); } +/// A channel carrying another supported ALPN than the one signaling agreed is refused. +#[tokio::test] +async fn unsignaled_protocol_is_refused() { + let versions = crate::connect::Config::default().versions(); + let [agreed, other, ..] = versions.alpns()[..] else { + panic!("expected two supported ALPNs"); + }; + let config = Config::default(); + let (offerer, offer) = Offerer::with_hosts(&config, other, vec![LOOPBACK]).await.unwrap(); + let (answerer, answer) = Answerer::with_hosts(&config, &offer, agreed, vec![LOOPBACK]) + .await + .unwrap(); + offerer.accept(&answer).await.unwrap(); + + let listener = crate::listen::Config::default() + .init_streams() + .unwrap() + .listen() + .await + .unwrap(); + let accepted = tokio::time::timeout(TIMEOUT, listener.accept_webrtc(&answerer)) + .await + .expect("the channel did not open"); + assert!(matches!( + accepted, + Err(crate::Error::WebRtc(err)) if matches!(&*err, Error::Protocol(protocol) if protocol == other) + )); +} + #[test] fn stun_urls() { let parse = |url: &str| url.parse::().map(|stun| stun.to_string()).ok(); @@ -297,3 +332,33 @@ async fn ip_candidates_parse_without_mdns() { assert_eq!(candidate.addr(), "192.168.1.2:54321".parse().unwrap()); assert!(parse_candidate("candidate:2 1 udp nope").await.is_err()); } + +/// Building a peer mutates its `Rtc` several times before the agent's first drain: +/// a host candidate per address, then the channel or the answer. str0m wants a +/// drain after each, which is a no-op while none of them queues output; pin that, +/// so a str0m upgrade that changes it fails here. +#[test] +fn construction_queues_no_output() { + fn idle(rtc: &mut Rtc) { + assert!(matches!(rtc.poll_output(), Ok(str0m::Output::Timeout(_)))); + } + fn host(rtc: &mut Rtc, addr: &str) { + let candidate = str0m::Candidate::host(addr.parse().unwrap(), "udp").unwrap(); + rtc.add_local_candidate(candidate); + idle(rtc); + } + + let mut offerer = Rtc::builder().build(Instant::now()); + host(&mut offerer, "127.0.0.1:5000"); + host(&mut offerer, "[::1]:5000"); + let mut api = offerer.sdp_api(); + api.add_channel(LABEL.to_string()); + let (offer, _pending) = api.apply().unwrap(); + idle(&mut offerer); + + let mut answerer = Rtc::builder().build(Instant::now()); + host(&mut answerer, "127.0.0.1:5001"); + host(&mut answerer, "[::1]:5001"); + answerer.sdp_api().accept_offer(offer).unwrap(); + idle(&mut answerer); +}